1/* 2 * Copyright (c) 2001-2004 by David Brownell 3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License as published by the 7 * Free Software Foundation; either version 2 of the License, or (at your 8 * option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software Foundation, 17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 18 */ 19 20/* this file is part of ehci-hcd.c */ 21 22/*-------------------------------------------------------------------------*/ 23 24/* 25 * EHCI scheduled transaction support: interrupt, iso, split iso 26 * These are called "periodic" transactions in the EHCI spec. 27 * 28 * Note that for interrupt transfers, the QH/QTD manipulation is shared 29 * with the "asynchronous" transaction support (control/bulk transfers). 30 * The only real difference is in how interrupt transfers are scheduled. 31 * 32 * For ISO, we make an "iso_stream" head to serve the same role as a QH. 33 * It keeps track of every ITD (or SITD) that's linked, and holds enough 34 * pre-calculated schedule data to make appending to the queue be quick. 35 */ 36 37static int ehci_get_frame (struct usb_hcd *hcd); 38 39/*-------------------------------------------------------------------------*/ 40 41/* 42 * periodic_next_shadow - return "next" pointer on shadow list 43 * @periodic: host pointer to qh/itd/sitd 44 * @tag: hardware tag for type of this record 45 */ 46static union ehci_shadow * 47periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic, 48 __hc32 tag) 49{ 50 switch (hc32_to_cpu(ehci, tag)) { 51 case Q_TYPE_QH: 52 return &periodic->qh->qh_next; 53 case Q_TYPE_FSTN: 54 return &periodic->fstn->fstn_next; 55 case Q_TYPE_ITD: 56 return &periodic->itd->itd_next; 57 // case Q_TYPE_SITD: 58 default: 59 return &periodic->sitd->sitd_next; 60 } 61} 62 63static __hc32 * 64shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic, 65 __hc32 tag) 66{ 67 switch (hc32_to_cpu(ehci, tag)) { 68 /* our ehci_shadow.qh is actually software part */ 69 case Q_TYPE_QH: 70 return &periodic->qh->hw->hw_next; 71 /* others are hw parts */ 72 default: 73 return periodic->hw_next; 74 } 75} 76 77/* caller must hold ehci->lock */ 78static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr) 79{ 80 union ehci_shadow *prev_p = &ehci->pshadow[frame]; 81 __hc32 *hw_p = &ehci->periodic[frame]; 82 union ehci_shadow here = *prev_p; 83 84 /* find predecessor of "ptr"; hw and shadow lists are in sync */ 85 while (here.ptr && here.ptr != ptr) { 86 prev_p = periodic_next_shadow(ehci, prev_p, 87 Q_NEXT_TYPE(ehci, *hw_p)); 88 hw_p = shadow_next_periodic(ehci, &here, 89 Q_NEXT_TYPE(ehci, *hw_p)); 90 here = *prev_p; 91 } 92 /* an interrupt entry (at list end) could have been shared */ 93 if (!here.ptr) 94 return; 95 96 /* update shadow and hardware lists ... the old "next" pointers 97 * from ptr may still be in use, the caller updates them. 98 */ 99 *prev_p = *periodic_next_shadow(ehci, &here, 100 Q_NEXT_TYPE(ehci, *hw_p)); 101 *hw_p = *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p)); 102} 103 104/* how many of the uframe's 125 usecs are allocated? */ 105static unsigned short 106periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe) 107{ 108 __hc32 *hw_p = &ehci->periodic [frame]; 109 union ehci_shadow *q = &ehci->pshadow [frame]; 110 unsigned usecs = 0; 111 struct ehci_qh_hw *hw; 112 113 while (q->ptr) { 114 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 115 case Q_TYPE_QH: 116 hw = q->qh->hw; 117 /* is it in the S-mask? */ 118 if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe)) 119 usecs += q->qh->usecs; 120 /* ... or C-mask? */ 121 if (hw->hw_info2 & cpu_to_hc32(ehci, 122 1 << (8 + uframe))) 123 usecs += q->qh->c_usecs; 124 hw_p = &hw->hw_next; 125 q = &q->qh->qh_next; 126 break; 127 // case Q_TYPE_FSTN: 128 default: 129 /* for "save place" FSTNs, count the relevant INTR 130 * bandwidth from the previous frame 131 */ 132 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) { 133 ehci_dbg (ehci, "ignoring FSTN cost ...\n"); 134 } 135 hw_p = &q->fstn->hw_next; 136 q = &q->fstn->fstn_next; 137 break; 138 case Q_TYPE_ITD: 139 if (q->itd->hw_transaction[uframe]) 140 usecs += q->itd->stream->usecs; 141 hw_p = &q->itd->hw_next; 142 q = &q->itd->itd_next; 143 break; 144 case Q_TYPE_SITD: 145 /* is it in the S-mask? (count SPLIT, DATA) */ 146 if (q->sitd->hw_uframe & cpu_to_hc32(ehci, 147 1 << uframe)) { 148 if (q->sitd->hw_fullspeed_ep & 149 cpu_to_hc32(ehci, 1<<31)) 150 usecs += q->sitd->stream->usecs; 151 else /* worst case for OUT start-split */ 152 usecs += HS_USECS_ISO (188); 153 } 154 155 /* ... C-mask? (count CSPLIT, DATA) */ 156 if (q->sitd->hw_uframe & 157 cpu_to_hc32(ehci, 1 << (8 + uframe))) { 158 /* worst case for IN complete-split */ 159 usecs += q->sitd->stream->c_usecs; 160 } 161 162 hw_p = &q->sitd->hw_next; 163 q = &q->sitd->sitd_next; 164 break; 165 } 166 } 167#ifdef DEBUG 168 if (usecs > 100) 169 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n", 170 frame * 8 + uframe, usecs); 171#endif 172 return usecs; 173} 174 175/*-------------------------------------------------------------------------*/ 176 177static int same_tt (struct usb_device *dev1, struct usb_device *dev2) 178{ 179 if (!dev1->tt || !dev2->tt) 180 return 0; 181 if (dev1->tt != dev2->tt) 182 return 0; 183 if (dev1->tt->multi) 184 return dev1->ttport == dev2->ttport; 185 else 186 return 1; 187} 188 189#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 190 191/* Which uframe does the low/fullspeed transfer start in? 192 * 193 * The parameter is the mask of ssplits in "H-frame" terms 194 * and this returns the transfer start uframe in "B-frame" terms, 195 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0 196 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag 197 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7. 198 */ 199static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask) 200{ 201 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask); 202 if (!smask) { 203 ehci_err(ehci, "invalid empty smask!\n"); 204 /* uframe 7 can't have bw so this will indicate failure */ 205 return 7; 206 } 207 return ffs(smask) - 1; 208} 209 210static const unsigned char 211max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 }; 212 213/* carryover low/fullspeed bandwidth that crosses uframe boundries */ 214static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8]) 215{ 216 int i; 217 for (i=0; i<7; i++) { 218 if (max_tt_usecs[i] < tt_usecs[i]) { 219 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i]; 220 tt_usecs[i] = max_tt_usecs[i]; 221 } 222 } 223} 224 225/* How many of the tt's periodic downstream 1000 usecs are allocated? 226 * 227 * While this measures the bandwidth in terms of usecs/uframe, 228 * the low/fullspeed bus has no notion of uframes, so any particular 229 * low/fullspeed transfer can "carry over" from one uframe to the next, 230 * since the TT just performs downstream transfers in sequence. 231 * 232 * For example two separate 100 usec transfers can start in the same uframe, 233 * and the second one would "carry over" 75 usecs into the next uframe. 234 */ 235static void 236periodic_tt_usecs ( 237 struct ehci_hcd *ehci, 238 struct usb_device *dev, 239 unsigned frame, 240 unsigned short tt_usecs[8] 241) 242{ 243 __hc32 *hw_p = &ehci->periodic [frame]; 244 union ehci_shadow *q = &ehci->pshadow [frame]; 245 unsigned char uf; 246 247 memset(tt_usecs, 0, 16); 248 249 while (q->ptr) { 250 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 251 case Q_TYPE_ITD: 252 hw_p = &q->itd->hw_next; 253 q = &q->itd->itd_next; 254 continue; 255 case Q_TYPE_QH: 256 if (same_tt(dev, q->qh->dev)) { 257 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2); 258 tt_usecs[uf] += q->qh->tt_usecs; 259 } 260 hw_p = &q->qh->hw->hw_next; 261 q = &q->qh->qh_next; 262 continue; 263 case Q_TYPE_SITD: 264 if (same_tt(dev, q->sitd->urb->dev)) { 265 uf = tt_start_uframe(ehci, q->sitd->hw_uframe); 266 tt_usecs[uf] += q->sitd->stream->tt_usecs; 267 } 268 hw_p = &q->sitd->hw_next; 269 q = &q->sitd->sitd_next; 270 continue; 271 // case Q_TYPE_FSTN: 272 default: 273 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n", 274 frame); 275 hw_p = &q->fstn->hw_next; 276 q = &q->fstn->fstn_next; 277 } 278 } 279 280 carryover_tt_bandwidth(tt_usecs); 281 282 if (max_tt_usecs[7] < tt_usecs[7]) 283 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n", 284 frame, tt_usecs[7] - max_tt_usecs[7]); 285} 286 287/* 288 * Return true if the device's tt's downstream bus is available for a 289 * periodic transfer of the specified length (usecs), starting at the 290 * specified frame/uframe. Note that (as summarized in section 11.19 291 * of the usb 2.0 spec) TTs can buffer multiple transactions for each 292 * uframe. 293 * 294 * The uframe parameter is when the fullspeed/lowspeed transfer 295 * should be executed in "B-frame" terms, which is the same as the 296 * highspeed ssplit's uframe (which is in "H-frame" terms). For example 297 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0. 298 * See the EHCI spec sec 4.5 and fig 4.7. 299 * 300 * This checks if the full/lowspeed bus, at the specified starting uframe, 301 * has the specified bandwidth available, according to rules listed 302 * in USB 2.0 spec section 11.18.1 fig 11-60. 303 * 304 * This does not check if the transfer would exceed the max ssplit 305 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4, 306 * since proper scheduling limits ssplits to less than 16 per uframe. 307 */ 308static int tt_available ( 309 struct ehci_hcd *ehci, 310 unsigned period, 311 struct usb_device *dev, 312 unsigned frame, 313 unsigned uframe, 314 u16 usecs 315) 316{ 317 if ((period == 0) || (uframe >= 7)) /* error */ 318 return 0; 319 320 for (; frame < ehci->periodic_size; frame += period) { 321 unsigned short tt_usecs[8]; 322 323 periodic_tt_usecs (ehci, dev, frame, tt_usecs); 324 325 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in" 326 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n", 327 frame, usecs, uframe, 328 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3], 329 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]); 330 331 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) { 332 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n", 333 frame, uframe); 334 return 0; 335 } 336 337 /* special case for isoc transfers larger than 125us: 338 * the first and each subsequent fully used uframe 339 * must be empty, so as to not illegally delay 340 * already scheduled transactions 341 */ 342 if (125 < usecs) { 343 int ufs = (usecs / 125); 344 int i; 345 for (i = uframe; i < (uframe + ufs) && i < 8; i++) 346 if (0 < tt_usecs[i]) { 347 ehci_vdbg(ehci, 348 "multi-uframe xfer can't fit " 349 "in frame %d uframe %d\n", 350 frame, i); 351 return 0; 352 } 353 } 354 355 tt_usecs[uframe] += usecs; 356 357 carryover_tt_bandwidth(tt_usecs); 358 359 /* fail if the carryover pushed bw past the last uframe's limit */ 360 if (max_tt_usecs[7] < tt_usecs[7]) { 361 ehci_vdbg(ehci, 362 "tt unavailable usecs %d frame %d uframe %d\n", 363 usecs, frame, uframe); 364 return 0; 365 } 366 } 367 368 return 1; 369} 370 371#else 372 373/* return true iff the device's transaction translator is available 374 * for a periodic transfer starting at the specified frame, using 375 * all the uframes in the mask. 376 */ 377static int tt_no_collision ( 378 struct ehci_hcd *ehci, 379 unsigned period, 380 struct usb_device *dev, 381 unsigned frame, 382 u32 uf_mask 383) 384{ 385 if (period == 0) /* error */ 386 return 0; 387 388 /* note bandwidth wastage: split never follows csplit 389 * (different dev or endpoint) until the next uframe. 390 * calling convention doesn't make that distinction. 391 */ 392 for (; frame < ehci->periodic_size; frame += period) { 393 union ehci_shadow here; 394 __hc32 type; 395 struct ehci_qh_hw *hw; 396 397 here = ehci->pshadow [frame]; 398 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]); 399 while (here.ptr) { 400 switch (hc32_to_cpu(ehci, type)) { 401 case Q_TYPE_ITD: 402 type = Q_NEXT_TYPE(ehci, here.itd->hw_next); 403 here = here.itd->itd_next; 404 continue; 405 case Q_TYPE_QH: 406 hw = here.qh->hw; 407 if (same_tt (dev, here.qh->dev)) { 408 u32 mask; 409 410 mask = hc32_to_cpu(ehci, 411 hw->hw_info2); 412 /* "knows" no gap is needed */ 413 mask |= mask >> 8; 414 if (mask & uf_mask) 415 break; 416 } 417 type = Q_NEXT_TYPE(ehci, hw->hw_next); 418 here = here.qh->qh_next; 419 continue; 420 case Q_TYPE_SITD: 421 if (same_tt (dev, here.sitd->urb->dev)) { 422 u16 mask; 423 424 mask = hc32_to_cpu(ehci, here.sitd 425 ->hw_uframe); 426 mask |= mask >> 8; 427 if (mask & uf_mask) 428 break; 429 } 430 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next); 431 here = here.sitd->sitd_next; 432 continue; 433 // case Q_TYPE_FSTN: 434 default: 435 ehci_dbg (ehci, 436 "periodic frame %d bogus type %d\n", 437 frame, type); 438 } 439 440 /* collision or error */ 441 return 0; 442 } 443 } 444 445 /* no collision */ 446 return 1; 447} 448 449#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */ 450 451/*-------------------------------------------------------------------------*/ 452 453static int enable_periodic (struct ehci_hcd *ehci) 454{ 455 u32 cmd; 456 int status; 457 458 if (ehci->periodic_sched++) 459 return 0; 460 461 /* did clearing PSE did take effect yet? 462 * takes effect only at frame boundaries... 463 */ 464 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 465 STS_PSS, 0, 9 * 125); 466 if (status) 467 return status; 468 469 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE; 470 ehci_writel(ehci, cmd, &ehci->regs->command); 471 /* posted write ... PSS happens later */ 472 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING; 473 474 /* make sure ehci_work scans these */ 475 ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index) 476 % (ehci->periodic_size << 3); 477 if (unlikely(ehci->broken_periodic)) 478 ehci->last_periodic_enable = ktime_get_real(); 479 return 0; 480} 481 482static int disable_periodic (struct ehci_hcd *ehci) 483{ 484 u32 cmd; 485 int status; 486 487 if (--ehci->periodic_sched) 488 return 0; 489 490 if (unlikely(ehci->broken_periodic)) { 491 /* delay experimentally determined */ 492 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000); 493 ktime_t now = ktime_get_real(); 494 s64 delay = ktime_us_delta(safe, now); 495 496 if (unlikely(delay > 0)) 497 udelay(delay); 498 } 499 500 /* did setting PSE not take effect yet? 501 * takes effect only at frame boundaries... 502 */ 503 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 504 STS_PSS, STS_PSS, 9 * 125); 505 if (status) 506 return status; 507 508 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE; 509 ehci_writel(ehci, cmd, &ehci->regs->command); 510 /* posted write ... */ 511 512 free_cached_lists(ehci); 513 514 ehci->next_uframe = -1; 515 return 0; 516} 517 518/*-------------------------------------------------------------------------*/ 519 520/* periodic schedule slots have iso tds (normal or split) first, then a 521 * sparse tree for active interrupt transfers. 522 * 523 * this just links in a qh; caller guarantees uframe masks are set right. 524 * no FSTN support (yet; ehci 0.96+) 525 */ 526static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) 527{ 528 unsigned i; 529 unsigned period = qh->period; 530 531 dev_dbg (&qh->dev->dev, 532 "link qh%d-%04x/%p start %d [%d/%d us]\n", 533 period, hc32_to_cpup(ehci, &qh->hw->hw_info2) 534 & (QH_CMASK | QH_SMASK), 535 qh, qh->start, qh->usecs, qh->c_usecs); 536 537 /* high bandwidth, or otherwise every microframe */ 538 if (period == 0) 539 period = 1; 540 541 for (i = qh->start; i < ehci->periodic_size; i += period) { 542 union ehci_shadow *prev = &ehci->pshadow[i]; 543 __hc32 *hw_p = &ehci->periodic[i]; 544 union ehci_shadow here = *prev; 545 __hc32 type = 0; 546 547 /* skip the iso nodes at list head */ 548 while (here.ptr) { 549 type = Q_NEXT_TYPE(ehci, *hw_p); 550 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 551 break; 552 prev = periodic_next_shadow(ehci, prev, type); 553 hw_p = shadow_next_periodic(ehci, &here, type); 554 here = *prev; 555 } 556 557 /* sorting each branch by period (slow-->fast) 558 * enables sharing interior tree nodes 559 */ 560 while (here.ptr && qh != here.qh) { 561 if (qh->period > here.qh->period) 562 break; 563 prev = &here.qh->qh_next; 564 hw_p = &here.qh->hw->hw_next; 565 here = *prev; 566 } 567 /* link in this qh, unless some earlier pass did that */ 568 if (qh != here.qh) { 569 qh->qh_next = here; 570 if (here.qh) 571 qh->hw->hw_next = *hw_p; 572 wmb (); 573 prev->qh = qh; 574 *hw_p = QH_NEXT (ehci, qh->qh_dma); 575 } 576 } 577 qh->qh_state = QH_STATE_LINKED; 578 qh->xacterrs = 0; 579 qh_get (qh); 580 581 /* update per-qh bandwidth for usbfs */ 582 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period 583 ? ((qh->usecs + qh->c_usecs) / qh->period) 584 : (qh->usecs * 8); 585 586 /* maybe enable periodic schedule processing */ 587 return enable_periodic(ehci); 588} 589 590static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh) 591{ 592 unsigned i; 593 unsigned period; 594 595 // IF this isn't high speed 596 // and this qh is active in the current uframe 597 // (and overlay token SplitXstate is false?) 598 // THEN 599 // qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */); 600 601 /* high bandwidth, or otherwise part of every microframe */ 602 if ((period = qh->period) == 0) 603 period = 1; 604 605 for (i = qh->start; i < ehci->periodic_size; i += period) 606 periodic_unlink (ehci, i, qh); 607 608 /* update per-qh bandwidth for usbfs */ 609 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period 610 ? ((qh->usecs + qh->c_usecs) / qh->period) 611 : (qh->usecs * 8); 612 613 dev_dbg (&qh->dev->dev, 614 "unlink qh%d-%04x/%p start %d [%d/%d us]\n", 615 qh->period, 616 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK), 617 qh, qh->start, qh->usecs, qh->c_usecs); 618 619 /* qh->qh_next still "live" to HC */ 620 qh->qh_state = QH_STATE_UNLINK; 621 qh->qh_next.ptr = NULL; 622 qh_put (qh); 623 624 /* maybe turn off periodic schedule */ 625 return disable_periodic(ehci); 626} 627 628static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) 629{ 630 unsigned wait; 631 struct ehci_qh_hw *hw = qh->hw; 632 int rc; 633 634 /* If the QH isn't linked then there's nothing we can do 635 * unless we were called during a giveback, in which case 636 * qh_completions() has to deal with it. 637 */ 638 if (qh->qh_state != QH_STATE_LINKED) { 639 if (qh->qh_state == QH_STATE_COMPLETING) 640 qh->needs_rescan = 1; 641 return; 642 } 643 644 qh_unlink_periodic (ehci, qh); 645 646 /* simple/paranoid: always delay, expecting the HC needs to read 647 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and 648 * expect khubd to clean up after any CSPLITs we won't issue. 649 * active high speed queues may need bigger delays... 650 */ 651 if (list_empty (&qh->qtd_list) 652 || (cpu_to_hc32(ehci, QH_CMASK) 653 & hw->hw_info2) != 0) 654 wait = 2; 655 else 656 wait = 55; /* worst case: 3 * 1024 */ 657 658 udelay (wait); 659 qh->qh_state = QH_STATE_IDLE; 660 hw->hw_next = EHCI_LIST_END(ehci); 661 wmb (); 662 663 qh_completions(ehci, qh); 664 665 /* reschedule QH iff another request is queued */ 666 if (!list_empty(&qh->qtd_list) && 667 HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) { 668 rc = qh_schedule(ehci, qh); 669 670 if (rc != 0) 671 ehci_err(ehci, "can't reschedule qh %p, err %d\n", 672 qh, rc); 673 } 674} 675 676/*-------------------------------------------------------------------------*/ 677 678static int check_period ( 679 struct ehci_hcd *ehci, 680 unsigned frame, 681 unsigned uframe, 682 unsigned period, 683 unsigned usecs 684) { 685 int claimed; 686 687 /* complete split running into next frame? 688 * given FSTN support, we could sometimes check... 689 */ 690 if (uframe >= 8) 691 return 0; 692 693 /* 694 * 80% periodic == 100 usec/uframe available 695 * convert "usecs we need" to "max already claimed" 696 */ 697 usecs = 100 - usecs; 698 699 /* we "know" 2 and 4 uframe intervals were rejected; so 700 * for period 0, check _every_ microframe in the schedule. 701 */ 702 if (unlikely (period == 0)) { 703 do { 704 for (uframe = 0; uframe < 7; uframe++) { 705 claimed = periodic_usecs (ehci, frame, uframe); 706 if (claimed > usecs) 707 return 0; 708 } 709 } while ((frame += 1) < ehci->periodic_size); 710 711 /* just check the specified uframe, at that period */ 712 } else { 713 do { 714 claimed = periodic_usecs (ehci, frame, uframe); 715 if (claimed > usecs) 716 return 0; 717 } while ((frame += period) < ehci->periodic_size); 718 } 719 720 // success! 721 return 1; 722} 723 724static int check_intr_schedule ( 725 struct ehci_hcd *ehci, 726 unsigned frame, 727 unsigned uframe, 728 const struct ehci_qh *qh, 729 __hc32 *c_maskp 730) 731{ 732 int retval = -ENOSPC; 733 u8 mask = 0; 734 735 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */ 736 goto done; 737 738 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs)) 739 goto done; 740 if (!qh->c_usecs) { 741 retval = 0; 742 *c_maskp = 0; 743 goto done; 744 } 745 746#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 747 if (tt_available (ehci, qh->period, qh->dev, frame, uframe, 748 qh->tt_usecs)) { 749 unsigned i; 750 751 /* TODO : this may need FSTN for SSPLIT in uframe 5. */ 752 for (i=uframe+1; i<8 && i<uframe+4; i++) 753 if (!check_period (ehci, frame, i, 754 qh->period, qh->c_usecs)) 755 goto done; 756 else 757 mask |= 1 << i; 758 759 retval = 0; 760 761 *c_maskp = cpu_to_hc32(ehci, mask << 8); 762 } 763#else 764 /* Make sure this tt's buffer is also available for CSPLITs. 765 * We pessimize a bit; probably the typical full speed case 766 * doesn't need the second CSPLIT. 767 * 768 * NOTE: both SPLIT and CSPLIT could be checked in just 769 * one smart pass... 770 */ 771 mask = 0x03 << (uframe + qh->gap_uf); 772 *c_maskp = cpu_to_hc32(ehci, mask << 8); 773 774 mask |= 1 << uframe; 775 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) { 776 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1, 777 qh->period, qh->c_usecs)) 778 goto done; 779 if (!check_period (ehci, frame, uframe + qh->gap_uf, 780 qh->period, qh->c_usecs)) 781 goto done; 782 retval = 0; 783 } 784#endif 785done: 786 return retval; 787} 788 789/* "first fit" scheduling policy used the first time through, 790 * or when the previous schedule slot can't be re-used. 791 */ 792static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh) 793{ 794 int status; 795 unsigned uframe; 796 __hc32 c_mask; 797 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */ 798 struct ehci_qh_hw *hw = qh->hw; 799 800 qh_refresh(ehci, qh); 801 hw->hw_next = EHCI_LIST_END(ehci); 802 frame = qh->start; 803 804 /* reuse the previous schedule slots, if we can */ 805 if (frame < qh->period) { 806 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK); 807 status = check_intr_schedule (ehci, frame, --uframe, 808 qh, &c_mask); 809 } else { 810 uframe = 0; 811 c_mask = 0; 812 status = -ENOSPC; 813 } 814 815 /* else scan the schedule to find a group of slots such that all 816 * uframes have enough periodic bandwidth available. 817 */ 818 if (status) { 819 /* "normal" case, uframing flexible except with splits */ 820 if (qh->period) { 821 int i; 822 823 for (i = qh->period; status && i > 0; --i) { 824 frame = ++ehci->random_frame % qh->period; 825 for (uframe = 0; uframe < 8; uframe++) { 826 status = check_intr_schedule (ehci, 827 frame, uframe, qh, 828 &c_mask); 829 if (status == 0) 830 break; 831 } 832 } 833 834 /* qh->period == 0 means every uframe */ 835 } else { 836 frame = 0; 837 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask); 838 } 839 if (status) 840 goto done; 841 qh->start = frame; 842 843 /* reset S-frame and (maybe) C-frame masks */ 844 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK)); 845 hw->hw_info2 |= qh->period 846 ? cpu_to_hc32(ehci, 1 << uframe) 847 : cpu_to_hc32(ehci, QH_SMASK); 848 hw->hw_info2 |= c_mask; 849 } else 850 ehci_dbg (ehci, "reused qh %p schedule\n", qh); 851 852 /* stuff into the periodic schedule */ 853 status = qh_link_periodic (ehci, qh); 854done: 855 return status; 856} 857 858static int intr_submit ( 859 struct ehci_hcd *ehci, 860 struct urb *urb, 861 struct list_head *qtd_list, 862 gfp_t mem_flags 863) { 864 unsigned epnum; 865 unsigned long flags; 866 struct ehci_qh *qh; 867 int status; 868 struct list_head empty; 869 870 /* get endpoint and transfer/schedule data */ 871 epnum = urb->ep->desc.bEndpointAddress; 872 873 spin_lock_irqsave (&ehci->lock, flags); 874 875 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 876 status = -ESHUTDOWN; 877 goto done_not_linked; 878 } 879 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 880 if (unlikely(status)) 881 goto done_not_linked; 882 883 /* get qh and force any scheduling errors */ 884 INIT_LIST_HEAD (&empty); 885 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv); 886 if (qh == NULL) { 887 status = -ENOMEM; 888 goto done; 889 } 890 if (qh->qh_state == QH_STATE_IDLE) { 891 if ((status = qh_schedule (ehci, qh)) != 0) 892 goto done; 893 } 894 895 /* then queue the urb's tds to the qh */ 896 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv); 897 BUG_ON (qh == NULL); 898 899 /* ... update usbfs periodic stats */ 900 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++; 901 902done: 903 if (unlikely(status)) 904 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 905done_not_linked: 906 spin_unlock_irqrestore (&ehci->lock, flags); 907 if (status) 908 qtd_list_free (ehci, urb, qtd_list); 909 910 return status; 911} 912 913/*-------------------------------------------------------------------------*/ 914 915/* ehci_iso_stream ops work with both ITD and SITD */ 916 917static struct ehci_iso_stream * 918iso_stream_alloc (gfp_t mem_flags) 919{ 920 struct ehci_iso_stream *stream; 921 922 stream = kzalloc(sizeof *stream, mem_flags); 923 if (likely (stream != NULL)) { 924 INIT_LIST_HEAD(&stream->td_list); 925 INIT_LIST_HEAD(&stream->free_list); 926 stream->next_uframe = -1; 927 stream->refcount = 1; 928 } 929 return stream; 930} 931 932static void 933iso_stream_init ( 934 struct ehci_hcd *ehci, 935 struct ehci_iso_stream *stream, 936 struct usb_device *dev, 937 int pipe, 938 unsigned interval 939) 940{ 941 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; 942 943 u32 buf1; 944 unsigned epnum, maxp; 945 int is_input; 946 long bandwidth; 947 948 /* 949 * this might be a "high bandwidth" highspeed endpoint, 950 * as encoded in the ep descriptor's wMaxPacket field 951 */ 952 epnum = usb_pipeendpoint (pipe); 953 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0; 954 maxp = usb_maxpacket(dev, pipe, !is_input); 955 if (is_input) { 956 buf1 = (1 << 11); 957 } else { 958 buf1 = 0; 959 } 960 961 /* knows about ITD vs SITD */ 962 if (dev->speed == USB_SPEED_HIGH) { 963 unsigned multi = hb_mult(maxp); 964 965 stream->highspeed = 1; 966 967 maxp = max_packet(maxp); 968 buf1 |= maxp; 969 maxp *= multi; 970 971 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum); 972 stream->buf1 = cpu_to_hc32(ehci, buf1); 973 stream->buf2 = cpu_to_hc32(ehci, multi); 974 975 /* usbfs wants to report the average usecs per frame tied up 976 * when transfers on this endpoint are scheduled ... 977 */ 978 stream->usecs = HS_USECS_ISO (maxp); 979 bandwidth = stream->usecs * 8; 980 bandwidth /= interval; 981 982 } else { 983 u32 addr; 984 int think_time; 985 int hs_transfers; 986 987 addr = dev->ttport << 24; 988 if (!ehci_is_TDI(ehci) 989 || (dev->tt->hub != 990 ehci_to_hcd(ehci)->self.root_hub)) 991 addr |= dev->tt->hub->devnum << 16; 992 addr |= epnum << 8; 993 addr |= dev->devnum; 994 stream->usecs = HS_USECS_ISO (maxp); 995 think_time = dev->tt ? dev->tt->think_time : 0; 996 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time ( 997 dev->speed, is_input, 1, maxp)); 998 hs_transfers = max (1u, (maxp + 187) / 188); 999 if (is_input) { 1000 u32 tmp; 1001 1002 addr |= 1 << 31; 1003 stream->c_usecs = stream->usecs; 1004 stream->usecs = HS_USECS_ISO (1); 1005 stream->raw_mask = 1; 1006 1007 /* c-mask as specified in USB 2.0 11.18.4 3.c */ 1008 tmp = (1 << (hs_transfers + 2)) - 1; 1009 stream->raw_mask |= tmp << (8 + 2); 1010 } else 1011 stream->raw_mask = smask_out [hs_transfers - 1]; 1012 bandwidth = stream->usecs + stream->c_usecs; 1013 bandwidth /= interval << 3; 1014 1015 /* stream->splits gets created from raw_mask later */ 1016 stream->address = cpu_to_hc32(ehci, addr); 1017 } 1018 stream->bandwidth = bandwidth; 1019 1020 stream->udev = dev; 1021 1022 stream->bEndpointAddress = is_input | epnum; 1023 stream->interval = interval; 1024 stream->maxp = maxp; 1025} 1026 1027static void 1028iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream) 1029{ 1030 stream->refcount--; 1031 1032 /* free whenever just a dev->ep reference remains. 1033 * not like a QH -- no persistent state (toggle, halt) 1034 */ 1035 if (stream->refcount == 1) { 1036 int is_in; 1037 1038 // BUG_ON (!list_empty(&stream->td_list)); 1039 1040 while (!list_empty (&stream->free_list)) { 1041 struct list_head *entry; 1042 1043 entry = stream->free_list.next; 1044 list_del (entry); 1045 1046 /* knows about ITD vs SITD */ 1047 if (stream->highspeed) { 1048 struct ehci_itd *itd; 1049 1050 itd = list_entry (entry, struct ehci_itd, 1051 itd_list); 1052 dma_pool_free (ehci->itd_pool, itd, 1053 itd->itd_dma); 1054 } else { 1055 struct ehci_sitd *sitd; 1056 1057 sitd = list_entry (entry, struct ehci_sitd, 1058 sitd_list); 1059 dma_pool_free (ehci->sitd_pool, sitd, 1060 sitd->sitd_dma); 1061 } 1062 } 1063 1064 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0; 1065 stream->bEndpointAddress &= 0x0f; 1066 if (stream->ep) 1067 stream->ep->hcpriv = NULL; 1068 1069 kfree(stream); 1070 } 1071} 1072 1073static inline struct ehci_iso_stream * 1074iso_stream_get (struct ehci_iso_stream *stream) 1075{ 1076 if (likely (stream != NULL)) 1077 stream->refcount++; 1078 return stream; 1079} 1080 1081static struct ehci_iso_stream * 1082iso_stream_find (struct ehci_hcd *ehci, struct urb *urb) 1083{ 1084 unsigned epnum; 1085 struct ehci_iso_stream *stream; 1086 struct usb_host_endpoint *ep; 1087 unsigned long flags; 1088 1089 epnum = usb_pipeendpoint (urb->pipe); 1090 if (usb_pipein(urb->pipe)) 1091 ep = urb->dev->ep_in[epnum]; 1092 else 1093 ep = urb->dev->ep_out[epnum]; 1094 1095 spin_lock_irqsave (&ehci->lock, flags); 1096 stream = ep->hcpriv; 1097 1098 if (unlikely (stream == NULL)) { 1099 stream = iso_stream_alloc(GFP_ATOMIC); 1100 if (likely (stream != NULL)) { 1101 /* dev->ep owns the initial refcount */ 1102 ep->hcpriv = stream; 1103 stream->ep = ep; 1104 iso_stream_init(ehci, stream, urb->dev, urb->pipe, 1105 urb->interval); 1106 } 1107 1108 /* if dev->ep [epnum] is a QH, hw is set */ 1109 } else if (unlikely (stream->hw != NULL)) { 1110 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n", 1111 urb->dev->devpath, epnum, 1112 usb_pipein(urb->pipe) ? "in" : "out"); 1113 stream = NULL; 1114 } 1115 1116 /* caller guarantees an eventual matching iso_stream_put */ 1117 stream = iso_stream_get (stream); 1118 1119 spin_unlock_irqrestore (&ehci->lock, flags); 1120 return stream; 1121} 1122 1123/*-------------------------------------------------------------------------*/ 1124 1125/* ehci_iso_sched ops can be ITD-only or SITD-only */ 1126 1127static struct ehci_iso_sched * 1128iso_sched_alloc (unsigned packets, gfp_t mem_flags) 1129{ 1130 struct ehci_iso_sched *iso_sched; 1131 int size = sizeof *iso_sched; 1132 1133 size += packets * sizeof (struct ehci_iso_packet); 1134 iso_sched = kzalloc(size, mem_flags); 1135 if (likely (iso_sched != NULL)) { 1136 INIT_LIST_HEAD (&iso_sched->td_list); 1137 } 1138 return iso_sched; 1139} 1140 1141static inline void 1142itd_sched_init( 1143 struct ehci_hcd *ehci, 1144 struct ehci_iso_sched *iso_sched, 1145 struct ehci_iso_stream *stream, 1146 struct urb *urb 1147) 1148{ 1149 unsigned i; 1150 dma_addr_t dma = urb->transfer_dma; 1151 1152 /* how many uframes are needed for these transfers */ 1153 iso_sched->span = urb->number_of_packets * stream->interval; 1154 1155 /* figure out per-uframe itd fields that we'll need later 1156 * when we fit new itds into the schedule. 1157 */ 1158 for (i = 0; i < urb->number_of_packets; i++) { 1159 struct ehci_iso_packet *uframe = &iso_sched->packet [i]; 1160 unsigned length; 1161 dma_addr_t buf; 1162 u32 trans; 1163 1164 length = urb->iso_frame_desc [i].length; 1165 buf = dma + urb->iso_frame_desc [i].offset; 1166 1167 trans = EHCI_ISOC_ACTIVE; 1168 trans |= buf & 0x0fff; 1169 if (unlikely (((i + 1) == urb->number_of_packets)) 1170 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1171 trans |= EHCI_ITD_IOC; 1172 trans |= length << 16; 1173 uframe->transaction = cpu_to_hc32(ehci, trans); 1174 1175 /* might need to cross a buffer page within a uframe */ 1176 uframe->bufp = (buf & ~(u64)0x0fff); 1177 buf += length; 1178 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff)))) 1179 uframe->cross = 1; 1180 } 1181} 1182 1183static void 1184iso_sched_free ( 1185 struct ehci_iso_stream *stream, 1186 struct ehci_iso_sched *iso_sched 1187) 1188{ 1189 if (!iso_sched) 1190 return; 1191 // caller must hold ehci->lock! 1192 list_splice (&iso_sched->td_list, &stream->free_list); 1193 kfree (iso_sched); 1194} 1195 1196static int 1197itd_urb_transaction ( 1198 struct ehci_iso_stream *stream, 1199 struct ehci_hcd *ehci, 1200 struct urb *urb, 1201 gfp_t mem_flags 1202) 1203{ 1204 struct ehci_itd *itd; 1205 dma_addr_t itd_dma; 1206 int i; 1207 unsigned num_itds; 1208 struct ehci_iso_sched *sched; 1209 unsigned long flags; 1210 1211 sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1212 if (unlikely (sched == NULL)) 1213 return -ENOMEM; 1214 1215 itd_sched_init(ehci, sched, stream, urb); 1216 1217 if (urb->interval < 8) 1218 num_itds = 1 + (sched->span + 7) / 8; 1219 else 1220 num_itds = urb->number_of_packets; 1221 1222 /* allocate/init ITDs */ 1223 spin_lock_irqsave (&ehci->lock, flags); 1224 for (i = 0; i < num_itds; i++) { 1225 1226 /* free_list.next might be cache-hot ... but maybe 1227 * the HC caches it too. avoid that issue for now. 1228 */ 1229 1230 /* prefer previously-allocated itds */ 1231 if (likely (!list_empty(&stream->free_list))) { 1232 itd = list_entry (stream->free_list.prev, 1233 struct ehci_itd, itd_list); 1234 list_del (&itd->itd_list); 1235 itd_dma = itd->itd_dma; 1236 } else { 1237 spin_unlock_irqrestore (&ehci->lock, flags); 1238 itd = dma_pool_alloc (ehci->itd_pool, mem_flags, 1239 &itd_dma); 1240 spin_lock_irqsave (&ehci->lock, flags); 1241 if (!itd) { 1242 iso_sched_free(stream, sched); 1243 spin_unlock_irqrestore(&ehci->lock, flags); 1244 return -ENOMEM; 1245 } 1246 } 1247 1248 memset (itd, 0, sizeof *itd); 1249 itd->itd_dma = itd_dma; 1250 list_add (&itd->itd_list, &sched->td_list); 1251 } 1252 spin_unlock_irqrestore (&ehci->lock, flags); 1253 1254 /* temporarily store schedule info in hcpriv */ 1255 urb->hcpriv = sched; 1256 urb->error_count = 0; 1257 return 0; 1258} 1259 1260/*-------------------------------------------------------------------------*/ 1261 1262static inline int 1263itd_slot_ok ( 1264 struct ehci_hcd *ehci, 1265 u32 mod, 1266 u32 uframe, 1267 u8 usecs, 1268 u32 period 1269) 1270{ 1271 uframe %= period; 1272 do { 1273 /* can't commit more than 80% periodic == 100 usec */ 1274 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) 1275 > (100 - usecs)) 1276 return 0; 1277 1278 /* we know urb->interval is 2^N uframes */ 1279 uframe += period; 1280 } while (uframe < mod); 1281 return 1; 1282} 1283 1284static inline int 1285sitd_slot_ok ( 1286 struct ehci_hcd *ehci, 1287 u32 mod, 1288 struct ehci_iso_stream *stream, 1289 u32 uframe, 1290 struct ehci_iso_sched *sched, 1291 u32 period_uframes 1292) 1293{ 1294 u32 mask, tmp; 1295 u32 frame, uf; 1296 1297 mask = stream->raw_mask << (uframe & 7); 1298 1299 /* for IN, don't wrap CSPLIT into the next frame */ 1300 if (mask & ~0xffff) 1301 return 0; 1302 1303 /* this multi-pass logic is simple, but performance may 1304 * suffer when the schedule data isn't cached. 1305 */ 1306 1307 /* check bandwidth */ 1308 uframe %= period_uframes; 1309 do { 1310 u32 max_used; 1311 1312 frame = uframe >> 3; 1313 uf = uframe & 7; 1314 1315#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 1316 /* The tt's fullspeed bus bandwidth must be available. 1317 * tt_available scheduling guarantees 10+% for control/bulk. 1318 */ 1319 if (!tt_available (ehci, period_uframes << 3, 1320 stream->udev, frame, uf, stream->tt_usecs)) 1321 return 0; 1322#else 1323 /* tt must be idle for start(s), any gap, and csplit. 1324 * assume scheduling slop leaves 10+% for control/bulk. 1325 */ 1326 if (!tt_no_collision (ehci, period_uframes << 3, 1327 stream->udev, frame, mask)) 1328 return 0; 1329#endif 1330 1331 /* check starts (OUT uses more than one) */ 1332 max_used = 100 - stream->usecs; 1333 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) { 1334 if (periodic_usecs (ehci, frame, uf) > max_used) 1335 return 0; 1336 } 1337 1338 /* for IN, check CSPLIT */ 1339 if (stream->c_usecs) { 1340 uf = uframe & 7; 1341 max_used = 100 - stream->c_usecs; 1342 do { 1343 tmp = 1 << uf; 1344 tmp <<= 8; 1345 if ((stream->raw_mask & tmp) == 0) 1346 continue; 1347 if (periodic_usecs (ehci, frame, uf) 1348 > max_used) 1349 return 0; 1350 } while (++uf < 8); 1351 } 1352 1353 /* we know urb->interval is 2^N uframes */ 1354 uframe += period_uframes; 1355 } while (uframe < mod); 1356 1357 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7)); 1358 return 1; 1359} 1360 1361/* 1362 * This scheduler plans almost as far into the future as it has actual 1363 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to 1364 * "as small as possible" to be cache-friendlier.) That limits the size 1365 * transfers you can stream reliably; avoid more than 64 msec per urb. 1366 * Also avoid queue depths of less than ehci's worst irq latency (affected 1367 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, 1368 * and other factors); or more than about 230 msec total (for portability, 1369 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! 1370 */ 1371 1372#define SCHEDULE_SLOP 80 /* microframes */ 1373 1374static int 1375iso_stream_schedule ( 1376 struct ehci_hcd *ehci, 1377 struct urb *urb, 1378 struct ehci_iso_stream *stream 1379) 1380{ 1381 u32 now, next, start, period, span; 1382 int status; 1383 unsigned mod = ehci->periodic_size << 3; 1384 struct ehci_iso_sched *sched = urb->hcpriv; 1385 1386 period = urb->interval; 1387 span = sched->span; 1388 if (!stream->highspeed) { 1389 period <<= 3; 1390 span <<= 3; 1391 } 1392 1393 if (span > mod - SCHEDULE_SLOP) { 1394 ehci_dbg (ehci, "iso request %p too long\n", urb); 1395 status = -EFBIG; 1396 goto fail; 1397 } 1398 1399 now = ehci_readl(ehci, &ehci->regs->frame_index) & (mod - 1); 1400 1401 /* Typical case: reuse current schedule, stream is still active. 1402 * Hopefully there are no gaps from the host falling behind 1403 * (irq delays etc), but if there are we'll take the next 1404 * slot in the schedule, implicitly assuming URB_ISO_ASAP. 1405 */ 1406 if (likely (!list_empty (&stream->td_list))) { 1407 u32 excess; 1408 1409 if (!stream->highspeed && ehci->fs_i_thresh) 1410 next = now + ehci->i_thresh; 1411 else 1412 next = now; 1413 1414 /* Fell behind (by up to twice the slop amount)? 1415 * We decide based on the time of the last currently-scheduled 1416 * slot, not the time of the next available slot. 1417 */ 1418 excess = (stream->next_uframe - period - next) & (mod - 1); 1419 if (excess >= mod - 2 * SCHEDULE_SLOP) 1420 start = next + excess - mod + period * 1421 DIV_ROUND_UP(mod - excess, period); 1422 else 1423 start = next + excess + period; 1424 if (start - now >= mod) { 1425 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1426 urb, start - now - period, period, 1427 mod); 1428 status = -EFBIG; 1429 goto fail; 1430 } 1431 } 1432 1433 /* need to schedule; when's the next (u)frame we could start? 1434 * this is bigger than ehci->i_thresh allows; scheduling itself 1435 * isn't free, the slop should handle reasonably slow cpus. it 1436 * can also help high bandwidth if the dma and irq loads don't 1437 * jump until after the queue is primed. 1438 */ 1439 else { 1440 start = SCHEDULE_SLOP + (now & ~0x07); 1441 1442 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */ 1443 1444 /* find a uframe slot with enough bandwidth */ 1445 next = start + period; 1446 for (; start < next; start++) { 1447 1448 /* check schedule: enough space? */ 1449 if (stream->highspeed) { 1450 if (itd_slot_ok(ehci, mod, start, 1451 stream->usecs, period)) 1452 break; 1453 } else { 1454 if ((start % 8) >= 6) 1455 continue; 1456 if (sitd_slot_ok(ehci, mod, stream, 1457 start, sched, period)) 1458 break; 1459 } 1460 } 1461 1462 /* no room in the schedule */ 1463 if (start == next) { 1464 ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n", 1465 urb, now, now + mod); 1466 status = -ENOSPC; 1467 goto fail; 1468 } 1469 } 1470 1471 /* Tried to schedule too far into the future? */ 1472 if (unlikely(start - now + span - period 1473 >= mod - 2 * SCHEDULE_SLOP)) { 1474 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1475 urb, start - now, span - period, 1476 mod - 2 * SCHEDULE_SLOP); 1477 status = -EFBIG; 1478 goto fail; 1479 } 1480 1481 stream->next_uframe = start & (mod - 1); 1482 1483 /* report high speed start in uframes; full speed, in frames */ 1484 urb->start_frame = stream->next_uframe; 1485 if (!stream->highspeed) 1486 urb->start_frame >>= 3; 1487 return 0; 1488 1489 fail: 1490 iso_sched_free(stream, sched); 1491 urb->hcpriv = NULL; 1492 return status; 1493} 1494 1495/*-------------------------------------------------------------------------*/ 1496 1497static inline void 1498itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream, 1499 struct ehci_itd *itd) 1500{ 1501 int i; 1502 1503 /* it's been recently zeroed */ 1504 itd->hw_next = EHCI_LIST_END(ehci); 1505 itd->hw_bufp [0] = stream->buf0; 1506 itd->hw_bufp [1] = stream->buf1; 1507 itd->hw_bufp [2] = stream->buf2; 1508 1509 for (i = 0; i < 8; i++) 1510 itd->index[i] = -1; 1511 1512 /* All other fields are filled when scheduling */ 1513} 1514 1515static inline void 1516itd_patch( 1517 struct ehci_hcd *ehci, 1518 struct ehci_itd *itd, 1519 struct ehci_iso_sched *iso_sched, 1520 unsigned index, 1521 u16 uframe 1522) 1523{ 1524 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 1525 unsigned pg = itd->pg; 1526 1527 // BUG_ON (pg == 6 && uf->cross); 1528 1529 uframe &= 0x07; 1530 itd->index [uframe] = index; 1531 1532 itd->hw_transaction[uframe] = uf->transaction; 1533 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12); 1534 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0); 1535 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32)); 1536 1537 /* iso_frame_desc[].offset must be strictly increasing */ 1538 if (unlikely (uf->cross)) { 1539 u64 bufp = uf->bufp + 4096; 1540 1541 itd->pg = ++pg; 1542 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0); 1543 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32)); 1544 } 1545} 1546 1547static inline void 1548itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) 1549{ 1550 union ehci_shadow *prev = &ehci->pshadow[frame]; 1551 __hc32 *hw_p = &ehci->periodic[frame]; 1552 union ehci_shadow here = *prev; 1553 __hc32 type = 0; 1554 1555 /* skip any iso nodes which might belong to previous microframes */ 1556 while (here.ptr) { 1557 type = Q_NEXT_TYPE(ehci, *hw_p); 1558 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 1559 break; 1560 prev = periodic_next_shadow(ehci, prev, type); 1561 hw_p = shadow_next_periodic(ehci, &here, type); 1562 here = *prev; 1563 } 1564 1565 itd->itd_next = here; 1566 itd->hw_next = *hw_p; 1567 prev->itd = itd; 1568 itd->frame = frame; 1569 wmb (); 1570 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD); 1571} 1572 1573#define AB_REG_BAR_LOW 0xe0 1574#define AB_REG_BAR_HIGH 0xe1 1575#define AB_INDX(addr) ((addr) + 0x00) 1576#define AB_DATA(addr) ((addr) + 0x04) 1577#define NB_PCIE_INDX_ADDR 0xe0 1578#define NB_PCIE_INDX_DATA 0xe4 1579#define NB_PIF0_PWRDOWN_0 0x01100012 1580#define NB_PIF0_PWRDOWN_1 0x01100013 1581 1582static void ehci_quirk_amd_L1(struct ehci_hcd *ehci, int disable) 1583{ 1584 u32 addr, addr_low, addr_high, val; 1585 1586 outb_p(AB_REG_BAR_LOW, 0xcd6); 1587 addr_low = inb_p(0xcd7); 1588 outb_p(AB_REG_BAR_HIGH, 0xcd6); 1589 addr_high = inb_p(0xcd7); 1590 addr = addr_high << 8 | addr_low; 1591 outl_p(0x30, AB_INDX(addr)); 1592 outl_p(0x40, AB_DATA(addr)); 1593 outl_p(0x34, AB_INDX(addr)); 1594 val = inl_p(AB_DATA(addr)); 1595 1596 if (disable) { 1597 val &= ~0x8; 1598 val |= (1 << 4) | (1 << 9); 1599 } else { 1600 val |= 0x8; 1601 val &= ~((1 << 4) | (1 << 9)); 1602 } 1603 outl_p(val, AB_DATA(addr)); 1604 1605 if (amd_nb_dev) { 1606 addr = NB_PIF0_PWRDOWN_0; 1607 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_ADDR, addr); 1608 pci_read_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, &val); 1609 if (disable) 1610 val &= ~(0x3f << 7); 1611 else 1612 val |= 0x3f << 7; 1613 1614 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, val); 1615 1616 addr = NB_PIF0_PWRDOWN_1; 1617 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_ADDR, addr); 1618 pci_read_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, &val); 1619 if (disable) 1620 val &= ~(0x3f << 7); 1621 else 1622 val |= 0x3f << 7; 1623 1624 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, val); 1625 } 1626 1627 return; 1628} 1629 1630/* fit urb's itds into the selected schedule slot; activate as needed */ 1631static int 1632itd_link_urb ( 1633 struct ehci_hcd *ehci, 1634 struct urb *urb, 1635 unsigned mod, 1636 struct ehci_iso_stream *stream 1637) 1638{ 1639 int packet; 1640 unsigned next_uframe, uframe, frame; 1641 struct ehci_iso_sched *iso_sched = urb->hcpriv; 1642 struct ehci_itd *itd; 1643 1644 next_uframe = stream->next_uframe & (mod - 1); 1645 1646 if (unlikely (list_empty(&stream->td_list))) { 1647 ehci_to_hcd(ehci)->self.bandwidth_allocated 1648 += stream->bandwidth; 1649 ehci_vdbg (ehci, 1650 "schedule devp %s ep%d%s-iso period %d start %d.%d\n", 1651 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 1652 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 1653 urb->interval, 1654 next_uframe >> 3, next_uframe & 0x7); 1655 } 1656 1657 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1658 if (ehci->amd_l1_fix == 1) 1659 ehci_quirk_amd_L1(ehci, 1); 1660 } 1661 1662 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 1663 1664 /* fill iTDs uframe by uframe */ 1665 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) { 1666 if (itd == NULL) { 1667 /* ASSERT: we have all necessary itds */ 1668 // BUG_ON (list_empty (&iso_sched->td_list)); 1669 1670 /* ASSERT: no itds for this endpoint in this uframe */ 1671 1672 itd = list_entry (iso_sched->td_list.next, 1673 struct ehci_itd, itd_list); 1674 list_move_tail (&itd->itd_list, &stream->td_list); 1675 itd->stream = iso_stream_get (stream); 1676 itd->urb = urb; 1677 itd_init (ehci, stream, itd); 1678 } 1679 1680 uframe = next_uframe & 0x07; 1681 frame = next_uframe >> 3; 1682 1683 itd_patch(ehci, itd, iso_sched, packet, uframe); 1684 1685 next_uframe += stream->interval; 1686 next_uframe &= mod - 1; 1687 packet++; 1688 1689 /* link completed itds into the schedule */ 1690 if (((next_uframe >> 3) != frame) 1691 || packet == urb->number_of_packets) { 1692 itd_link(ehci, frame & (ehci->periodic_size - 1), itd); 1693 itd = NULL; 1694 } 1695 } 1696 stream->next_uframe = next_uframe; 1697 1698 /* don't need that schedule data any more */ 1699 iso_sched_free (stream, iso_sched); 1700 urb->hcpriv = NULL; 1701 1702 timer_action (ehci, TIMER_IO_WATCHDOG); 1703 return enable_periodic(ehci); 1704} 1705 1706#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) 1707 1708/* Process and recycle a completed ITD. Return true iff its urb completed, 1709 * and hence its completion callback probably added things to the hardware 1710 * schedule. 1711 * 1712 * Note that we carefully avoid recycling this descriptor until after any 1713 * completion callback runs, so that it won't be reused quickly. That is, 1714 * assuming (a) no more than two urbs per frame on this endpoint, and also 1715 * (b) only this endpoint's completions submit URBs. It seems some silicon 1716 * corrupts things if you reuse completed descriptors very quickly... 1717 */ 1718static unsigned 1719itd_complete ( 1720 struct ehci_hcd *ehci, 1721 struct ehci_itd *itd 1722) { 1723 struct urb *urb = itd->urb; 1724 struct usb_iso_packet_descriptor *desc; 1725 u32 t; 1726 unsigned uframe; 1727 int urb_index = -1; 1728 struct ehci_iso_stream *stream = itd->stream; 1729 struct usb_device *dev; 1730 unsigned retval = false; 1731 1732 /* for each uframe with a packet */ 1733 for (uframe = 0; uframe < 8; uframe++) { 1734 if (likely (itd->index[uframe] == -1)) 1735 continue; 1736 urb_index = itd->index[uframe]; 1737 desc = &urb->iso_frame_desc [urb_index]; 1738 1739 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]); 1740 itd->hw_transaction [uframe] = 0; 1741 1742 /* report transfer status */ 1743 if (unlikely (t & ISO_ERRS)) { 1744 urb->error_count++; 1745 if (t & EHCI_ISOC_BUF_ERR) 1746 desc->status = usb_pipein (urb->pipe) 1747 ? -ENOSR /* hc couldn't read */ 1748 : -ECOMM; /* hc couldn't write */ 1749 else if (t & EHCI_ISOC_BABBLE) 1750 desc->status = -EOVERFLOW; 1751 else /* (t & EHCI_ISOC_XACTERR) */ 1752 desc->status = -EPROTO; 1753 1754 /* HC need not update length with this error */ 1755 if (!(t & EHCI_ISOC_BABBLE)) { 1756 desc->actual_length = EHCI_ITD_LENGTH(t); 1757 urb->actual_length += desc->actual_length; 1758 } 1759 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) { 1760 desc->status = 0; 1761 desc->actual_length = EHCI_ITD_LENGTH(t); 1762 urb->actual_length += desc->actual_length; 1763 } else { 1764 /* URB was too late */ 1765 desc->status = -EXDEV; 1766 } 1767 } 1768 1769 /* handle completion now? */ 1770 if (likely ((urb_index + 1) != urb->number_of_packets)) 1771 goto done; 1772 1773 /* ASSERT: it's really the last itd for this urb 1774 list_for_each_entry (itd, &stream->td_list, itd_list) 1775 BUG_ON (itd->urb == urb); 1776 */ 1777 1778 /* give urb back to the driver; completion often (re)submits */ 1779 dev = urb->dev; 1780 ehci_urb_done(ehci, urb, 0); 1781 retval = true; 1782 urb = NULL; 1783 (void) disable_periodic(ehci); 1784 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 1785 1786 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1787 if (ehci->amd_l1_fix == 1) 1788 ehci_quirk_amd_L1(ehci, 0); 1789 } 1790 1791 if (unlikely(list_is_singular(&stream->td_list))) { 1792 ehci_to_hcd(ehci)->self.bandwidth_allocated 1793 -= stream->bandwidth; 1794 ehci_vdbg (ehci, 1795 "deschedule devp %s ep%d%s-iso\n", 1796 dev->devpath, stream->bEndpointAddress & 0x0f, 1797 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 1798 } 1799 iso_stream_put (ehci, stream); 1800 1801done: 1802 itd->urb = NULL; 1803 if (ehci->clock_frame != itd->frame || itd->index[7] != -1) { 1804 /* OK to recycle this ITD now. */ 1805 itd->stream = NULL; 1806 list_move(&itd->itd_list, &stream->free_list); 1807 iso_stream_put(ehci, stream); 1808 } else { 1809 /* HW might remember this ITD, so we can't recycle it yet. 1810 * Move it to a safe place until a new frame starts. 1811 */ 1812 list_move(&itd->itd_list, &ehci->cached_itd_list); 1813 if (stream->refcount == 2) { 1814 /* If iso_stream_put() were called here, stream 1815 * would be freed. Instead, just prevent reuse. 1816 */ 1817 stream->ep->hcpriv = NULL; 1818 stream->ep = NULL; 1819 } 1820 } 1821 return retval; 1822} 1823 1824/*-------------------------------------------------------------------------*/ 1825 1826static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, 1827 gfp_t mem_flags) 1828{ 1829 int status = -EINVAL; 1830 unsigned long flags; 1831 struct ehci_iso_stream *stream; 1832 1833 /* Get iso_stream head */ 1834 stream = iso_stream_find (ehci, urb); 1835 if (unlikely (stream == NULL)) { 1836 ehci_dbg (ehci, "can't get iso stream\n"); 1837 return -ENOMEM; 1838 } 1839 if (unlikely (urb->interval != stream->interval)) { 1840 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 1841 stream->interval, urb->interval); 1842 goto done; 1843 } 1844 1845#ifdef EHCI_URB_TRACE 1846 ehci_dbg (ehci, 1847 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", 1848 __func__, urb->dev->devpath, urb, 1849 usb_pipeendpoint (urb->pipe), 1850 usb_pipein (urb->pipe) ? "in" : "out", 1851 urb->transfer_buffer_length, 1852 urb->number_of_packets, urb->interval, 1853 stream); 1854#endif 1855 1856 /* allocate ITDs w/o locking anything */ 1857 status = itd_urb_transaction (stream, ehci, urb, mem_flags); 1858 if (unlikely (status < 0)) { 1859 ehci_dbg (ehci, "can't init itds\n"); 1860 goto done; 1861 } 1862 1863 /* schedule ... need to lock */ 1864 spin_lock_irqsave (&ehci->lock, flags); 1865 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 1866 status = -ESHUTDOWN; 1867 goto done_not_linked; 1868 } 1869 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 1870 if (unlikely(status)) 1871 goto done_not_linked; 1872 status = iso_stream_schedule(ehci, urb, stream); 1873 if (likely (status == 0)) 1874 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 1875 else 1876 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 1877done_not_linked: 1878 spin_unlock_irqrestore (&ehci->lock, flags); 1879 1880done: 1881 if (unlikely (status < 0)) 1882 iso_stream_put (ehci, stream); 1883 return status; 1884} 1885 1886/*-------------------------------------------------------------------------*/ 1887 1888/* 1889 * "Split ISO TDs" ... used for USB 1.1 devices going through the 1890 * TTs in USB 2.0 hubs. These need microframe scheduling. 1891 */ 1892 1893static inline void 1894sitd_sched_init( 1895 struct ehci_hcd *ehci, 1896 struct ehci_iso_sched *iso_sched, 1897 struct ehci_iso_stream *stream, 1898 struct urb *urb 1899) 1900{ 1901 unsigned i; 1902 dma_addr_t dma = urb->transfer_dma; 1903 1904 /* how many frames are needed for these transfers */ 1905 iso_sched->span = urb->number_of_packets * stream->interval; 1906 1907 /* figure out per-frame sitd fields that we'll need later 1908 * when we fit new sitds into the schedule. 1909 */ 1910 for (i = 0; i < urb->number_of_packets; i++) { 1911 struct ehci_iso_packet *packet = &iso_sched->packet [i]; 1912 unsigned length; 1913 dma_addr_t buf; 1914 u32 trans; 1915 1916 length = urb->iso_frame_desc [i].length & 0x03ff; 1917 buf = dma + urb->iso_frame_desc [i].offset; 1918 1919 trans = SITD_STS_ACTIVE; 1920 if (((i + 1) == urb->number_of_packets) 1921 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1922 trans |= SITD_IOC; 1923 trans |= length << 16; 1924 packet->transaction = cpu_to_hc32(ehci, trans); 1925 1926 /* might need to cross a buffer page within a td */ 1927 packet->bufp = buf; 1928 packet->buf1 = (buf + length) & ~0x0fff; 1929 if (packet->buf1 != (buf & ~(u64)0x0fff)) 1930 packet->cross = 1; 1931 1932 /* OUT uses multiple start-splits */ 1933 if (stream->bEndpointAddress & USB_DIR_IN) 1934 continue; 1935 length = (length + 187) / 188; 1936 if (length > 1) /* BEGIN vs ALL */ 1937 length |= 1 << 3; 1938 packet->buf1 |= length; 1939 } 1940} 1941 1942static int 1943sitd_urb_transaction ( 1944 struct ehci_iso_stream *stream, 1945 struct ehci_hcd *ehci, 1946 struct urb *urb, 1947 gfp_t mem_flags 1948) 1949{ 1950 struct ehci_sitd *sitd; 1951 dma_addr_t sitd_dma; 1952 int i; 1953 struct ehci_iso_sched *iso_sched; 1954 unsigned long flags; 1955 1956 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1957 if (iso_sched == NULL) 1958 return -ENOMEM; 1959 1960 sitd_sched_init(ehci, iso_sched, stream, urb); 1961 1962 /* allocate/init sITDs */ 1963 spin_lock_irqsave (&ehci->lock, flags); 1964 for (i = 0; i < urb->number_of_packets; i++) { 1965 1966 /* NOTE: for now, we don't try to handle wraparound cases 1967 * for IN (using sitd->hw_backpointer, like a FSTN), which 1968 * means we never need two sitds for full speed packets. 1969 */ 1970 1971 /* free_list.next might be cache-hot ... but maybe 1972 * the HC caches it too. avoid that issue for now. 1973 */ 1974 1975 /* prefer previously-allocated sitds */ 1976 if (!list_empty(&stream->free_list)) { 1977 sitd = list_entry (stream->free_list.prev, 1978 struct ehci_sitd, sitd_list); 1979 list_del (&sitd->sitd_list); 1980 sitd_dma = sitd->sitd_dma; 1981 } else { 1982 spin_unlock_irqrestore (&ehci->lock, flags); 1983 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags, 1984 &sitd_dma); 1985 spin_lock_irqsave (&ehci->lock, flags); 1986 if (!sitd) { 1987 iso_sched_free(stream, iso_sched); 1988 spin_unlock_irqrestore(&ehci->lock, flags); 1989 return -ENOMEM; 1990 } 1991 } 1992 1993 memset (sitd, 0, sizeof *sitd); 1994 sitd->sitd_dma = sitd_dma; 1995 list_add (&sitd->sitd_list, &iso_sched->td_list); 1996 } 1997 1998 /* temporarily store schedule info in hcpriv */ 1999 urb->hcpriv = iso_sched; 2000 urb->error_count = 0; 2001 2002 spin_unlock_irqrestore (&ehci->lock, flags); 2003 return 0; 2004} 2005 2006/*-------------------------------------------------------------------------*/ 2007 2008static inline void 2009sitd_patch( 2010 struct ehci_hcd *ehci, 2011 struct ehci_iso_stream *stream, 2012 struct ehci_sitd *sitd, 2013 struct ehci_iso_sched *iso_sched, 2014 unsigned index 2015) 2016{ 2017 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 2018 u64 bufp = uf->bufp; 2019 2020 sitd->hw_next = EHCI_LIST_END(ehci); 2021 sitd->hw_fullspeed_ep = stream->address; 2022 sitd->hw_uframe = stream->splits; 2023 sitd->hw_results = uf->transaction; 2024 sitd->hw_backpointer = EHCI_LIST_END(ehci); 2025 2026 bufp = uf->bufp; 2027 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp); 2028 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32); 2029 2030 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1); 2031 if (uf->cross) 2032 bufp += 4096; 2033 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32); 2034 sitd->index = index; 2035} 2036 2037static inline void 2038sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) 2039{ 2040 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ 2041 sitd->sitd_next = ehci->pshadow [frame]; 2042 sitd->hw_next = ehci->periodic [frame]; 2043 ehci->pshadow [frame].sitd = sitd; 2044 sitd->frame = frame; 2045 wmb (); 2046 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD); 2047} 2048 2049/* fit urb's sitds into the selected schedule slot; activate as needed */ 2050static int 2051sitd_link_urb ( 2052 struct ehci_hcd *ehci, 2053 struct urb *urb, 2054 unsigned mod, 2055 struct ehci_iso_stream *stream 2056) 2057{ 2058 int packet; 2059 unsigned next_uframe; 2060 struct ehci_iso_sched *sched = urb->hcpriv; 2061 struct ehci_sitd *sitd; 2062 2063 next_uframe = stream->next_uframe; 2064 2065 if (list_empty(&stream->td_list)) { 2066 /* usbfs ignores TT bandwidth */ 2067 ehci_to_hcd(ehci)->self.bandwidth_allocated 2068 += stream->bandwidth; 2069 ehci_vdbg (ehci, 2070 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n", 2071 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 2072 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 2073 (next_uframe >> 3) & (ehci->periodic_size - 1), 2074 stream->interval, hc32_to_cpu(ehci, stream->splits)); 2075 } 2076 2077 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2078 if (ehci->amd_l1_fix == 1) 2079 ehci_quirk_amd_L1(ehci, 1); 2080 } 2081 2082 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 2083 2084 /* fill sITDs frame by frame */ 2085 for (packet = 0, sitd = NULL; 2086 packet < urb->number_of_packets; 2087 packet++) { 2088 2089 /* ASSERT: we have all necessary sitds */ 2090 BUG_ON (list_empty (&sched->td_list)); 2091 2092 /* ASSERT: no itds for this endpoint in this frame */ 2093 2094 sitd = list_entry (sched->td_list.next, 2095 struct ehci_sitd, sitd_list); 2096 list_move_tail (&sitd->sitd_list, &stream->td_list); 2097 sitd->stream = iso_stream_get (stream); 2098 sitd->urb = urb; 2099 2100 sitd_patch(ehci, stream, sitd, sched, packet); 2101 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1), 2102 sitd); 2103 2104 next_uframe += stream->interval << 3; 2105 } 2106 stream->next_uframe = next_uframe & (mod - 1); 2107 2108 /* don't need that schedule data any more */ 2109 iso_sched_free (stream, sched); 2110 urb->hcpriv = NULL; 2111 2112 timer_action (ehci, TIMER_IO_WATCHDOG); 2113 return enable_periodic(ehci); 2114} 2115 2116/*-------------------------------------------------------------------------*/ 2117 2118#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ 2119 | SITD_STS_XACT | SITD_STS_MMF) 2120 2121/* Process and recycle a completed SITD. Return true iff its urb completed, 2122 * and hence its completion callback probably added things to the hardware 2123 * schedule. 2124 * 2125 * Note that we carefully avoid recycling this descriptor until after any 2126 * completion callback runs, so that it won't be reused quickly. That is, 2127 * assuming (a) no more than two urbs per frame on this endpoint, and also 2128 * (b) only this endpoint's completions submit URBs. It seems some silicon 2129 * corrupts things if you reuse completed descriptors very quickly... 2130 */ 2131static unsigned 2132sitd_complete ( 2133 struct ehci_hcd *ehci, 2134 struct ehci_sitd *sitd 2135) { 2136 struct urb *urb = sitd->urb; 2137 struct usb_iso_packet_descriptor *desc; 2138 u32 t; 2139 int urb_index = -1; 2140 struct ehci_iso_stream *stream = sitd->stream; 2141 struct usb_device *dev; 2142 unsigned retval = false; 2143 2144 urb_index = sitd->index; 2145 desc = &urb->iso_frame_desc [urb_index]; 2146 t = hc32_to_cpup(ehci, &sitd->hw_results); 2147 2148 /* report transfer status */ 2149 if (t & SITD_ERRS) { 2150 urb->error_count++; 2151 if (t & SITD_STS_DBE) 2152 desc->status = usb_pipein (urb->pipe) 2153 ? -ENOSR /* hc couldn't read */ 2154 : -ECOMM; /* hc couldn't write */ 2155 else if (t & SITD_STS_BABBLE) 2156 desc->status = -EOVERFLOW; 2157 else /* XACT, MMF, etc */ 2158 desc->status = -EPROTO; 2159 } else { 2160 desc->status = 0; 2161 desc->actual_length = desc->length - SITD_LENGTH(t); 2162 urb->actual_length += desc->actual_length; 2163 } 2164 2165 /* handle completion now? */ 2166 if ((urb_index + 1) != urb->number_of_packets) 2167 goto done; 2168 2169 /* ASSERT: it's really the last sitd for this urb 2170 list_for_each_entry (sitd, &stream->td_list, sitd_list) 2171 BUG_ON (sitd->urb == urb); 2172 */ 2173 2174 /* give urb back to the driver; completion often (re)submits */ 2175 dev = urb->dev; 2176 ehci_urb_done(ehci, urb, 0); 2177 retval = true; 2178 urb = NULL; 2179 (void) disable_periodic(ehci); 2180 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 2181 2182 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2183 if (ehci->amd_l1_fix == 1) 2184 ehci_quirk_amd_L1(ehci, 0); 2185 } 2186 2187 if (list_is_singular(&stream->td_list)) { 2188 ehci_to_hcd(ehci)->self.bandwidth_allocated 2189 -= stream->bandwidth; 2190 ehci_vdbg (ehci, 2191 "deschedule devp %s ep%d%s-iso\n", 2192 dev->devpath, stream->bEndpointAddress & 0x0f, 2193 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 2194 } 2195 iso_stream_put (ehci, stream); 2196 2197done: 2198 sitd->urb = NULL; 2199 if (ehci->clock_frame != sitd->frame) { 2200 /* OK to recycle this SITD now. */ 2201 sitd->stream = NULL; 2202 list_move(&sitd->sitd_list, &stream->free_list); 2203 iso_stream_put(ehci, stream); 2204 } else { 2205 /* HW might remember this SITD, so we can't recycle it yet. 2206 * Move it to a safe place until a new frame starts. 2207 */ 2208 list_move(&sitd->sitd_list, &ehci->cached_sitd_list); 2209 if (stream->refcount == 2) { 2210 /* If iso_stream_put() were called here, stream 2211 * would be freed. Instead, just prevent reuse. 2212 */ 2213 stream->ep->hcpriv = NULL; 2214 stream->ep = NULL; 2215 } 2216 } 2217 return retval; 2218} 2219 2220 2221static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb, 2222 gfp_t mem_flags) 2223{ 2224 int status = -EINVAL; 2225 unsigned long flags; 2226 struct ehci_iso_stream *stream; 2227 2228 /* Get iso_stream head */ 2229 stream = iso_stream_find (ehci, urb); 2230 if (stream == NULL) { 2231 ehci_dbg (ehci, "can't get iso stream\n"); 2232 return -ENOMEM; 2233 } 2234 if (urb->interval != stream->interval) { 2235 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 2236 stream->interval, urb->interval); 2237 goto done; 2238 } 2239 2240#ifdef EHCI_URB_TRACE 2241 ehci_dbg (ehci, 2242 "submit %p dev%s ep%d%s-iso len %d\n", 2243 urb, urb->dev->devpath, 2244 usb_pipeendpoint (urb->pipe), 2245 usb_pipein (urb->pipe) ? "in" : "out", 2246 urb->transfer_buffer_length); 2247#endif 2248 2249 /* allocate SITDs */ 2250 status = sitd_urb_transaction (stream, ehci, urb, mem_flags); 2251 if (status < 0) { 2252 ehci_dbg (ehci, "can't init sitds\n"); 2253 goto done; 2254 } 2255 2256 /* schedule ... need to lock */ 2257 spin_lock_irqsave (&ehci->lock, flags); 2258 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 2259 status = -ESHUTDOWN; 2260 goto done_not_linked; 2261 } 2262 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 2263 if (unlikely(status)) 2264 goto done_not_linked; 2265 status = iso_stream_schedule(ehci, urb, stream); 2266 if (status == 0) 2267 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 2268 else 2269 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 2270done_not_linked: 2271 spin_unlock_irqrestore (&ehci->lock, flags); 2272 2273done: 2274 if (status < 0) 2275 iso_stream_put (ehci, stream); 2276 return status; 2277} 2278 2279/*-------------------------------------------------------------------------*/ 2280 2281static void free_cached_lists(struct ehci_hcd *ehci) 2282{ 2283 struct ehci_itd *itd, *n; 2284 struct ehci_sitd *sitd, *sn; 2285 2286 list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) { 2287 struct ehci_iso_stream *stream = itd->stream; 2288 itd->stream = NULL; 2289 list_move(&itd->itd_list, &stream->free_list); 2290 iso_stream_put(ehci, stream); 2291 } 2292 2293 list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) { 2294 struct ehci_iso_stream *stream = sitd->stream; 2295 sitd->stream = NULL; 2296 list_move(&sitd->sitd_list, &stream->free_list); 2297 iso_stream_put(ehci, stream); 2298 } 2299} 2300 2301/*-------------------------------------------------------------------------*/ 2302 2303static void 2304scan_periodic (struct ehci_hcd *ehci) 2305{ 2306 unsigned now_uframe, frame, clock, clock_frame, mod; 2307 unsigned modified; 2308 2309 mod = ehci->periodic_size << 3; 2310 2311 /* 2312 * When running, scan from last scan point up to "now" 2313 * else clean up by scanning everything that's left. 2314 * Touches as few pages as possible: cache-friendly. 2315 */ 2316 now_uframe = ehci->next_uframe; 2317 if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) { 2318 clock = ehci_readl(ehci, &ehci->regs->frame_index); 2319 clock_frame = (clock >> 3) & (ehci->periodic_size - 1); 2320 } else { 2321 clock = now_uframe + mod - 1; 2322 clock_frame = -1; 2323 } 2324 if (ehci->clock_frame != clock_frame) { 2325 free_cached_lists(ehci); 2326 ehci->clock_frame = clock_frame; 2327 } 2328 clock &= mod - 1; 2329 clock_frame = clock >> 3; 2330 2331 for (;;) { 2332 union ehci_shadow q, *q_p; 2333 __hc32 type, *hw_p; 2334 unsigned incomplete = false; 2335 2336 frame = now_uframe >> 3; 2337 2338restart: 2339 /* scan each element in frame's queue for completions */ 2340 q_p = &ehci->pshadow [frame]; 2341 hw_p = &ehci->periodic [frame]; 2342 q.ptr = q_p->ptr; 2343 type = Q_NEXT_TYPE(ehci, *hw_p); 2344 modified = 0; 2345 2346 while (q.ptr != NULL) { 2347 unsigned uf; 2348 union ehci_shadow temp; 2349 int live; 2350 2351 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state); 2352 switch (hc32_to_cpu(ehci, type)) { 2353 case Q_TYPE_QH: 2354 /* handle any completions */ 2355 temp.qh = qh_get (q.qh); 2356 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next); 2357 q = q.qh->qh_next; 2358 modified = qh_completions (ehci, temp.qh); 2359 if (unlikely(list_empty(&temp.qh->qtd_list) || 2360 temp.qh->needs_rescan)) 2361 intr_deschedule (ehci, temp.qh); 2362 qh_put (temp.qh); 2363 break; 2364 case Q_TYPE_FSTN: 2365 /* for "save place" FSTNs, look at QH entries 2366 * in the previous frame for completions. 2367 */ 2368 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) { 2369 dbg ("ignoring completions from FSTNs"); 2370 } 2371 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next); 2372 q = q.fstn->fstn_next; 2373 break; 2374 case Q_TYPE_ITD: 2375 /* If this ITD is still active, leave it for 2376 * later processing ... check the next entry. 2377 * No need to check for activity unless the 2378 * frame is current. 2379 */ 2380 if (frame == clock_frame && live) { 2381 rmb(); 2382 for (uf = 0; uf < 8; uf++) { 2383 if (q.itd->hw_transaction[uf] & 2384 ITD_ACTIVE(ehci)) 2385 break; 2386 } 2387 if (uf < 8) { 2388 incomplete = true; 2389 q_p = &q.itd->itd_next; 2390 hw_p = &q.itd->hw_next; 2391 type = Q_NEXT_TYPE(ehci, 2392 q.itd->hw_next); 2393 q = *q_p; 2394 break; 2395 } 2396 } 2397 2398 /* Take finished ITDs out of the schedule 2399 * and process them: recycle, maybe report 2400 * URB completion. HC won't cache the 2401 * pointer for much longer, if at all. 2402 */ 2403 *q_p = q.itd->itd_next; 2404 *hw_p = q.itd->hw_next; 2405 type = Q_NEXT_TYPE(ehci, q.itd->hw_next); 2406 wmb(); 2407 modified = itd_complete (ehci, q.itd); 2408 q = *q_p; 2409 break; 2410 case Q_TYPE_SITD: 2411 /* If this SITD is still active, leave it for 2412 * later processing ... check the next entry. 2413 * No need to check for activity unless the 2414 * frame is current. 2415 */ 2416 if (((frame == clock_frame) || 2417 (((frame + 1) & (ehci->periodic_size - 1)) 2418 == clock_frame)) 2419 && live 2420 && (q.sitd->hw_results & 2421 SITD_ACTIVE(ehci))) { 2422 2423 incomplete = true; 2424 q_p = &q.sitd->sitd_next; 2425 hw_p = &q.sitd->hw_next; 2426 type = Q_NEXT_TYPE(ehci, 2427 q.sitd->hw_next); 2428 q = *q_p; 2429 break; 2430 } 2431 2432 /* Take finished SITDs out of the schedule 2433 * and process them: recycle, maybe report 2434 * URB completion. 2435 */ 2436 *q_p = q.sitd->sitd_next; 2437 *hw_p = q.sitd->hw_next; 2438 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next); 2439 wmb(); 2440 modified = sitd_complete (ehci, q.sitd); 2441 q = *q_p; 2442 break; 2443 default: 2444 dbg ("corrupt type %d frame %d shadow %p", 2445 type, frame, q.ptr); 2446 // BUG (); 2447 q.ptr = NULL; 2448 } 2449 2450 /* assume completion callbacks modify the queue */ 2451 if (unlikely (modified)) { 2452 if (likely(ehci->periodic_sched > 0)) 2453 goto restart; 2454 /* short-circuit this scan */ 2455 now_uframe = clock; 2456 break; 2457 } 2458 } 2459 2460 /* If we can tell we caught up to the hardware, stop now. 2461 * We can't advance our scan without collecting the ISO 2462 * transfers that are still pending in this frame. 2463 */ 2464 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) { 2465 ehci->next_uframe = now_uframe; 2466 break; 2467 } 2468 2469 // latencies climb; that should be rare, but... 2470 // detect it, and just go all the way around. 2471 // FLR might help detect this case, so long as latencies 2472 // don't exceed periodic_size msec (default 1.024 sec). 2473 2474 2475 if (now_uframe == clock) { 2476 unsigned now; 2477 2478 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state) 2479 || ehci->periodic_sched == 0) 2480 break; 2481 ehci->next_uframe = now_uframe; 2482 now = ehci_readl(ehci, &ehci->regs->frame_index) & 2483 (mod - 1); 2484 if (now_uframe == now) 2485 break; 2486 2487 /* rescan the rest of this frame, then ... */ 2488 clock = now; 2489 clock_frame = clock >> 3; 2490 if (ehci->clock_frame != clock_frame) { 2491 free_cached_lists(ehci); 2492 ehci->clock_frame = clock_frame; 2493 } 2494 } else { 2495 now_uframe++; 2496 now_uframe &= mod - 1; 2497 } 2498 } 2499} 2500