if_patm_tx.c revision 147256
1/*- 2 * Copyright (c) 2003 3 * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * The TST allocation algorithm is from the IDT driver which is: 28 * 29 * Copyright (c) 2000, 2001 Richard Hodges and Matriplex, inc. 30 * All rights reserved. 31 * 32 * Copyright (c) 1996, 1997, 1998, 1999 Mark Tinguely 33 * All rights reserved. 34 * 35 * Author: Hartmut Brandt <harti@freebsd.org> 36 * 37 * Driver for IDT77252 based cards like ProSum's. 38 */ 39 40#include <sys/cdefs.h> 41__FBSDID("$FreeBSD: head/sys/dev/patm/if_patm_tx.c 147256 2005-06-10 16:49:24Z brooks $"); 42 43#include "opt_inet.h" 44#include "opt_natm.h" 45 46#include <sys/types.h> 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/malloc.h> 50#include <sys/kernel.h> 51#include <sys/bus.h> 52#include <sys/errno.h> 53#include <sys/conf.h> 54#include <sys/module.h> 55#include <sys/lock.h> 56#include <sys/mutex.h> 57#include <sys/sysctl.h> 58#include <sys/queue.h> 59#include <sys/condvar.h> 60#include <sys/endian.h> 61#include <vm/uma.h> 62 63#include <sys/sockio.h> 64#include <sys/mbuf.h> 65#include <sys/socket.h> 66 67#include <net/if.h> 68#include <net/if_media.h> 69#include <net/if_atm.h> 70#include <net/route.h> 71#ifdef ENABLE_BPF 72#include <net/bpf.h> 73#endif 74#include <netinet/in.h> 75#include <netinet/if_atm.h> 76 77#include <machine/bus.h> 78#include <machine/resource.h> 79#include <sys/bus.h> 80#include <sys/rman.h> 81#include <sys/mbpool.h> 82 83#include <dev/utopia/utopia.h> 84#include <dev/patm/idt77252reg.h> 85#include <dev/patm/if_patmvar.h> 86 87static struct mbuf *patm_tx_pad(struct patm_softc *sc, struct mbuf *m0); 88static void patm_launch(struct patm_softc *sc, struct patm_scd *scd); 89 90static struct patm_txmap *patm_txmap_get(struct patm_softc *); 91static void patm_load_txbuf(void *, bus_dma_segment_t *, int, 92 bus_size_t, int); 93 94static void patm_tst_alloc(struct patm_softc *sc, struct patm_vcc *vcc); 95static void patm_tst_free(struct patm_softc *sc, struct patm_vcc *vcc); 96static void patm_tst_timer(void *p); 97static void patm_tst_update(struct patm_softc *); 98 99static void patm_tct_start(struct patm_softc *sc, struct patm_vcc *); 100 101static const char *dump_scd(struct patm_softc *sc, struct patm_scd *scd) 102 __unused; 103static void patm_tct_print(struct patm_softc *sc, u_int cid) __unused; 104 105/* 106 * Structure for communication with the loader function for transmission 107 */ 108struct txarg { 109 struct patm_softc *sc; 110 struct patm_scd *scd; /* scheduling channel */ 111 struct patm_vcc *vcc; /* the VCC of this PDU */ 112 struct mbuf *mbuf; 113 u_int hdr; /* cell header */ 114}; 115 116static __inline u_int 117cbr2slots(struct patm_softc *sc, struct patm_vcc *vcc) 118{ 119 /* compute the number of slots we need, make sure to get at least 120 * the specified PCR */ 121 return ((u_int)(((uint64_t)(sc->mmap->tst_size - 1) * 122 vcc->vcc.tparam.pcr + IFP2IFATM(sc->ifp)->mib.pcr - 1) / IFP2IFATM(sc->ifp)->mib.pcr)); 123} 124 125static __inline u_int 126slots2cr(struct patm_softc *sc, u_int slots) 127{ 128 return ((slots * IFP2IFATM(sc->ifp)->mib.pcr + sc->mmap->tst_size - 2) / 129 (sc->mmap->tst_size - 1)); 130} 131 132/* check if we can open this one */ 133int 134patm_tx_vcc_can_open(struct patm_softc *sc, struct patm_vcc *vcc) 135{ 136 137 /* check resources */ 138 switch (vcc->vcc.traffic) { 139 140 case ATMIO_TRAFFIC_CBR: 141 { 142 u_int slots = cbr2slots(sc, vcc); 143 144 if (slots > sc->tst_free + sc->tst_reserve) 145 return (EINVAL); 146 break; 147 } 148 149 case ATMIO_TRAFFIC_VBR: 150 if (vcc->vcc.tparam.scr > sc->bwrem) 151 return (EINVAL); 152 if (vcc->vcc.tparam.pcr > IFP2IFATM(sc->ifp)->mib.pcr) 153 return (EINVAL); 154 if (vcc->vcc.tparam.scr > vcc->vcc.tparam.pcr || 155 vcc->vcc.tparam.mbs == 0) 156 return (EINVAL); 157 break; 158 159 case ATMIO_TRAFFIC_ABR: 160 if (vcc->vcc.tparam.tbe == 0 || 161 vcc->vcc.tparam.nrm == 0) 162 /* needed to compute CRM */ 163 return (EINVAL); 164 if (vcc->vcc.tparam.pcr > IFP2IFATM(sc->ifp)->mib.pcr || 165 vcc->vcc.tparam.icr > vcc->vcc.tparam.pcr || 166 vcc->vcc.tparam.mcr > vcc->vcc.tparam.icr) 167 return (EINVAL); 168 if (vcc->vcc.tparam.mcr > sc->bwrem || 169 vcc->vcc.tparam.icr > sc->bwrem) 170 return (EINVAL); 171 break; 172 } 173 174 return (0); 175} 176 177#define NEXT_TAG(T) do { \ 178 (T) = ((T) + 1) % IDT_TSQE_TAG_SPACE; \ 179 } while (0) 180 181/* 182 * open it 183 */ 184void 185patm_tx_vcc_open(struct patm_softc *sc, struct patm_vcc *vcc) 186{ 187 struct patm_scd *scd; 188 189 if (vcc->vcc.traffic == ATMIO_TRAFFIC_UBR) { 190 /* we use UBR0 */ 191 vcc->scd = sc->scd0; 192 vcc->vflags |= PATM_VCC_TX_OPEN; 193 return; 194 } 195 196 /* get an SCD */ 197 scd = patm_scd_alloc(sc); 198 if (scd == NULL) { 199 /* should not happen */ 200 patm_printf(sc, "out of SCDs\n"); 201 return; 202 } 203 vcc->scd = scd; 204 patm_scd_setup(sc, scd); 205 patm_tct_setup(sc, scd, vcc); 206 207 if (vcc->vcc.traffic != ATMIO_TRAFFIC_CBR) 208 patm_tct_start(sc, vcc); 209 210 vcc->vflags |= PATM_VCC_TX_OPEN; 211} 212 213/* 214 * close the given vcc for transmission 215 */ 216void 217patm_tx_vcc_close(struct patm_softc *sc, struct patm_vcc *vcc) 218{ 219 struct patm_scd *scd; 220 struct mbuf *m; 221 222 vcc->vflags |= PATM_VCC_TX_CLOSING; 223 224 if (vcc->vcc.traffic == ATMIO_TRAFFIC_UBR) { 225 /* let the queue PDUs go out */ 226 vcc->scd = NULL; 227 vcc->vflags &= ~(PATM_VCC_TX_OPEN | PATM_VCC_TX_CLOSING); 228 return; 229 } 230 scd = vcc->scd; 231 232 /* empty the waitq */ 233 for (;;) { 234 _IF_DEQUEUE(&scd->q, m); 235 if (m == NULL) 236 break; 237 m_freem(m); 238 } 239 240 if (scd->num_on_card == 0) { 241 /* we are idle */ 242 vcc->vflags &= ~PATM_VCC_TX_OPEN; 243 244 if (vcc->vcc.traffic == ATMIO_TRAFFIC_CBR) 245 patm_tst_free(sc, vcc); 246 247 patm_sram_write4(sc, scd->sram + 0, 0, 0, 0, 0); 248 patm_sram_write4(sc, scd->sram + 4, 0, 0, 0, 0); 249 patm_scd_free(sc, scd); 250 251 vcc->scd = NULL; 252 vcc->vflags &= ~PATM_VCC_TX_CLOSING; 253 254 return; 255 } 256 257 /* speed up transmission */ 258 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_UIER(vcc->cid, 0xff)); 259 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_ULACR(vcc->cid, 0xff)); 260 261 /* wait for the interrupt to drop the number to 0 */ 262 patm_debug(sc, VCC, "%u buffers still on card", scd->num_on_card); 263} 264 265/* transmission side finally closed */ 266void 267patm_tx_vcc_closed(struct patm_softc *sc, struct patm_vcc *vcc) 268{ 269 270 patm_debug(sc, VCC, "%u.%u TX closed", vcc->vcc.vpi, vcc->vcc.vci); 271 272 if (vcc->vcc.traffic == ATMIO_TRAFFIC_VBR) 273 sc->bwrem += vcc->vcc.tparam.scr; 274} 275 276/* 277 * Pull off packets from the interface queue and try to transmit them. 278 * If the transmission fails because of a full transmit channel, we drop 279 * packets for CBR and queue them for other channels up to limit. 280 * This limit should depend on the CDVT for VBR and ABR, but it doesn't. 281 */ 282void 283patm_start(struct ifnet *ifp) 284{ 285 struct patm_softc *sc = (struct patm_softc *)ifp->if_softc; 286 struct mbuf *m; 287 struct atm_pseudohdr *aph; 288 u_int vpi, vci, cid; 289 struct patm_vcc *vcc; 290 291 mtx_lock(&sc->mtx); 292 if (!(ifp->if_flags & IFF_RUNNING)) { 293 mtx_unlock(&sc->mtx); 294 return; 295 } 296 297 while (1) { 298 /* get a new mbuf */ 299 IF_DEQUEUE(&ifp->if_snd, m); 300 if (m == NULL) 301 break; 302 303 /* split of pseudo header */ 304 if (m->m_len < sizeof(*aph) && 305 (m = m_pullup(m, sizeof(*aph))) == NULL) { 306 sc->ifp->if_oerrors++; 307 continue; 308 } 309 310 aph = mtod(m, struct atm_pseudohdr *); 311 vci = ATM_PH_VCI(aph); 312 vpi = ATM_PH_VPI(aph); 313 m_adj(m, sizeof(*aph)); 314 315 /* reject empty packets */ 316 if (m->m_pkthdr.len == 0) { 317 m_freem(m); 318 sc->ifp->if_oerrors++; 319 continue; 320 } 321 322 /* check whether this is a legal vcc */ 323 if (!LEGAL_VPI(sc, vpi) || !LEGAL_VCI(sc, vci) || vci == 0) { 324 m_freem(m); 325 sc->ifp->if_oerrors++; 326 continue; 327 } 328 cid = PATM_CID(sc, vpi, vci); 329 vcc = sc->vccs[cid]; 330 if (vcc == NULL) { 331 m_freem(m); 332 sc->ifp->if_oerrors++; 333 continue; 334 } 335 336 /* must be multiple of 48 if not AAL5 */ 337 if (vcc->vcc.aal == ATMIO_AAL_0 || 338 vcc->vcc.aal == ATMIO_AAL_34) { 339 /* XXX AAL3/4 format? */ 340 if (m->m_pkthdr.len % 48 != 0 && 341 (m = patm_tx_pad(sc, m)) == NULL) { 342 sc->ifp->if_oerrors++; 343 continue; 344 } 345 } else if (vcc->vcc.aal == ATMIO_AAL_RAW) { 346 switch (vcc->vflags & PATM_RAW_FORMAT) { 347 348 default: 349 case PATM_RAW_CELL: 350 if (m->m_pkthdr.len != 53) { 351 sc->ifp->if_oerrors++; 352 m_freem(m); 353 continue; 354 } 355 break; 356 357 case PATM_RAW_NOHEC: 358 if (m->m_pkthdr.len != 52) { 359 sc->ifp->if_oerrors++; 360 m_freem(m); 361 continue; 362 } 363 break; 364 365 case PATM_RAW_CS: 366 if (m->m_pkthdr.len != 64) { 367 sc->ifp->if_oerrors++; 368 m_freem(m); 369 continue; 370 } 371 break; 372 } 373 } 374 375 /* save data */ 376 m->m_pkthdr.header = vcc; 377 378 /* try to put it on the channels queue */ 379 if (_IF_QFULL(&vcc->scd->q)) { 380 sc->ifp->if_oerrors++; 381 sc->stats.tx_qfull++; 382 m_freem(m); 383 continue; 384 } 385 _IF_ENQUEUE(&vcc->scd->q, m); 386 387#ifdef ENABLE_BPF 388 if (!(vcc->vcc.flags & ATMIO_FLAG_NG) && 389 (vcc->vcc.aal == ATMIO_AAL_5) && 390 (vcc->vcc.flags & ATM_PH_LLCSNAP)) 391 BPF_MTAP(ifp, m); 392#endif 393 394 /* kick the channel to life */ 395 patm_launch(sc, vcc->scd); 396 397 } 398 mtx_unlock(&sc->mtx); 399} 400 401/* 402 * Pad non-AAL5 packet to a multiple of 48-byte. 403 * We assume AAL0 only. We have still to decide on the format of AAL3/4. 404 */ 405static struct mbuf * 406patm_tx_pad(struct patm_softc *sc, struct mbuf *m0) 407{ 408 struct mbuf *last, *m; 409 u_int plen, pad, space; 410 411 plen = m_length(m0, &last); 412 if (plen != m0->m_pkthdr.len) { 413 patm_printf(sc, "%s: mbuf length mismatch %d %u\n", __func__, 414 m0->m_pkthdr.len, plen); 415 m0->m_pkthdr.len = plen; 416 if (plen == 0) { 417 m_freem(m0); 418 sc->ifp->if_oerrors++; 419 return (NULL); 420 } 421 if (plen % 48 == 0) 422 return (m0); 423 } 424 pad = 48 - plen % 48; 425 m0->m_pkthdr.len += pad; 426 if (M_WRITABLE(last)) { 427 if (M_TRAILINGSPACE(last) >= pad) { 428 bzero(last->m_data + last->m_len, pad); 429 last->m_len += pad; 430 return (m0); 431 } 432 space = M_LEADINGSPACE(last); 433 if (space + M_TRAILINGSPACE(last) >= pad) { 434 bcopy(last->m_data, last->m_data + space, last->m_len); 435 last->m_data -= space; 436 bzero(last->m_data + last->m_len, pad); 437 last->m_len += pad; 438 return (m0); 439 } 440 } 441 MGET(m, M_DONTWAIT, MT_DATA); 442 if (m == 0) { 443 m_freem(m0); 444 sc->ifp->if_oerrors++; 445 return (NULL); 446 } 447 bzero(mtod(m, u_char *), pad); 448 m->m_len = pad; 449 last->m_next = m; 450 451 return (m0); 452} 453 454/* 455 * Try to put as many packets from the channels queue onto the channel 456 */ 457static void 458patm_launch(struct patm_softc *sc, struct patm_scd *scd) 459{ 460 struct txarg a; 461 struct mbuf *m, *tmp; 462 u_int segs; 463 struct patm_txmap *map; 464 int error; 465 466 a.sc = sc; 467 a.scd = scd; 468 469 /* limit the number of outstanding packets to the tag space */ 470 while (scd->num_on_card < IDT_TSQE_TAG_SPACE) { 471 /* get the next packet */ 472 _IF_DEQUEUE(&scd->q, m); 473 if (m == NULL) 474 break; 475 476 a.vcc = m->m_pkthdr.header; 477 478 /* we must know the number of segments beforehand - count 479 * this may actually give a wrong number of segments for 480 * AAL_RAW where we still need to remove the cell header */ 481 segs = 0; 482 for (tmp = m; tmp != NULL; tmp = tmp->m_next) 483 if (tmp->m_len != 0) 484 segs++; 485 486 /* check whether there is space in the queue */ 487 if (segs >= scd->space) { 488 /* put back */ 489 _IF_PREPEND(&scd->q, m); 490 sc->stats.tx_out_of_tbds++; 491 break; 492 } 493 494 /* get a DMA map */ 495 if ((map = patm_txmap_get(sc)) == NULL) { 496 _IF_PREPEND(&scd->q, m); 497 sc->stats.tx_out_of_maps++; 498 break; 499 } 500 501 /* load the map */ 502 m->m_pkthdr.header = map; 503 a.mbuf = m; 504 505 /* handle AAL_RAW */ 506 if (a.vcc->vcc.aal == ATMIO_AAL_RAW) { 507 u_char hdr[4]; 508 509 m_copydata(m, 0, 4, hdr); 510 a.hdr = (hdr[0] << 24) | (hdr[1] << 16) | 511 (hdr[2] << 8) | hdr[3]; 512 513 switch (a.vcc->vflags & PATM_RAW_FORMAT) { 514 515 default: 516 case PATM_RAW_CELL: 517 m_adj(m, 5); 518 break; 519 520 case PATM_RAW_NOHEC: 521 m_adj(m, 4); 522 break; 523 524 case PATM_RAW_CS: 525 m_adj(m, 16); 526 break; 527 } 528 } else 529 a.hdr = IDT_TBD_HDR(a.vcc->vcc.vpi, a.vcc->vcc.vci, 530 0, 0); 531 532 error = bus_dmamap_load_mbuf(sc->tx_tag, map->map, m, 533 patm_load_txbuf, &a, BUS_DMA_NOWAIT); 534 if (error == EFBIG) { 535 if ((m = m_defrag(m, M_DONTWAIT)) == NULL) { 536 sc->ifp->if_oerrors++; 537 continue; 538 } 539 error = bus_dmamap_load_mbuf(sc->tx_tag, map->map, m, 540 patm_load_txbuf, &a, BUS_DMA_NOWAIT); 541 } 542 if (error != 0) { 543 sc->stats.tx_load_err++; 544 sc->ifp->if_oerrors++; 545 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 546 m_freem(m); 547 continue; 548 } 549 550 sc->ifp->if_opackets++; 551 } 552} 553 554/* 555 * Load the DMA segments into the scheduling channel 556 */ 557static void 558patm_load_txbuf(void *uarg, bus_dma_segment_t *segs, int nseg, 559 bus_size_t mapsize, int error) 560{ 561 struct txarg *a= uarg; 562 struct patm_scd *scd = a->scd; 563 u_int w1, w3, cnt; 564 struct idt_tbd *tbd = NULL; 565 u_int rest = mapsize; 566 567 if (error != 0) 568 return; 569 570 cnt = 0; 571 while (nseg > 0) { 572 if (segs->ds_len == 0) { 573 /* transmit buffer length must be > 0 */ 574 nseg--; 575 segs++; 576 continue; 577 } 578 /* rest after this buffer */ 579 rest -= segs->ds_len; 580 581 /* put together status word */ 582 w1 = 0; 583 if (rest < 48 /* && a->vcc->vcc.aal != ATMIO_AAL_5 */) 584 /* last cell is in this buffer */ 585 w1 |= IDT_TBD_EPDU; 586 587 if (a->vcc->vcc.aal == ATMIO_AAL_5) 588 w1 |= IDT_TBD_AAL5; 589 else if (a->vcc->vcc.aal == ATMIO_AAL_34) 590 w1 |= IDT_TBD_AAL34; 591 else 592 w1 |= IDT_TBD_AAL0; 593 594 w1 |= segs->ds_len; 595 596 /* AAL5 PDU length (unpadded) */ 597 if (a->vcc->vcc.aal == ATMIO_AAL_5) 598 w3 = mapsize; 599 else 600 w3 = 0; 601 602 if (rest == 0) 603 w1 |= IDT_TBD_TSIF | IDT_TBD_GTSI | 604 (scd->tag << IDT_TBD_TAG_SHIFT); 605 606 tbd = &scd->scq[scd->tail]; 607 608 tbd->flags = htole32(w1); 609 tbd->addr = htole32(segs->ds_addr); 610 tbd->aal5 = htole32(w3); 611 tbd->hdr = htole32(a->hdr); 612 613 patm_debug(a->sc, TX, "TBD(%u): %08x %08x %08x %08x", 614 scd->tail, w1, segs->ds_addr, w3, a->hdr); 615 616 /* got to next entry */ 617 if (++scd->tail == IDT_SCQ_SIZE) 618 scd->tail = 0; 619 cnt++; 620 nseg--; 621 segs++; 622 } 623 scd->space -= cnt; 624 scd->num_on_card++; 625 626 KASSERT(rest == 0, ("bad mbuf")); 627 KASSERT(cnt > 0, ("no segs")); 628 KASSERT(scd->space > 0, ("scq full")); 629 630 KASSERT(scd->on_card[scd->tag] == NULL, 631 ("scd on_card wedged %u%s", scd->tag, dump_scd(a->sc, scd))); 632 scd->on_card[scd->tag] = a->mbuf; 633 a->mbuf->m_pkthdr.csum_data = cnt; 634 635 NEXT_TAG(scd->tag); 636 637 patm_debug(a->sc, TX, "SCD tail %u (%lx:%lx)", scd->tail, 638 (u_long)scd->phy, (u_long)scd->phy + (scd->tail << IDT_TBD_SHIFT)); 639 patm_sram_write(a->sc, scd->sram, 640 scd->phy + (scd->tail << IDT_TBD_SHIFT)); 641 642 if (patm_sram_read(a->sc, a->vcc->cid * 8 + 3) & IDT_TCT_IDLE) { 643 /* 644 * if the connection is idle start it. We cannot rely 645 * on a flag set by patm_tx_idle() here, because sometimes 646 * the card seems to place an idle TSI into the TSQ but 647 * forgets to raise an interrupt. 648 */ 649 patm_nor_write(a->sc, IDT_NOR_TCMDQ, 650 IDT_TCMDQ_START(a->vcc->cid)); 651 } 652} 653 654/* 655 * packet transmitted 656 */ 657void 658patm_tx(struct patm_softc *sc, u_int stamp, u_int status) 659{ 660 u_int cid, tag, last; 661 struct mbuf *m; 662 struct patm_vcc *vcc; 663 struct patm_scd *scd; 664 struct patm_txmap *map; 665 666 /* get the connection */ 667 cid = PATM_CID(sc, IDT_TBD_VPI(status), IDT_TBD_VCI(status)); 668 if ((vcc = sc->vccs[cid]) == NULL) { 669 /* closed UBR connection */ 670 return; 671 } 672 scd = vcc->scd; 673 674 tag = IDT_TSQE_TAG(stamp); 675 676 last = scd->last_tag; 677 if (tag == last) { 678 patm_printf(sc, "same tag %u\n", tag); 679 return; 680 } 681 682 /* Errata 12 requests us to free all entries up to the one 683 * with the given tag. */ 684 do { 685 /* next tag to try */ 686 NEXT_TAG(last); 687 688 m = scd->on_card[last]; 689 KASSERT(m != NULL, ("%stag=%u", dump_scd(sc, scd), tag)); 690 scd->on_card[last] = NULL; 691 patm_debug(sc, TX, "ok tag=%x", last); 692 693 map = m->m_pkthdr.header; 694 scd->space += m->m_pkthdr.csum_data; 695 696 bus_dmamap_sync(sc->tx_tag, map->map, 697 BUS_DMASYNC_POSTWRITE); 698 bus_dmamap_unload(sc->tx_tag, map->map); 699 m_freem(m); 700 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 701 scd->num_on_card--; 702 703 if (vcc->vflags & PATM_VCC_TX_CLOSING) { 704 if (scd->num_on_card == 0) { 705 /* done with this VCC */ 706 if (vcc->vcc.traffic == ATMIO_TRAFFIC_CBR) 707 patm_tst_free(sc, vcc); 708 709 patm_sram_write4(sc, scd->sram + 0, 0, 0, 0, 0); 710 patm_sram_write4(sc, scd->sram + 4, 0, 0, 0, 0); 711 patm_scd_free(sc, scd); 712 713 vcc->scd = NULL; 714 vcc->vflags &= ~PATM_VCC_TX_CLOSING; 715 716 if (vcc->vcc.flags & ATMIO_FLAG_ASYNC) { 717 patm_tx_vcc_closed(sc, vcc); 718 if (!(vcc->vflags & PATM_VCC_OPEN)) 719 patm_vcc_closed(sc, vcc); 720 } else 721 cv_signal(&sc->vcc_cv); 722 return; 723 } 724 patm_debug(sc, VCC, "%u buffers still on card", 725 scd->num_on_card); 726 727 if (vcc->vcc.traffic == ATMIO_TRAFFIC_ABR) { 728 /* insist on speeding up transmission for ABR */ 729 patm_nor_write(sc, IDT_NOR_TCMDQ, 730 IDT_TCMDQ_UIER(vcc->cid, 0xff)); 731 patm_nor_write(sc, IDT_NOR_TCMDQ, 732 IDT_TCMDQ_ULACR(vcc->cid, 0xff)); 733 } 734 } 735 736 } while (last != tag); 737 scd->last_tag = tag; 738 739 if (vcc->vcc.traffic == ATMIO_TRAFFIC_ABR) { 740 u_int acri, cps; 741 742 acri = (patm_sram_read(sc, 8 * cid + 2) >> IDT_TCT_ACRI_SHIFT) 743 & 0x3fff; 744 cps = IFP2IFATM(sc->ifp)->mib.pcr * 32 / 745 ((1 << (acri >> 10)) * (acri & 0x3ff)); 746 747 if (cps != vcc->cps) { 748 patm_debug(sc, VCC, "ACRI=%04x CPS=%u", acri, cps); 749 ATMEV_SEND_ACR_CHANGED(IFP2IFATM(sc->ifp), vcc->vcc.vpi, 750 vcc->vcc.vci, cps); 751 vcc->cps = cps; 752 } 753 } 754 755 patm_launch(sc, scd); 756} 757 758/* 759 * VBR/ABR connection went idle 760 * Either restart it or set the idle flag. 761 */ 762void 763patm_tx_idle(struct patm_softc *sc, u_int cid) 764{ 765 struct patm_vcc *vcc; 766 767 patm_debug(sc, VCC, "idle %u", cid); 768 769 if ((vcc = sc->vccs[cid]) != NULL && 770 (vcc->vflags & (PATM_VCC_TX_OPEN | PATM_VCC_TX_CLOSING)) != 0 && 771 vcc->scd != NULL && (vcc->scd->num_on_card != 0 || 772 _IF_QLEN(&vcc->scd->q) != 0)) { 773 /* 774 * If there is any packet outstanding in the SCD re-activate 775 * the channel and kick it. 776 */ 777 patm_nor_write(sc, IDT_NOR_TCMDQ, 778 IDT_TCMDQ_START(vcc->cid)); 779 780 patm_launch(sc, vcc->scd); 781 } 782} 783 784/* 785 * Convert a (24bit) rate to the atm-forum form 786 * Our rate is never larger than 19 bit. 787 */ 788static u_int 789cps2atmf(u_int cps) 790{ 791 u_int e; 792 793 if (cps == 0) 794 return (0); 795 cps <<= 9; 796 e = 0; 797 while (cps > (1024 - 1)) { 798 e++; 799 cps >>= 1; 800 } 801 return ((1 << 14) | (e << 9) | (cps & 0x1ff)); 802} 803 804/* 805 * Do a binary search on the log2rate table to convert the rate 806 * to its log form. This assumes that the ATM-Forum form is monotonically 807 * increasing with the plain cell rate. 808 */ 809static u_int 810rate2log(struct patm_softc *sc, u_int rate) 811{ 812 const uint32_t *tbl; 813 u_int lower, upper, mid, done, val, afr; 814 815 afr = cps2atmf(rate); 816 817 if (sc->flags & PATM_25M) 818 tbl = patm_rtables25; 819 else 820 tbl = patm_rtables155; 821 822 lower = 0; 823 upper = 255; 824 done = 0; 825 while (!done) { 826 mid = (lower + upper) / 2; 827 val = tbl[mid] >> 17; 828 if (val == afr || upper == lower) 829 break; 830 if (afr > val) 831 lower = mid + 1; 832 else 833 upper = mid - 1; 834 } 835 if (val > afr && mid > 0) 836 mid--; 837 return (mid); 838} 839 840/* 841 * Return the table index for an increase table. The increase table 842 * must be selected not by the RIF itself, but by PCR/2^RIF. Each table 843 * represents an additive increase of a cell rate that can be computed 844 * from the first table entry (the value in this entry will not be clamped 845 * by the link rate). 846 */ 847static u_int 848get_air_table(struct patm_softc *sc, u_int rif, u_int pcr) 849{ 850 const uint32_t *tbl; 851 u_int increase, base, lair0, ret, t, cps; 852 853#define GET_ENTRY(TAB, IDX) (0xffff & ((IDX & 1) ? \ 854 (tbl[512 + (IDX / 2) + 128 * (TAB)] >> 16) : \ 855 (tbl[512 + (IDX / 2) + 128 * (TAB)]))) 856 857#define MANT_BITS 10 858#define FRAC_BITS 16 859 860#define DIFF_TO_FP(D) (((D) & ((1 << MANT_BITS) - 1)) << ((D) >> MANT_BITS)) 861#define AFR_TO_INT(A) ((1 << (((A) >> 9) & 0x1f)) * \ 862 (512 + ((A) & 0x1ff)) / 512 * ((A) >> 14)) 863 864 if (sc->flags & PATM_25M) 865 tbl = patm_rtables25; 866 else 867 tbl = patm_rtables155; 868 if (rif >= patm_rtables_ntab) 869 rif = patm_rtables_ntab - 1; 870 increase = pcr >> rif; 871 872 ret = 0; 873 for (t = 0; t < patm_rtables_ntab; t++) { 874 /* get base rate of this table */ 875 base = GET_ENTRY(t, 0); 876 /* convert this to fixed point */ 877 lair0 = DIFF_TO_FP(base) >> FRAC_BITS; 878 879 /* get the CPS from the log2rate table */ 880 cps = AFR_TO_INT(tbl[lair0] >> 17) - 10; 881 882 if (increase >= cps) 883 break; 884 885 ret = t; 886 } 887 return (ret + 4); 888} 889 890/* 891 * Setup the TCT 892 */ 893void 894patm_tct_setup(struct patm_softc *sc, struct patm_scd *scd, 895 struct patm_vcc *vcc) 896{ 897 uint32_t tct[8]; 898 u_int sram; 899 u_int mbs, token; 900 u_int tmp, crm, rdf, cdf, air, mcr; 901 902 bzero(tct, sizeof(tct)); 903 if (vcc == NULL) { 904 /* special case for UBR0 */ 905 sram = 0; 906 tct[0] = IDT_TCT_UBR | scd->sram; 907 tct[7] = IDT_TCT_UBR_FLG; 908 909 } else { 910 sram = vcc->cid * 8; 911 switch (vcc->vcc.traffic) { 912 913 case ATMIO_TRAFFIC_CBR: 914 patm_tst_alloc(sc, vcc); 915 tct[0] = IDT_TCT_CBR | scd->sram; 916 /* must account for what was really allocated */ 917 break; 918 919 case ATMIO_TRAFFIC_VBR: 920 /* compute parameters for the TCT */ 921 scd->init_er = rate2log(sc, vcc->vcc.tparam.pcr); 922 scd->lacr = rate2log(sc, vcc->vcc.tparam.scr); 923 924 /* get the 16-bit fraction of SCR/PCR 925 * both a 24 bit. Do it the simple way. */ 926 token = (uint64_t)(vcc->vcc.tparam.scr << 16) / 927 vcc->vcc.tparam.pcr; 928 929 patm_debug(sc, VCC, "VBR: init_er=%u lacr=%u " 930 "token=0x%04x\n", scd->init_er, scd->lacr, token); 931 932 tct[0] = IDT_TCT_VBR | scd->sram; 933 tct[2] = IDT_TCT_TSIF; 934 tct[3] = IDT_TCT_IDLE | IDT_TCT_HALT; 935 tct[4] = IDT_TCT_MAXIDLE; 936 tct[5] = 0x01000000; 937 if ((mbs = vcc->vcc.tparam.mbs) > 0xff) 938 mbs = 0xff; 939 tct[6] = (mbs << 16) | token; 940 sc->bwrem -= vcc->vcc.tparam.scr; 941 break; 942 943 case ATMIO_TRAFFIC_ABR: 944 scd->init_er = rate2log(sc, vcc->vcc.tparam.pcr); 945 scd->lacr = rate2log(sc, vcc->vcc.tparam.icr); 946 mcr = rate2log(sc, vcc->vcc.tparam.mcr); 947 948 /* compute CRM */ 949 tmp = vcc->vcc.tparam.tbe / vcc->vcc.tparam.nrm; 950 if (tmp * vcc->vcc.tparam.nrm < vcc->vcc.tparam.tbe) 951 tmp++; 952 for (crm = 1; tmp > (1 << crm); crm++) 953 ; 954 if (crm > 0x7) 955 crm = 7; 956 957 air = get_air_table(sc, vcc->vcc.tparam.rif, 958 vcc->vcc.tparam.pcr); 959 960 if ((rdf = vcc->vcc.tparam.rdf) >= patm_rtables_ntab) 961 rdf = patm_rtables_ntab - 1; 962 rdf += patm_rtables_ntab + 4; 963 964 if ((cdf = vcc->vcc.tparam.cdf) >= patm_rtables_ntab) 965 cdf = patm_rtables_ntab - 1; 966 cdf += patm_rtables_ntab + 4; 967 968 patm_debug(sc, VCC, "ABR: init_er=%u lacr=%u mcr=%u " 969 "crm=%u air=%u rdf=%u cdf=%u\n", scd->init_er, 970 scd->lacr, mcr, crm, air, rdf, cdf); 971 972 tct[0] = IDT_TCT_ABR | scd->sram; 973 tct[1] = crm << IDT_TCT_CRM_SHIFT; 974 tct[3] = IDT_TCT_HALT | IDT_TCT_IDLE | 975 (4 << IDT_TCT_NAGE_SHIFT); 976 tct[4] = mcr << IDT_TCT_LMCR_SHIFT; 977 tct[5] = (cdf << IDT_TCT_CDF_SHIFT) | 978 (rdf << IDT_TCT_RDF_SHIFT) | 979 (air << IDT_TCT_AIR_SHIFT); 980 981 sc->bwrem -= vcc->vcc.tparam.mcr; 982 break; 983 } 984 } 985 986 patm_sram_write4(sc, sram + 0, tct[0], tct[1], tct[2], tct[3]); 987 patm_sram_write4(sc, sram + 4, tct[4], tct[5], tct[6], tct[7]); 988 989 patm_debug(sc, VCC, "TCT[%u]: %08x %08x %08x %08x %08x %08x %08x %08x", 990 sram / 8, patm_sram_read(sc, sram + 0), 991 patm_sram_read(sc, sram + 1), patm_sram_read(sc, sram + 2), 992 patm_sram_read(sc, sram + 3), patm_sram_read(sc, sram + 4), 993 patm_sram_read(sc, sram + 5), patm_sram_read(sc, sram + 6), 994 patm_sram_read(sc, sram + 7)); 995} 996 997/* 998 * Start a channel 999 */ 1000static void 1001patm_tct_start(struct patm_softc *sc, struct patm_vcc *vcc) 1002{ 1003 1004 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_UIER(vcc->cid, 1005 vcc->scd->init_er)); 1006 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_SLACR(vcc->cid, 1007 vcc->scd->lacr)); 1008} 1009 1010static void 1011patm_tct_print(struct patm_softc *sc, u_int cid) 1012{ 1013#ifdef PATM_DEBUG 1014 u_int sram = cid * 8; 1015#endif 1016 1017 patm_debug(sc, VCC, "TCT[%u]: %08x %08x %08x %08x %08x %08x %08x %08x", 1018 sram / 8, patm_sram_read(sc, sram + 0), 1019 patm_sram_read(sc, sram + 1), patm_sram_read(sc, sram + 2), 1020 patm_sram_read(sc, sram + 3), patm_sram_read(sc, sram + 4), 1021 patm_sram_read(sc, sram + 5), patm_sram_read(sc, sram + 6), 1022 patm_sram_read(sc, sram + 7)); 1023} 1024 1025/* 1026 * Setup the SCD 1027 */ 1028void 1029patm_scd_setup(struct patm_softc *sc, struct patm_scd *scd) 1030{ 1031 patm_sram_write4(sc, scd->sram + 0, 1032 scd->phy, 0, 0xffffffff, 0); 1033 patm_sram_write4(sc, scd->sram + 4, 1034 0, 0, 0, 0); 1035 1036 patm_debug(sc, VCC, "SCD(%x): %08x %08x %08x %08x %08x %08x %08x %08x", 1037 scd->sram, 1038 patm_sram_read(sc, scd->sram + 0), 1039 patm_sram_read(sc, scd->sram + 1), 1040 patm_sram_read(sc, scd->sram + 2), 1041 patm_sram_read(sc, scd->sram + 3), 1042 patm_sram_read(sc, scd->sram + 4), 1043 patm_sram_read(sc, scd->sram + 5), 1044 patm_sram_read(sc, scd->sram + 6), 1045 patm_sram_read(sc, scd->sram + 7)); 1046} 1047 1048/* 1049 * Grow the TX map table if possible 1050 */ 1051static void 1052patm_txmaps_grow(struct patm_softc *sc) 1053{ 1054 u_int i; 1055 struct patm_txmap *map; 1056 int err; 1057 1058 if (sc->tx_nmaps >= sc->tx_maxmaps) 1059 return; 1060 1061 for (i = sc->tx_nmaps; i < sc->tx_nmaps + PATM_CFG_TXMAPS_STEP; i++) { 1062 map = uma_zalloc(sc->tx_mapzone, M_NOWAIT); 1063 err = bus_dmamap_create(sc->tx_tag, 0, &map->map); 1064 if (err) { 1065 uma_zfree(sc->tx_mapzone, map); 1066 break; 1067 } 1068 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 1069 } 1070 1071 sc->tx_nmaps = i; 1072} 1073 1074/* 1075 * Allocate a transmission map 1076 */ 1077static struct patm_txmap * 1078patm_txmap_get(struct patm_softc *sc) 1079{ 1080 struct patm_txmap *map; 1081 1082 if ((map = SLIST_FIRST(&sc->tx_maps_free)) == NULL) { 1083 patm_txmaps_grow(sc); 1084 if ((map = SLIST_FIRST(&sc->tx_maps_free)) == NULL) 1085 return (NULL); 1086 } 1087 SLIST_REMOVE_HEAD(&sc->tx_maps_free, link); 1088 return (map); 1089} 1090 1091/* 1092 * Look whether we are in the process of updating the TST on the chip. 1093 * If we are set the flag that we need another update. 1094 * If we are not start the update. 1095 */ 1096static __inline void 1097patm_tst_start(struct patm_softc *sc) 1098{ 1099 1100 if (!(sc->tst_state & TST_PENDING)) { 1101 sc->tst_state |= TST_PENDING; 1102 if (!(sc->tst_state & TST_WAIT)) { 1103 /* timer not running */ 1104 patm_tst_update(sc); 1105 } 1106 } 1107} 1108 1109/* 1110 * Allocate TST entries to a CBR connection 1111 */ 1112static void 1113patm_tst_alloc(struct patm_softc *sc, struct patm_vcc *vcc) 1114{ 1115 u_int slots; 1116 u_int qptr, pptr; 1117 u_int qmax, pmax; 1118 u_int pspc, last; 1119 1120 mtx_lock(&sc->tst_lock); 1121 1122 /* compute the number of slots we need, make sure to get at least 1123 * the specified PCR */ 1124 slots = cbr2slots(sc, vcc); 1125 vcc->scd->slots = slots; 1126 sc->bwrem -= slots2cr(sc, slots); 1127 1128 patm_debug(sc, TST, "tst_alloc: cbr=%u link=%u tst=%u slots=%u", 1129 vcc->vcc.tparam.pcr, IFP2IFATM(sc->ifp)->mib.pcr, sc->mmap->tst_size, slots); 1130 1131 qmax = sc->mmap->tst_size - 1; 1132 pmax = qmax << 8; 1133 1134 pspc = pmax / slots; 1135 1136 pptr = pspc >> 1; /* starting point */ 1137 qptr = pptr >> 8; 1138 1139 last = qptr; 1140 1141 while (slots > 0) { 1142 if (qptr >= qmax) 1143 qptr -= qmax; 1144 if (sc->tst_soft[qptr] != IDT_TST_VBR) { 1145 /* used - try next */ 1146 qptr++; 1147 continue; 1148 } 1149 patm_debug(sc, TST, "slot[%u] = %u.%u diff=%d", qptr, 1150 vcc->vcc.vpi, vcc->vcc.vci, (int)qptr - (int)last); 1151 last = qptr; 1152 1153 sc->tst_soft[qptr] = IDT_TST_CBR | vcc->cid | TST_BOTH; 1154 sc->tst_free--; 1155 1156 if ((pptr += pspc) >= pmax) 1157 pptr -= pmax; 1158 qptr = pptr >> 8; 1159 1160 slots--; 1161 } 1162 patm_tst_start(sc); 1163 mtx_unlock(&sc->tst_lock); 1164} 1165 1166/* 1167 * Free a CBR connection's TST entries 1168 */ 1169static void 1170patm_tst_free(struct patm_softc *sc, struct patm_vcc *vcc) 1171{ 1172 u_int i; 1173 1174 mtx_lock(&sc->tst_lock); 1175 for (i = 0; i < sc->mmap->tst_size - 1; i++) { 1176 if ((sc->tst_soft[i] & IDT_TST_MASK) == vcc->cid) { 1177 sc->tst_soft[i] = IDT_TST_VBR | TST_BOTH; 1178 sc->tst_free++; 1179 } 1180 } 1181 sc->bwrem += slots2cr(sc, vcc->scd->slots); 1182 patm_tst_start(sc); 1183 mtx_unlock(&sc->tst_lock); 1184} 1185 1186/* 1187 * Write the soft TST into the idle incore TST and start the wait timer. 1188 * We assume that we hold the tst lock. 1189 */ 1190static void 1191patm_tst_update(struct patm_softc *sc) 1192{ 1193 u_int flag; /* flag to clear from soft TST */ 1194 u_int idle; /* the idle TST */ 1195 u_int act; /* the active TST */ 1196 u_int i; 1197 1198 if (sc->tst_state & TST_ACT1) { 1199 act = 1; 1200 idle = 0; 1201 flag = TST_CH0; 1202 } else { 1203 act = 0; 1204 idle = 1; 1205 flag = TST_CH1; 1206 } 1207 /* update the idle one */ 1208 for (i = 0; i < sc->mmap->tst_size - 1; i++) 1209 if (sc->tst_soft[i] & flag) { 1210 patm_sram_write(sc, sc->tst_base[idle] + i, 1211 sc->tst_soft[i] & ~TST_BOTH); 1212 sc->tst_soft[i] &= ~flag; 1213 } 1214 /* the used one jump to the idle one */ 1215 patm_sram_write(sc, sc->tst_jump[act], 1216 IDT_TST_BR | (sc->tst_base[idle] << 2)); 1217 1218 /* wait for the chip to jump */ 1219 sc->tst_state &= ~TST_PENDING; 1220 sc->tst_state |= TST_WAIT; 1221 1222 callout_reset(&sc->tst_callout, 1, patm_tst_timer, sc); 1223} 1224 1225/* 1226 * Timer for TST updates 1227 */ 1228static void 1229patm_tst_timer(void *p) 1230{ 1231 struct patm_softc *sc = p; 1232 u_int act; /* active TST */ 1233 u_int now; /* current place in TST */ 1234 1235 mtx_lock(&sc->tst_lock); 1236 1237 if (sc->tst_state & TST_WAIT) { 1238 /* ignore the PENDING state while we are waiting for 1239 * the chip to switch tables. Once the switch is done, 1240 * we will again lock at PENDING */ 1241 act = (sc->tst_state & TST_ACT1) ? 1 : 0; 1242 now = patm_nor_read(sc, IDT_NOR_NOW) >> 2; 1243 if (now >= sc->tst_base[act] && now <= sc->tst_jump[act]) { 1244 /* not yet */ 1245 callout_reset(&sc->tst_callout, 1, patm_tst_timer, sc); 1246 goto done; 1247 } 1248 sc->tst_state &= ~TST_WAIT; 1249 /* change back jump */ 1250 patm_sram_write(sc, sc->tst_jump[act], 1251 IDT_TST_BR | (sc->tst_base[act] << 2)); 1252 1253 /* switch */ 1254 sc->tst_state ^= TST_ACT1; 1255 } 1256 1257 if (sc->tst_state & TST_PENDING) 1258 /* we got another update request while the timer was running. */ 1259 patm_tst_update(sc); 1260 1261 done: 1262 mtx_unlock(&sc->tst_lock); 1263} 1264 1265static const char * 1266dump_scd(struct patm_softc *sc, struct patm_scd *scd) 1267{ 1268 u_int i; 1269 1270 for (i = 0; i < IDT_TSQE_TAG_SPACE; i++) 1271 printf("on_card[%u] = %p\n", i, scd->on_card[i]); 1272 printf("space=%u tag=%u num_on_card=%u last_tag=%u\n", 1273 scd->space, scd->tag, scd->num_on_card, scd->last_tag); 1274 1275 return (""); 1276} 1277