1/*- 2 * Copyright (c) 2003 3 * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * The TST allocation algorithm is from the IDT driver which is: 28 * 29 * Copyright (c) 2000, 2001 Richard Hodges and Matriplex, inc. 30 * All rights reserved. 31 * 32 * Copyright (c) 1996, 1997, 1998, 1999 Mark Tinguely 33 * All rights reserved. 34 * 35 * Author: Hartmut Brandt <harti@freebsd.org> 36 * 37 * Driver for IDT77252 based cards like ProSum's. 38 */ 39 40#include <sys/cdefs.h> 41__FBSDID("$FreeBSD: stable/11/sys/dev/patm/if_patm_tx.c 315221 2017-03-14 02:06:03Z pfg $"); 42 43#include "opt_inet.h" 44#include "opt_natm.h" 45 46#include <sys/types.h> 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/malloc.h> 50#include <sys/kernel.h> 51#include <sys/bus.h> 52#include <sys/errno.h> 53#include <sys/conf.h> 54#include <sys/module.h> 55#include <sys/lock.h> 56#include <sys/mutex.h> 57#include <sys/sysctl.h> 58#include <sys/queue.h> 59#include <sys/condvar.h> 60#include <sys/endian.h> 61#include <vm/uma.h> 62 63#include <sys/sockio.h> 64#include <sys/mbuf.h> 65#include <sys/socket.h> 66 67#include <net/if.h> 68#include <net/if_var.h> 69#include <net/if_media.h> 70#include <net/if_atm.h> 71#include <net/route.h> 72#ifdef ENABLE_BPF 73#include <net/bpf.h> 74#endif 75#include <netinet/in.h> 76#include <netinet/if_atm.h> 77 78#include <machine/bus.h> 79#include <machine/resource.h> 80#include <sys/bus.h> 81#include <sys/rman.h> 82#include <sys/mbpool.h> 83 84#include <dev/utopia/utopia.h> 85#include <dev/patm/idt77252reg.h> 86#include <dev/patm/if_patmvar.h> 87 88static struct mbuf *patm_tx_pad(struct patm_softc *sc, struct mbuf *m0); 89static void patm_launch(struct patm_softc *sc, struct patm_scd *scd); 90 91static struct patm_txmap *patm_txmap_get(struct patm_softc *); 92static void patm_load_txbuf(void *, bus_dma_segment_t *, int, 93 bus_size_t, int); 94 95static void patm_tst_alloc(struct patm_softc *sc, struct patm_vcc *vcc); 96static void patm_tst_free(struct patm_softc *sc, struct patm_vcc *vcc); 97static void patm_tst_timer(void *p); 98static void patm_tst_update(struct patm_softc *); 99 100static void patm_tct_start(struct patm_softc *sc, struct patm_vcc *); 101 102static const char *dump_scd(struct patm_softc *sc, struct patm_scd *scd) 103 __unused; 104static void patm_tct_print(struct patm_softc *sc, u_int cid) __unused; 105 106/* 107 * Structure for communication with the loader function for transmission 108 */ 109struct txarg { 110 struct patm_softc *sc; 111 struct patm_scd *scd; /* scheduling channel */ 112 struct patm_vcc *vcc; /* the VCC of this PDU */ 113 struct mbuf *mbuf; 114 u_int hdr; /* cell header */ 115}; 116 117static __inline u_int 118cbr2slots(struct patm_softc *sc, struct patm_vcc *vcc) 119{ 120 /* compute the number of slots we need, make sure to get at least 121 * the specified PCR */ 122 return ((u_int)howmany((uint64_t)(sc->mmap->tst_size - 1) * 123 vcc->vcc.tparam.pcr, IFP2IFATM(sc->ifp)->mib.pcr)); 124} 125 126static __inline u_int 127slots2cr(struct patm_softc *sc, u_int slots) 128{ 129 return ((slots * IFP2IFATM(sc->ifp)->mib.pcr + sc->mmap->tst_size - 2) / 130 (sc->mmap->tst_size - 1)); 131} 132 133/* check if we can open this one */ 134int 135patm_tx_vcc_can_open(struct patm_softc *sc, struct patm_vcc *vcc) 136{ 137 138 /* check resources */ 139 switch (vcc->vcc.traffic) { 140 141 case ATMIO_TRAFFIC_CBR: 142 { 143 u_int slots = cbr2slots(sc, vcc); 144 145 if (slots > sc->tst_free + sc->tst_reserve) 146 return (EINVAL); 147 break; 148 } 149 150 case ATMIO_TRAFFIC_VBR: 151 if (vcc->vcc.tparam.scr > sc->bwrem) 152 return (EINVAL); 153 if (vcc->vcc.tparam.pcr > IFP2IFATM(sc->ifp)->mib.pcr) 154 return (EINVAL); 155 if (vcc->vcc.tparam.scr > vcc->vcc.tparam.pcr || 156 vcc->vcc.tparam.mbs == 0) 157 return (EINVAL); 158 break; 159 160 case ATMIO_TRAFFIC_ABR: 161 if (vcc->vcc.tparam.tbe == 0 || 162 vcc->vcc.tparam.nrm == 0) 163 /* needed to compute CRM */ 164 return (EINVAL); 165 if (vcc->vcc.tparam.pcr > IFP2IFATM(sc->ifp)->mib.pcr || 166 vcc->vcc.tparam.icr > vcc->vcc.tparam.pcr || 167 vcc->vcc.tparam.mcr > vcc->vcc.tparam.icr) 168 return (EINVAL); 169 if (vcc->vcc.tparam.mcr > sc->bwrem || 170 vcc->vcc.tparam.icr > sc->bwrem) 171 return (EINVAL); 172 break; 173 } 174 175 return (0); 176} 177 178#define NEXT_TAG(T) do { \ 179 (T) = ((T) + 1) % IDT_TSQE_TAG_SPACE; \ 180 } while (0) 181 182/* 183 * open it 184 */ 185void 186patm_tx_vcc_open(struct patm_softc *sc, struct patm_vcc *vcc) 187{ 188 struct patm_scd *scd; 189 190 if (vcc->vcc.traffic == ATMIO_TRAFFIC_UBR) { 191 /* we use UBR0 */ 192 vcc->scd = sc->scd0; 193 vcc->vflags |= PATM_VCC_TX_OPEN; 194 return; 195 } 196 197 /* get an SCD */ 198 scd = patm_scd_alloc(sc); 199 if (scd == NULL) { 200 /* should not happen */ 201 patm_printf(sc, "out of SCDs\n"); 202 return; 203 } 204 vcc->scd = scd; 205 patm_scd_setup(sc, scd); 206 patm_tct_setup(sc, scd, vcc); 207 208 if (vcc->vcc.traffic != ATMIO_TRAFFIC_CBR) 209 patm_tct_start(sc, vcc); 210 211 vcc->vflags |= PATM_VCC_TX_OPEN; 212} 213 214/* 215 * close the given vcc for transmission 216 */ 217void 218patm_tx_vcc_close(struct patm_softc *sc, struct patm_vcc *vcc) 219{ 220 struct patm_scd *scd; 221 struct mbuf *m; 222 223 vcc->vflags |= PATM_VCC_TX_CLOSING; 224 225 if (vcc->vcc.traffic == ATMIO_TRAFFIC_UBR) { 226 /* let the queue PDUs go out */ 227 vcc->scd = NULL; 228 vcc->vflags &= ~(PATM_VCC_TX_OPEN | PATM_VCC_TX_CLOSING); 229 return; 230 } 231 scd = vcc->scd; 232 233 /* empty the waitq */ 234 for (;;) { 235 _IF_DEQUEUE(&scd->q, m); 236 if (m == NULL) 237 break; 238 m_freem(m); 239 } 240 241 if (scd->num_on_card == 0) { 242 /* we are idle */ 243 vcc->vflags &= ~PATM_VCC_TX_OPEN; 244 245 if (vcc->vcc.traffic == ATMIO_TRAFFIC_CBR) 246 patm_tst_free(sc, vcc); 247 248 patm_sram_write4(sc, scd->sram + 0, 0, 0, 0, 0); 249 patm_sram_write4(sc, scd->sram + 4, 0, 0, 0, 0); 250 patm_scd_free(sc, scd); 251 252 vcc->scd = NULL; 253 vcc->vflags &= ~PATM_VCC_TX_CLOSING; 254 255 return; 256 } 257 258 /* speed up transmission */ 259 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_UIER(vcc->cid, 0xff)); 260 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_ULACR(vcc->cid, 0xff)); 261 262 /* wait for the interrupt to drop the number to 0 */ 263 patm_debug(sc, VCC, "%u buffers still on card", scd->num_on_card); 264} 265 266/* transmission side finally closed */ 267void 268patm_tx_vcc_closed(struct patm_softc *sc, struct patm_vcc *vcc) 269{ 270 271 patm_debug(sc, VCC, "%u.%u TX closed", vcc->vcc.vpi, vcc->vcc.vci); 272 273 if (vcc->vcc.traffic == ATMIO_TRAFFIC_VBR) 274 sc->bwrem += vcc->vcc.tparam.scr; 275} 276 277/* 278 * Pull off packets from the interface queue and try to transmit them. 279 * If the transmission fails because of a full transmit channel, we drop 280 * packets for CBR and queue them for other channels up to limit. 281 * This limit should depend on the CDVT for VBR and ABR, but it doesn't. 282 */ 283void 284patm_start(struct ifnet *ifp) 285{ 286 struct patm_softc *sc = ifp->if_softc; 287 struct mbuf *m; 288 struct atm_pseudohdr *aph; 289 u_int vpi, vci, cid; 290 struct patm_vcc *vcc; 291 292 mtx_lock(&sc->mtx); 293 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 294 mtx_unlock(&sc->mtx); 295 return; 296 } 297 298 while (1) { 299 /* get a new mbuf */ 300 IF_DEQUEUE(&ifp->if_snd, m); 301 if (m == NULL) 302 break; 303 304 /* split of pseudo header */ 305 if (m->m_len < sizeof(*aph) && 306 (m = m_pullup(m, sizeof(*aph))) == NULL) { 307 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 308 continue; 309 } 310 311 aph = mtod(m, struct atm_pseudohdr *); 312 vci = ATM_PH_VCI(aph); 313 vpi = ATM_PH_VPI(aph); 314 m_adj(m, sizeof(*aph)); 315 316 /* reject empty packets */ 317 if (m->m_pkthdr.len == 0) { 318 m_freem(m); 319 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 320 continue; 321 } 322 323 /* check whether this is a legal vcc */ 324 if (!LEGAL_VPI(sc, vpi) || !LEGAL_VCI(sc, vci) || vci == 0) { 325 m_freem(m); 326 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 327 continue; 328 } 329 cid = PATM_CID(sc, vpi, vci); 330 vcc = sc->vccs[cid]; 331 if (vcc == NULL) { 332 m_freem(m); 333 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 334 continue; 335 } 336 337 /* must be multiple of 48 if not AAL5 */ 338 if (vcc->vcc.aal == ATMIO_AAL_0 || 339 vcc->vcc.aal == ATMIO_AAL_34) { 340 /* XXX AAL3/4 format? */ 341 if (m->m_pkthdr.len % 48 != 0 && 342 (m = patm_tx_pad(sc, m)) == NULL) { 343 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 344 continue; 345 } 346 } else if (vcc->vcc.aal == ATMIO_AAL_RAW) { 347 switch (vcc->vflags & PATM_RAW_FORMAT) { 348 349 default: 350 case PATM_RAW_CELL: 351 if (m->m_pkthdr.len != 53) { 352 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 353 m_freem(m); 354 continue; 355 } 356 break; 357 358 case PATM_RAW_NOHEC: 359 if (m->m_pkthdr.len != 52) { 360 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 361 m_freem(m); 362 continue; 363 } 364 break; 365 366 case PATM_RAW_CS: 367 if (m->m_pkthdr.len != 64) { 368 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 369 m_freem(m); 370 continue; 371 } 372 break; 373 } 374 } 375 376 /* save data */ 377 m->m_pkthdr.PH_loc.ptr = vcc; 378 379 /* try to put it on the channels queue */ 380 if (_IF_QFULL(&vcc->scd->q)) { 381 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 382 sc->stats.tx_qfull++; 383 m_freem(m); 384 continue; 385 } 386 _IF_ENQUEUE(&vcc->scd->q, m); 387 388#ifdef ENABLE_BPF 389 if (!(vcc->vcc.flags & ATMIO_FLAG_NG) && 390 (vcc->vcc.aal == ATMIO_AAL_5) && 391 (vcc->vcc.flags & ATM_PH_LLCSNAP)) 392 BPF_MTAP(ifp, m); 393#endif 394 395 /* kick the channel to life */ 396 patm_launch(sc, vcc->scd); 397 398 } 399 mtx_unlock(&sc->mtx); 400} 401 402/* 403 * Pad non-AAL5 packet to a multiple of 48-byte. 404 * We assume AAL0 only. We have still to decide on the format of AAL3/4. 405 */ 406static struct mbuf * 407patm_tx_pad(struct patm_softc *sc, struct mbuf *m0) 408{ 409 struct mbuf *last, *m; 410 u_int plen, pad, space; 411 412 plen = m_length(m0, &last); 413 if (plen != m0->m_pkthdr.len) { 414 patm_printf(sc, "%s: mbuf length mismatch %d %u\n", __func__, 415 m0->m_pkthdr.len, plen); 416 m0->m_pkthdr.len = plen; 417 if (plen == 0) { 418 m_freem(m0); 419 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 420 return (NULL); 421 } 422 if (plen % 48 == 0) 423 return (m0); 424 } 425 pad = 48 - plen % 48; 426 m0->m_pkthdr.len += pad; 427 if (M_WRITABLE(last)) { 428 if (M_TRAILINGSPACE(last) >= pad) { 429 bzero(last->m_data + last->m_len, pad); 430 last->m_len += pad; 431 return (m0); 432 } 433 space = M_LEADINGSPACE(last); 434 if (space + M_TRAILINGSPACE(last) >= pad) { 435 bcopy(last->m_data, last->m_data + space, last->m_len); 436 last->m_data -= space; 437 bzero(last->m_data + last->m_len, pad); 438 last->m_len += pad; 439 return (m0); 440 } 441 } 442 MGET(m, M_NOWAIT, MT_DATA); 443 if (m == NULL) { 444 m_freem(m0); 445 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 446 return (NULL); 447 } 448 bzero(mtod(m, u_char *), pad); 449 m->m_len = pad; 450 last->m_next = m; 451 452 return (m0); 453} 454 455/* 456 * Try to put as many packets from the channels queue onto the channel 457 */ 458static void 459patm_launch(struct patm_softc *sc, struct patm_scd *scd) 460{ 461 struct txarg a; 462 struct mbuf *m, *tmp; 463 u_int segs; 464 struct patm_txmap *map; 465 int error; 466 467 a.sc = sc; 468 a.scd = scd; 469 470 /* limit the number of outstanding packets to the tag space */ 471 while (scd->num_on_card < IDT_TSQE_TAG_SPACE) { 472 /* get the next packet */ 473 _IF_DEQUEUE(&scd->q, m); 474 if (m == NULL) 475 break; 476 477 a.vcc = m->m_pkthdr.PH_loc.ptr; 478 479 /* we must know the number of segments beforehand - count 480 * this may actually give a wrong number of segments for 481 * AAL_RAW where we still need to remove the cell header */ 482 segs = 0; 483 for (tmp = m; tmp != NULL; tmp = tmp->m_next) 484 if (tmp->m_len != 0) 485 segs++; 486 487 /* check whether there is space in the queue */ 488 if (segs >= scd->space) { 489 /* put back */ 490 _IF_PREPEND(&scd->q, m); 491 sc->stats.tx_out_of_tbds++; 492 break; 493 } 494 495 /* get a DMA map */ 496 if ((map = patm_txmap_get(sc)) == NULL) { 497 _IF_PREPEND(&scd->q, m); 498 sc->stats.tx_out_of_maps++; 499 break; 500 } 501 502 /* load the map */ 503 m->m_pkthdr.PH_loc.ptr = map; 504 a.mbuf = m; 505 506 /* handle AAL_RAW */ 507 if (a.vcc->vcc.aal == ATMIO_AAL_RAW) { 508 u_char hdr[4]; 509 510 m_copydata(m, 0, 4, hdr); 511 a.hdr = (hdr[0] << 24) | (hdr[1] << 16) | 512 (hdr[2] << 8) | hdr[3]; 513 514 switch (a.vcc->vflags & PATM_RAW_FORMAT) { 515 516 default: 517 case PATM_RAW_CELL: 518 m_adj(m, 5); 519 break; 520 521 case PATM_RAW_NOHEC: 522 m_adj(m, 4); 523 break; 524 525 case PATM_RAW_CS: 526 m_adj(m, 16); 527 break; 528 } 529 } else 530 a.hdr = IDT_TBD_HDR(a.vcc->vcc.vpi, a.vcc->vcc.vci, 531 0, 0); 532 533 error = bus_dmamap_load_mbuf(sc->tx_tag, map->map, m, 534 patm_load_txbuf, &a, BUS_DMA_NOWAIT); 535 if (error == EFBIG) { 536 if ((m = m_defrag(m, M_NOWAIT)) == NULL) { 537 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 538 continue; 539 } 540 error = bus_dmamap_load_mbuf(sc->tx_tag, map->map, m, 541 patm_load_txbuf, &a, BUS_DMA_NOWAIT); 542 } 543 if (error != 0) { 544 sc->stats.tx_load_err++; 545 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 546 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 547 m_freem(m); 548 continue; 549 } 550 551 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 552 } 553} 554 555/* 556 * Load the DMA segments into the scheduling channel 557 */ 558static void 559patm_load_txbuf(void *uarg, bus_dma_segment_t *segs, int nseg, 560 bus_size_t mapsize, int error) 561{ 562 struct txarg *a= uarg; 563 struct patm_scd *scd = a->scd; 564 u_int w1, w3, cnt; 565 struct idt_tbd *tbd = NULL; 566 u_int rest = mapsize; 567 568 if (error != 0) 569 return; 570 571 cnt = 0; 572 while (nseg > 0) { 573 if (segs->ds_len == 0) { 574 /* transmit buffer length must be > 0 */ 575 nseg--; 576 segs++; 577 continue; 578 } 579 /* rest after this buffer */ 580 rest -= segs->ds_len; 581 582 /* put together status word */ 583 w1 = 0; 584 if (rest < 48 /* && a->vcc->vcc.aal != ATMIO_AAL_5 */) 585 /* last cell is in this buffer */ 586 w1 |= IDT_TBD_EPDU; 587 588 if (a->vcc->vcc.aal == ATMIO_AAL_5) 589 w1 |= IDT_TBD_AAL5; 590 else if (a->vcc->vcc.aal == ATMIO_AAL_34) 591 w1 |= IDT_TBD_AAL34; 592 else 593 w1 |= IDT_TBD_AAL0; 594 595 w1 |= segs->ds_len; 596 597 /* AAL5 PDU length (unpadded) */ 598 if (a->vcc->vcc.aal == ATMIO_AAL_5) 599 w3 = mapsize; 600 else 601 w3 = 0; 602 603 if (rest == 0) 604 w1 |= IDT_TBD_TSIF | IDT_TBD_GTSI | 605 (scd->tag << IDT_TBD_TAG_SHIFT); 606 607 tbd = &scd->scq[scd->tail]; 608 609 tbd->flags = htole32(w1); 610 tbd->addr = htole32(segs->ds_addr); 611 tbd->aal5 = htole32(w3); 612 tbd->hdr = htole32(a->hdr); 613 614 patm_debug(a->sc, TX, "TBD(%u): %08x %08x %08x %08x", 615 scd->tail, w1, segs->ds_addr, w3, a->hdr); 616 617 /* got to next entry */ 618 if (++scd->tail == IDT_SCQ_SIZE) 619 scd->tail = 0; 620 cnt++; 621 nseg--; 622 segs++; 623 } 624 scd->space -= cnt; 625 scd->num_on_card++; 626 627 KASSERT(rest == 0, ("bad mbuf")); 628 KASSERT(cnt > 0, ("no segs")); 629 KASSERT(scd->space > 0, ("scq full")); 630 631 KASSERT(scd->on_card[scd->tag] == NULL, 632 ("scd on_card wedged %u%s", scd->tag, dump_scd(a->sc, scd))); 633 scd->on_card[scd->tag] = a->mbuf; 634 a->mbuf->m_pkthdr.csum_data = cnt; 635 636 NEXT_TAG(scd->tag); 637 638 patm_debug(a->sc, TX, "SCD tail %u (%lx:%lx)", scd->tail, 639 (u_long)scd->phy, (u_long)scd->phy + (scd->tail << IDT_TBD_SHIFT)); 640 patm_sram_write(a->sc, scd->sram, 641 scd->phy + (scd->tail << IDT_TBD_SHIFT)); 642 643 if (patm_sram_read(a->sc, a->vcc->cid * 8 + 3) & IDT_TCT_IDLE) { 644 /* 645 * if the connection is idle start it. We cannot rely 646 * on a flag set by patm_tx_idle() here, because sometimes 647 * the card seems to place an idle TSI into the TSQ but 648 * forgets to raise an interrupt. 649 */ 650 patm_nor_write(a->sc, IDT_NOR_TCMDQ, 651 IDT_TCMDQ_START(a->vcc->cid)); 652 } 653} 654 655/* 656 * packet transmitted 657 */ 658void 659patm_tx(struct patm_softc *sc, u_int stamp, u_int status) 660{ 661 u_int cid, tag, last; 662 struct mbuf *m; 663 struct patm_vcc *vcc; 664 struct patm_scd *scd; 665 struct patm_txmap *map; 666 667 /* get the connection */ 668 cid = PATM_CID(sc, IDT_TBD_VPI(status), IDT_TBD_VCI(status)); 669 if ((vcc = sc->vccs[cid]) == NULL) { 670 /* closed UBR connection */ 671 return; 672 } 673 scd = vcc->scd; 674 675 tag = IDT_TSQE_TAG(stamp); 676 677 last = scd->last_tag; 678 if (tag == last) { 679 patm_printf(sc, "same tag %u\n", tag); 680 return; 681 } 682 683 /* Errata 12 requests us to free all entries up to the one 684 * with the given tag. */ 685 do { 686 /* next tag to try */ 687 NEXT_TAG(last); 688 689 m = scd->on_card[last]; 690 KASSERT(m != NULL, ("%stag=%u", dump_scd(sc, scd), tag)); 691 scd->on_card[last] = NULL; 692 patm_debug(sc, TX, "ok tag=%x", last); 693 694 map = m->m_pkthdr.PH_loc.ptr; 695 scd->space += m->m_pkthdr.csum_data; 696 697 bus_dmamap_sync(sc->tx_tag, map->map, 698 BUS_DMASYNC_POSTWRITE); 699 bus_dmamap_unload(sc->tx_tag, map->map); 700 m_freem(m); 701 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 702 scd->num_on_card--; 703 704 if (vcc->vflags & PATM_VCC_TX_CLOSING) { 705 if (scd->num_on_card == 0) { 706 /* done with this VCC */ 707 if (vcc->vcc.traffic == ATMIO_TRAFFIC_CBR) 708 patm_tst_free(sc, vcc); 709 710 patm_sram_write4(sc, scd->sram + 0, 0, 0, 0, 0); 711 patm_sram_write4(sc, scd->sram + 4, 0, 0, 0, 0); 712 patm_scd_free(sc, scd); 713 714 vcc->scd = NULL; 715 vcc->vflags &= ~PATM_VCC_TX_CLOSING; 716 717 if (vcc->vcc.flags & ATMIO_FLAG_ASYNC) { 718 patm_tx_vcc_closed(sc, vcc); 719 if (!(vcc->vflags & PATM_VCC_OPEN)) 720 patm_vcc_closed(sc, vcc); 721 } else 722 cv_signal(&sc->vcc_cv); 723 return; 724 } 725 patm_debug(sc, VCC, "%u buffers still on card", 726 scd->num_on_card); 727 728 if (vcc->vcc.traffic == ATMIO_TRAFFIC_ABR) { 729 /* insist on speeding up transmission for ABR */ 730 patm_nor_write(sc, IDT_NOR_TCMDQ, 731 IDT_TCMDQ_UIER(vcc->cid, 0xff)); 732 patm_nor_write(sc, IDT_NOR_TCMDQ, 733 IDT_TCMDQ_ULACR(vcc->cid, 0xff)); 734 } 735 } 736 737 } while (last != tag); 738 scd->last_tag = tag; 739 740 if (vcc->vcc.traffic == ATMIO_TRAFFIC_ABR) { 741 u_int acri, cps; 742 743 acri = (patm_sram_read(sc, 8 * cid + 2) >> IDT_TCT_ACRI_SHIFT) 744 & 0x3fff; 745 cps = IFP2IFATM(sc->ifp)->mib.pcr * 32 / 746 ((1 << (acri >> 10)) * (acri & 0x3ff)); 747 748 if (cps != vcc->cps) { 749 patm_debug(sc, VCC, "ACRI=%04x CPS=%u", acri, cps); 750 ATMEV_SEND_ACR_CHANGED(IFP2IFATM(sc->ifp), vcc->vcc.vpi, 751 vcc->vcc.vci, cps); 752 vcc->cps = cps; 753 } 754 } 755 756 patm_launch(sc, scd); 757} 758 759/* 760 * VBR/ABR connection went idle 761 * Either restart it or set the idle flag. 762 */ 763void 764patm_tx_idle(struct patm_softc *sc, u_int cid) 765{ 766 struct patm_vcc *vcc; 767 768 patm_debug(sc, VCC, "idle %u", cid); 769 770 if ((vcc = sc->vccs[cid]) != NULL && 771 (vcc->vflags & (PATM_VCC_TX_OPEN | PATM_VCC_TX_CLOSING)) != 0 && 772 vcc->scd != NULL && (vcc->scd->num_on_card != 0 || 773 _IF_QLEN(&vcc->scd->q) != 0)) { 774 /* 775 * If there is any packet outstanding in the SCD re-activate 776 * the channel and kick it. 777 */ 778 patm_nor_write(sc, IDT_NOR_TCMDQ, 779 IDT_TCMDQ_START(vcc->cid)); 780 781 patm_launch(sc, vcc->scd); 782 } 783} 784 785/* 786 * Convert a (24bit) rate to the atm-forum form 787 * Our rate is never larger than 19 bit. 788 */ 789static u_int 790cps2atmf(u_int cps) 791{ 792 u_int e; 793 794 if (cps == 0) 795 return (0); 796 cps <<= 9; 797 e = 0; 798 while (cps > (1024 - 1)) { 799 e++; 800 cps >>= 1; 801 } 802 return ((1 << 14) | (e << 9) | (cps & 0x1ff)); 803} 804 805/* 806 * Do a binary search on the log2rate table to convert the rate 807 * to its log form. This assumes that the ATM-Forum form is monotonically 808 * increasing with the plain cell rate. 809 */ 810static u_int 811rate2log(struct patm_softc *sc, u_int rate) 812{ 813 const uint32_t *tbl; 814 u_int lower, upper, mid, done, val, afr; 815 816 afr = cps2atmf(rate); 817 818 if (sc->flags & PATM_25M) 819 tbl = patm_rtables25; 820 else 821 tbl = patm_rtables155; 822 823 lower = 0; 824 upper = 255; 825 done = 0; 826 while (!done) { 827 mid = (lower + upper) / 2; 828 val = tbl[mid] >> 17; 829 if (val == afr || upper == lower) 830 break; 831 if (afr > val) 832 lower = mid + 1; 833 else 834 upper = mid - 1; 835 } 836 if (val > afr && mid > 0) 837 mid--; 838 return (mid); 839} 840 841/* 842 * Return the table index for an increase table. The increase table 843 * must be selected not by the RIF itself, but by PCR/2^RIF. Each table 844 * represents an additive increase of a cell rate that can be computed 845 * from the first table entry (the value in this entry will not be clamped 846 * by the link rate). 847 */ 848static u_int 849get_air_table(struct patm_softc *sc, u_int rif, u_int pcr) 850{ 851 const uint32_t *tbl; 852 u_int increase, base, lair0, ret, t, cps; 853 854#define GET_ENTRY(TAB, IDX) (0xffff & ((IDX & 1) ? \ 855 (tbl[512 + (IDX / 2) + 128 * (TAB)] >> 16) : \ 856 (tbl[512 + (IDX / 2) + 128 * (TAB)]))) 857 858#define MANT_BITS 10 859#define FRAC_BITS 16 860 861#define DIFF_TO_FP(D) (((D) & ((1 << MANT_BITS) - 1)) << ((D) >> MANT_BITS)) 862#define AFR_TO_INT(A) ((1 << (((A) >> 9) & 0x1f)) * \ 863 (512 + ((A) & 0x1ff)) / 512 * ((A) >> 14)) 864 865 if (sc->flags & PATM_25M) 866 tbl = patm_rtables25; 867 else 868 tbl = patm_rtables155; 869 if (rif >= patm_rtables_ntab) 870 rif = patm_rtables_ntab - 1; 871 increase = pcr >> rif; 872 873 ret = 0; 874 for (t = 0; t < patm_rtables_ntab; t++) { 875 /* get base rate of this table */ 876 base = GET_ENTRY(t, 0); 877 /* convert this to fixed point */ 878 lair0 = DIFF_TO_FP(base) >> FRAC_BITS; 879 880 /* get the CPS from the log2rate table */ 881 cps = AFR_TO_INT(tbl[lair0] >> 17) - 10; 882 883 if (increase >= cps) 884 break; 885 886 ret = t; 887 } 888 return (ret + 4); 889} 890 891/* 892 * Setup the TCT 893 */ 894void 895patm_tct_setup(struct patm_softc *sc, struct patm_scd *scd, 896 struct patm_vcc *vcc) 897{ 898 uint32_t tct[8]; 899 u_int sram; 900 u_int mbs, token; 901 u_int tmp, crm, rdf, cdf, air, mcr; 902 903 bzero(tct, sizeof(tct)); 904 if (vcc == NULL) { 905 /* special case for UBR0 */ 906 sram = 0; 907 tct[0] = IDT_TCT_UBR | scd->sram; 908 tct[7] = IDT_TCT_UBR_FLG; 909 910 } else { 911 sram = vcc->cid * 8; 912 switch (vcc->vcc.traffic) { 913 914 case ATMIO_TRAFFIC_CBR: 915 patm_tst_alloc(sc, vcc); 916 tct[0] = IDT_TCT_CBR | scd->sram; 917 /* must account for what was really allocated */ 918 break; 919 920 case ATMIO_TRAFFIC_VBR: 921 /* compute parameters for the TCT */ 922 scd->init_er = rate2log(sc, vcc->vcc.tparam.pcr); 923 scd->lacr = rate2log(sc, vcc->vcc.tparam.scr); 924 925 /* get the 16-bit fraction of SCR/PCR 926 * both a 24 bit. Do it the simple way. */ 927 token = (uint64_t)(vcc->vcc.tparam.scr << 16) / 928 vcc->vcc.tparam.pcr; 929 930 patm_debug(sc, VCC, "VBR: init_er=%u lacr=%u " 931 "token=0x%04x\n", scd->init_er, scd->lacr, token); 932 933 tct[0] = IDT_TCT_VBR | scd->sram; 934 tct[2] = IDT_TCT_TSIF; 935 tct[3] = IDT_TCT_IDLE | IDT_TCT_HALT; 936 tct[4] = IDT_TCT_MAXIDLE; 937 tct[5] = 0x01000000; 938 if ((mbs = vcc->vcc.tparam.mbs) > 0xff) 939 mbs = 0xff; 940 tct[6] = (mbs << 16) | token; 941 sc->bwrem -= vcc->vcc.tparam.scr; 942 break; 943 944 case ATMIO_TRAFFIC_ABR: 945 scd->init_er = rate2log(sc, vcc->vcc.tparam.pcr); 946 scd->lacr = rate2log(sc, vcc->vcc.tparam.icr); 947 mcr = rate2log(sc, vcc->vcc.tparam.mcr); 948 949 /* compute CRM */ 950 tmp = vcc->vcc.tparam.tbe / vcc->vcc.tparam.nrm; 951 if (tmp * vcc->vcc.tparam.nrm < vcc->vcc.tparam.tbe) 952 tmp++; 953 for (crm = 1; tmp > (1 << crm); crm++) 954 ; 955 if (crm > 0x7) 956 crm = 7; 957 958 air = get_air_table(sc, vcc->vcc.tparam.rif, 959 vcc->vcc.tparam.pcr); 960 961 if ((rdf = vcc->vcc.tparam.rdf) >= patm_rtables_ntab) 962 rdf = patm_rtables_ntab - 1; 963 rdf += patm_rtables_ntab + 4; 964 965 if ((cdf = vcc->vcc.tparam.cdf) >= patm_rtables_ntab) 966 cdf = patm_rtables_ntab - 1; 967 cdf += patm_rtables_ntab + 4; 968 969 patm_debug(sc, VCC, "ABR: init_er=%u lacr=%u mcr=%u " 970 "crm=%u air=%u rdf=%u cdf=%u\n", scd->init_er, 971 scd->lacr, mcr, crm, air, rdf, cdf); 972 973 tct[0] = IDT_TCT_ABR | scd->sram; 974 tct[1] = crm << IDT_TCT_CRM_SHIFT; 975 tct[3] = IDT_TCT_HALT | IDT_TCT_IDLE | 976 (4 << IDT_TCT_NAGE_SHIFT); 977 tct[4] = mcr << IDT_TCT_LMCR_SHIFT; 978 tct[5] = (cdf << IDT_TCT_CDF_SHIFT) | 979 (rdf << IDT_TCT_RDF_SHIFT) | 980 (air << IDT_TCT_AIR_SHIFT); 981 982 sc->bwrem -= vcc->vcc.tparam.mcr; 983 break; 984 } 985 } 986 987 patm_sram_write4(sc, sram + 0, tct[0], tct[1], tct[2], tct[3]); 988 patm_sram_write4(sc, sram + 4, tct[4], tct[5], tct[6], tct[7]); 989 990 patm_debug(sc, VCC, "TCT[%u]: %08x %08x %08x %08x %08x %08x %08x %08x", 991 sram / 8, patm_sram_read(sc, sram + 0), 992 patm_sram_read(sc, sram + 1), patm_sram_read(sc, sram + 2), 993 patm_sram_read(sc, sram + 3), patm_sram_read(sc, sram + 4), 994 patm_sram_read(sc, sram + 5), patm_sram_read(sc, sram + 6), 995 patm_sram_read(sc, sram + 7)); 996} 997 998/* 999 * Start a channel 1000 */ 1001static void 1002patm_tct_start(struct patm_softc *sc, struct patm_vcc *vcc) 1003{ 1004 1005 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_UIER(vcc->cid, 1006 vcc->scd->init_er)); 1007 patm_nor_write(sc, IDT_NOR_TCMDQ, IDT_TCMDQ_SLACR(vcc->cid, 1008 vcc->scd->lacr)); 1009} 1010 1011static void 1012patm_tct_print(struct patm_softc *sc, u_int cid) 1013{ 1014#ifdef PATM_DEBUG 1015 u_int sram = cid * 8; 1016#endif 1017 1018 patm_debug(sc, VCC, "TCT[%u]: %08x %08x %08x %08x %08x %08x %08x %08x", 1019 sram / 8, patm_sram_read(sc, sram + 0), 1020 patm_sram_read(sc, sram + 1), patm_sram_read(sc, sram + 2), 1021 patm_sram_read(sc, sram + 3), patm_sram_read(sc, sram + 4), 1022 patm_sram_read(sc, sram + 5), patm_sram_read(sc, sram + 6), 1023 patm_sram_read(sc, sram + 7)); 1024} 1025 1026/* 1027 * Setup the SCD 1028 */ 1029void 1030patm_scd_setup(struct patm_softc *sc, struct patm_scd *scd) 1031{ 1032 patm_sram_write4(sc, scd->sram + 0, 1033 scd->phy, 0, 0xffffffff, 0); 1034 patm_sram_write4(sc, scd->sram + 4, 1035 0, 0, 0, 0); 1036 1037 patm_debug(sc, VCC, "SCD(%x): %08x %08x %08x %08x %08x %08x %08x %08x", 1038 scd->sram, 1039 patm_sram_read(sc, scd->sram + 0), 1040 patm_sram_read(sc, scd->sram + 1), 1041 patm_sram_read(sc, scd->sram + 2), 1042 patm_sram_read(sc, scd->sram + 3), 1043 patm_sram_read(sc, scd->sram + 4), 1044 patm_sram_read(sc, scd->sram + 5), 1045 patm_sram_read(sc, scd->sram + 6), 1046 patm_sram_read(sc, scd->sram + 7)); 1047} 1048 1049/* 1050 * Grow the TX map table if possible 1051 */ 1052static void 1053patm_txmaps_grow(struct patm_softc *sc) 1054{ 1055 u_int i; 1056 struct patm_txmap *map; 1057 int err; 1058 1059 if (sc->tx_nmaps >= sc->tx_maxmaps) 1060 return; 1061 1062 for (i = sc->tx_nmaps; i < sc->tx_nmaps + PATM_CFG_TXMAPS_STEP; i++) { 1063 map = uma_zalloc(sc->tx_mapzone, M_NOWAIT); 1064 err = bus_dmamap_create(sc->tx_tag, 0, &map->map); 1065 if (err) { 1066 uma_zfree(sc->tx_mapzone, map); 1067 break; 1068 } 1069 SLIST_INSERT_HEAD(&sc->tx_maps_free, map, link); 1070 } 1071 1072 sc->tx_nmaps = i; 1073} 1074 1075/* 1076 * Allocate a transmission map 1077 */ 1078static struct patm_txmap * 1079patm_txmap_get(struct patm_softc *sc) 1080{ 1081 struct patm_txmap *map; 1082 1083 if ((map = SLIST_FIRST(&sc->tx_maps_free)) == NULL) { 1084 patm_txmaps_grow(sc); 1085 if ((map = SLIST_FIRST(&sc->tx_maps_free)) == NULL) 1086 return (NULL); 1087 } 1088 SLIST_REMOVE_HEAD(&sc->tx_maps_free, link); 1089 return (map); 1090} 1091 1092/* 1093 * Look whether we are in the process of updating the TST on the chip. 1094 * If we are set the flag that we need another update. 1095 * If we are not start the update. 1096 */ 1097static __inline void 1098patm_tst_start(struct patm_softc *sc) 1099{ 1100 1101 if (!(sc->tst_state & TST_PENDING)) { 1102 sc->tst_state |= TST_PENDING; 1103 if (!(sc->tst_state & TST_WAIT)) { 1104 /* timer not running */ 1105 patm_tst_update(sc); 1106 } 1107 } 1108} 1109 1110/* 1111 * Allocate TST entries to a CBR connection 1112 */ 1113static void 1114patm_tst_alloc(struct patm_softc *sc, struct patm_vcc *vcc) 1115{ 1116 u_int slots; 1117 u_int qptr, pptr; 1118 u_int qmax, pmax; 1119 u_int pspc, last; 1120 1121 mtx_lock(&sc->tst_lock); 1122 1123 /* compute the number of slots we need, make sure to get at least 1124 * the specified PCR */ 1125 slots = cbr2slots(sc, vcc); 1126 vcc->scd->slots = slots; 1127 sc->bwrem -= slots2cr(sc, slots); 1128 1129 patm_debug(sc, TST, "tst_alloc: cbr=%u link=%u tst=%u slots=%u", 1130 vcc->vcc.tparam.pcr, IFP2IFATM(sc->ifp)->mib.pcr, sc->mmap->tst_size, slots); 1131 1132 qmax = sc->mmap->tst_size - 1; 1133 pmax = qmax << 8; 1134 1135 pspc = pmax / slots; 1136 1137 pptr = pspc >> 1; /* starting point */ 1138 qptr = pptr >> 8; 1139 1140 last = qptr; 1141 1142 while (slots > 0) { 1143 if (qptr >= qmax) 1144 qptr -= qmax; 1145 if (sc->tst_soft[qptr] != IDT_TST_VBR) { 1146 /* used - try next */ 1147 qptr++; 1148 continue; 1149 } 1150 patm_debug(sc, TST, "slot[%u] = %u.%u diff=%d", qptr, 1151 vcc->vcc.vpi, vcc->vcc.vci, (int)qptr - (int)last); 1152 last = qptr; 1153 1154 sc->tst_soft[qptr] = IDT_TST_CBR | vcc->cid | TST_BOTH; 1155 sc->tst_free--; 1156 1157 if ((pptr += pspc) >= pmax) 1158 pptr -= pmax; 1159 qptr = pptr >> 8; 1160 1161 slots--; 1162 } 1163 patm_tst_start(sc); 1164 mtx_unlock(&sc->tst_lock); 1165} 1166 1167/* 1168 * Free a CBR connection's TST entries 1169 */ 1170static void 1171patm_tst_free(struct patm_softc *sc, struct patm_vcc *vcc) 1172{ 1173 u_int i; 1174 1175 mtx_lock(&sc->tst_lock); 1176 for (i = 0; i < sc->mmap->tst_size - 1; i++) { 1177 if ((sc->tst_soft[i] & IDT_TST_MASK) == vcc->cid) { 1178 sc->tst_soft[i] = IDT_TST_VBR | TST_BOTH; 1179 sc->tst_free++; 1180 } 1181 } 1182 sc->bwrem += slots2cr(sc, vcc->scd->slots); 1183 patm_tst_start(sc); 1184 mtx_unlock(&sc->tst_lock); 1185} 1186 1187/* 1188 * Write the soft TST into the idle incore TST and start the wait timer. 1189 * We assume that we hold the tst lock. 1190 */ 1191static void 1192patm_tst_update(struct patm_softc *sc) 1193{ 1194 u_int flag; /* flag to clear from soft TST */ 1195 u_int idle; /* the idle TST */ 1196 u_int act; /* the active TST */ 1197 u_int i; 1198 1199 if (sc->tst_state & TST_ACT1) { 1200 act = 1; 1201 idle = 0; 1202 flag = TST_CH0; 1203 } else { 1204 act = 0; 1205 idle = 1; 1206 flag = TST_CH1; 1207 } 1208 /* update the idle one */ 1209 for (i = 0; i < sc->mmap->tst_size - 1; i++) 1210 if (sc->tst_soft[i] & flag) { 1211 patm_sram_write(sc, sc->tst_base[idle] + i, 1212 sc->tst_soft[i] & ~TST_BOTH); 1213 sc->tst_soft[i] &= ~flag; 1214 } 1215 /* the used one jump to the idle one */ 1216 patm_sram_write(sc, sc->tst_jump[act], 1217 IDT_TST_BR | (sc->tst_base[idle] << 2)); 1218 1219 /* wait for the chip to jump */ 1220 sc->tst_state &= ~TST_PENDING; 1221 sc->tst_state |= TST_WAIT; 1222 1223 callout_reset(&sc->tst_callout, 1, patm_tst_timer, sc); 1224} 1225 1226/* 1227 * Timer for TST updates 1228 */ 1229static void 1230patm_tst_timer(void *p) 1231{ 1232 struct patm_softc *sc = p; 1233 u_int act; /* active TST */ 1234 u_int now; /* current place in TST */ 1235 1236 mtx_lock(&sc->tst_lock); 1237 1238 if (sc->tst_state & TST_WAIT) { 1239 /* ignore the PENDING state while we are waiting for 1240 * the chip to switch tables. Once the switch is done, 1241 * we will again lock at PENDING */ 1242 act = (sc->tst_state & TST_ACT1) ? 1 : 0; 1243 now = patm_nor_read(sc, IDT_NOR_NOW) >> 2; 1244 if (now >= sc->tst_base[act] && now <= sc->tst_jump[act]) { 1245 /* not yet */ 1246 callout_reset(&sc->tst_callout, 1, patm_tst_timer, sc); 1247 goto done; 1248 } 1249 sc->tst_state &= ~TST_WAIT; 1250 /* change back jump */ 1251 patm_sram_write(sc, sc->tst_jump[act], 1252 IDT_TST_BR | (sc->tst_base[act] << 2)); 1253 1254 /* switch */ 1255 sc->tst_state ^= TST_ACT1; 1256 } 1257 1258 if (sc->tst_state & TST_PENDING) 1259 /* we got another update request while the timer was running. */ 1260 patm_tst_update(sc); 1261 1262 done: 1263 mtx_unlock(&sc->tst_lock); 1264} 1265 1266static const char * 1267dump_scd(struct patm_softc *sc, struct patm_scd *scd) 1268{ 1269 u_int i; 1270 1271 for (i = 0; i < IDT_TSQE_TAG_SPACE; i++) 1272 printf("on_card[%u] = %p\n", i, scd->on_card[i]); 1273 printf("space=%u tag=%u num_on_card=%u last_tag=%u\n", 1274 scd->space, scd->tag, scd->num_on_card, scd->last_tag); 1275 1276 return (""); 1277} 1278