1139749Simp/*- 2116491Sharti * Copyright (c) 2001-2003 3116491Sharti * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4116491Sharti * All rights reserved. 5119418Sobrien * Author: Hartmut Brandt <harti@freebsd.org> 6116491Sharti * 7116491Sharti * Redistribution and use in source and binary forms, with or without 8116491Sharti * modification, are permitted provided that the following conditions 9116491Sharti * are met: 10116491Sharti * 1. Redistributions of source code must retain the above copyright 11116491Sharti * notice, this list of conditions and the following disclaimer. 12116491Sharti * 2. Redistributions in binary form must reproduce the above copyright 13116491Sharti * notice, this list of conditions and the following disclaimer in the 14116491Sharti * documentation and/or other materials provided with the distribution. 15116491Sharti * 16116491Sharti * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17116491Sharti * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18116491Sharti * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19116491Sharti * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20116491Sharti * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21116491Sharti * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22116491Sharti * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23116491Sharti * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24116491Sharti * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25116491Sharti * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26116491Sharti * SUCH DAMAGE. 27119418Sobrien */ 28119418Sobrien 29119418Sobrien#include <sys/cdefs.h> 30119418Sobrien__FBSDID("$FreeBSD: releng/10.3/sys/dev/hatm/if_hatm_intr.c 254842 2013-08-25 10:57:09Z andre $"); 31119418Sobrien 32119418Sobrien/* 33116491Sharti * ForeHE driver. 34116491Sharti * 35116491Sharti * Interrupt handler. 36116491Sharti */ 37116491Sharti 38116491Sharti#include "opt_inet.h" 39116491Sharti#include "opt_natm.h" 40116491Sharti 41116491Sharti#include <sys/types.h> 42116491Sharti#include <sys/param.h> 43116491Sharti#include <sys/systm.h> 44116491Sharti#include <sys/malloc.h> 45116491Sharti#include <sys/kernel.h> 46116491Sharti#include <sys/bus.h> 47116491Sharti#include <sys/errno.h> 48116491Sharti#include <sys/conf.h> 49116491Sharti#include <sys/module.h> 50116491Sharti#include <sys/queue.h> 51116491Sharti#include <sys/syslog.h> 52116491Sharti#include <sys/condvar.h> 53116491Sharti#include <sys/sysctl.h> 54116491Sharti#include <vm/uma.h> 55116491Sharti 56116491Sharti#include <sys/sockio.h> 57116491Sharti#include <sys/mbuf.h> 58116491Sharti#include <sys/socket.h> 59116491Sharti 60116491Sharti#include <net/if.h> 61116491Sharti#include <net/if_media.h> 62116491Sharti#include <net/if_atm.h> 63116491Sharti#include <net/route.h> 64116491Sharti#include <netinet/in.h> 65116491Sharti#include <netinet/if_atm.h> 66116491Sharti 67116491Sharti#include <machine/bus.h> 68116491Sharti#include <machine/resource.h> 69116491Sharti#include <sys/bus.h> 70116491Sharti#include <sys/rman.h> 71119280Simp#include <dev/pci/pcireg.h> 72119280Simp#include <dev/pci/pcivar.h> 73116491Sharti 74116491Sharti#include <dev/utopia/utopia.h> 75116491Sharti#include <dev/hatm/if_hatmconf.h> 76116491Sharti#include <dev/hatm/if_hatmreg.h> 77116491Sharti#include <dev/hatm/if_hatmvar.h> 78116491Sharti 79116491ShartiCTASSERT(sizeof(struct mbuf_page) == MBUF_ALLOC_SIZE); 80116491ShartiCTASSERT(sizeof(struct mbuf0_chunk) == MBUF0_CHUNK); 81116491ShartiCTASSERT(sizeof(struct mbuf1_chunk) == MBUF1_CHUNK); 82116491ShartiCTASSERT(sizeof(((struct mbuf0_chunk *)NULL)->storage) >= MBUF0_SIZE); 83116491ShartiCTASSERT(sizeof(((struct mbuf1_chunk *)NULL)->storage) >= MBUF1_SIZE); 84116491ShartiCTASSERT(sizeof(struct tpd) <= HE_TPD_SIZE); 85116491Sharti 86121729ShartiCTASSERT(MBUF0_PER_PAGE <= 256); 87121729ShartiCTASSERT(MBUF1_PER_PAGE <= 256); 88121729Sharti 89121675Shartistatic void hatm_mbuf_page_alloc(struct hatm_softc *sc, u_int group); 90121675Sharti 91116491Sharti/* 92121675Sharti * Free an external mbuf to a list. We use atomic functions so that 93121675Sharti * we don't need a mutex for the list. 94121744Sharti * 95121744Sharti * Note that in general this algorithm is not safe when multiple readers 96121744Sharti * and writers are present. To cite from a mail from David Schultz 97121744Sharti * <das@freebsd.org>: 98121744Sharti * 99121744Sharti * It looks like this is subject to the ABA problem. For instance, 100121744Sharti * suppose X, Y, and Z are the top things on the freelist and a 101121744Sharti * thread attempts to make an allocation. You set buf to X and load 102121744Sharti * buf->link (Y) into a register. Then the thread get preempted, and 103121744Sharti * another thread allocates both X and Y, then frees X. When the 104121744Sharti * original thread gets the CPU again, X is still on top of the 105121744Sharti * freelist, so the atomic operation succeeds. However, the atomic 106121744Sharti * op places Y on top of the freelist, even though Y is no longer 107121744Sharti * free. 108121744Sharti * 109121744Sharti * We are, however sure that we have only one thread that ever allocates 110121744Sharti * buffers because the only place we're call from is the interrupt handler. 111121744Sharti * Under these circumstances the code looks safe. 112121675Sharti */ 113170411Smjacobvoid 114121675Shartihatm_ext_free(struct mbufx_free **list, struct mbufx_free *buf) 115121675Sharti{ 116121675Sharti for (;;) { 117121675Sharti buf->link = *list; 118148067Sjhb if (atomic_cmpset_ptr((uintptr_t *)list, (uintptr_t)buf->link, 119148067Sjhb (uintptr_t)buf)) 120121675Sharti break; 121121675Sharti } 122121675Sharti} 123121675Sharti 124121675Shartistatic __inline struct mbufx_free * 125121675Shartihatm_ext_alloc(struct hatm_softc *sc, u_int g) 126121675Sharti{ 127121675Sharti struct mbufx_free *buf; 128121675Sharti 129121675Sharti for (;;) { 130121675Sharti if ((buf = sc->mbuf_list[g]) == NULL) 131121675Sharti break; 132148067Sjhb if (atomic_cmpset_ptr((uintptr_t *)&sc->mbuf_list[g], 133148067Sjhb (uintptr_t)buf, (uintptr_t)buf->link)) 134121675Sharti break; 135121675Sharti } 136121675Sharti if (buf == NULL) { 137121675Sharti hatm_mbuf_page_alloc(sc, g); 138121675Sharti for (;;) { 139121675Sharti if ((buf = sc->mbuf_list[g]) == NULL) 140121675Sharti break; 141148067Sjhb if (atomic_cmpset_ptr((uintptr_t *)&sc->mbuf_list[g], 142148067Sjhb (uintptr_t)buf, (uintptr_t)buf->link)) 143121675Sharti break; 144121675Sharti } 145121675Sharti } 146121675Sharti return (buf); 147121675Sharti} 148121675Sharti 149121675Sharti/* 150116491Sharti * Either the queue treshold was crossed or a TPD with the INTR bit set 151116491Sharti * was transmitted. 152116491Sharti */ 153116491Shartistatic void 154116491Shartihe_intr_tbrq(struct hatm_softc *sc, struct hetbrq *q, u_int group) 155116491Sharti{ 156116491Sharti uint32_t *tailp = &sc->hsp->group[group].tbrq_tail; 157116491Sharti u_int no; 158116491Sharti 159116491Sharti while (q->head != (*tailp >> 2)) { 160116491Sharti no = (q->tbrq[q->head].addr & HE_REGM_TBRQ_ADDR) >> 161116491Sharti HE_REGS_TPD_ADDR; 162116491Sharti hatm_tx_complete(sc, TPD_ADDR(sc, no), 163116491Sharti (q->tbrq[q->head].addr & HE_REGM_TBRQ_FLAGS)); 164116491Sharti 165116491Sharti if (++q->head == q->size) 166116491Sharti q->head = 0; 167116491Sharti } 168116491Sharti WRITE4(sc, HE_REGO_TBRQ_H(group), q->head << 2); 169116491Sharti} 170116491Sharti 171116491Sharti/* 172116491Sharti * DMA loader function for external mbuf page. 173116491Sharti */ 174116491Shartistatic void 175116491Shartihatm_extbuf_helper(void *arg, bus_dma_segment_t *segs, int nsegs, 176116491Sharti int error) 177116491Sharti{ 178116491Sharti if (error) { 179116491Sharti printf("%s: mapping error %d\n", __func__, error); 180116491Sharti return; 181116491Sharti } 182116491Sharti KASSERT(nsegs == 1, 183116491Sharti ("too many segments for DMA: %d", nsegs)); 184116491Sharti KASSERT(segs[0].ds_addr <= 0xffffffffLU, 185116491Sharti ("phys addr too large %lx", (u_long)segs[0].ds_addr)); 186116491Sharti 187116491Sharti *(uint32_t *)arg = segs[0].ds_addr; 188116491Sharti} 189116491Sharti 190116491Sharti/* 191116491Sharti * Allocate a page of external mbuf storage for the small pools. 192116491Sharti * Create a DMA map and load it. Put all the chunks onto the right 193116491Sharti * free list. 194116491Sharti */ 195116491Shartistatic void 196116491Shartihatm_mbuf_page_alloc(struct hatm_softc *sc, u_int group) 197116491Sharti{ 198116491Sharti struct mbuf_page *pg; 199116491Sharti int err; 200116491Sharti u_int i; 201116491Sharti 202121686Sharti if (sc->mbuf_npages == sc->mbuf_max_pages) 203116491Sharti return; 204116491Sharti if ((pg = malloc(MBUF_ALLOC_SIZE, M_DEVBUF, M_NOWAIT)) == NULL) 205116491Sharti return; 206116491Sharti 207116491Sharti err = bus_dmamap_create(sc->mbuf_tag, 0, &pg->hdr.map); 208116491Sharti if (err != 0) { 209147256Sbrooks if_printf(sc->ifp, "%s -- bus_dmamap_create: %d\n", 210116491Sharti __func__, err); 211116491Sharti free(pg, M_DEVBUF); 212116491Sharti return; 213116491Sharti } 214116491Sharti err = bus_dmamap_load(sc->mbuf_tag, pg->hdr.map, pg, MBUF_ALLOC_SIZE, 215117382Sharti hatm_extbuf_helper, &pg->hdr.phys, BUS_DMA_NOWAIT); 216116491Sharti if (err != 0) { 217147256Sbrooks if_printf(sc->ifp, "%s -- mbuf mapping failed %d\n", 218116491Sharti __func__, err); 219116491Sharti bus_dmamap_destroy(sc->mbuf_tag, pg->hdr.map); 220116491Sharti free(pg, M_DEVBUF); 221116491Sharti return; 222116491Sharti } 223116491Sharti 224116491Sharti sc->mbuf_pages[sc->mbuf_npages] = pg; 225116491Sharti 226116491Sharti if (group == 0) { 227116491Sharti struct mbuf0_chunk *c; 228116491Sharti 229121729Sharti pg->hdr.pool = 0; 230116491Sharti pg->hdr.nchunks = MBUF0_PER_PAGE; 231116491Sharti pg->hdr.chunksize = MBUF0_CHUNK; 232116491Sharti pg->hdr.hdroff = sizeof(c->storage); 233116491Sharti c = (struct mbuf0_chunk *)pg; 234116491Sharti for (i = 0; i < MBUF0_PER_PAGE; i++, c++) { 235116491Sharti c->hdr.pageno = sc->mbuf_npages; 236116491Sharti c->hdr.chunkno = i; 237122111Sharti c->hdr.flags = 0; 238121675Sharti hatm_ext_free(&sc->mbuf_list[0], 239121675Sharti (struct mbufx_free *)c); 240116491Sharti } 241116491Sharti } else { 242116491Sharti struct mbuf1_chunk *c; 243116491Sharti 244121729Sharti pg->hdr.pool = 1; 245116491Sharti pg->hdr.nchunks = MBUF1_PER_PAGE; 246116491Sharti pg->hdr.chunksize = MBUF1_CHUNK; 247116491Sharti pg->hdr.hdroff = sizeof(c->storage); 248116491Sharti c = (struct mbuf1_chunk *)pg; 249116491Sharti for (i = 0; i < MBUF1_PER_PAGE; i++, c++) { 250116491Sharti c->hdr.pageno = sc->mbuf_npages; 251116491Sharti c->hdr.chunkno = i; 252122111Sharti c->hdr.flags = 0; 253121675Sharti hatm_ext_free(&sc->mbuf_list[1], 254121675Sharti (struct mbufx_free *)c); 255116491Sharti } 256116491Sharti } 257116491Sharti sc->mbuf_npages++; 258116491Sharti} 259116491Sharti 260116491Sharti/* 261116491Sharti * Free an mbuf and put it onto the free list. 262116491Sharti */ 263254842Sandrestatic int 264254799Sandrehatm_mbuf0_free(struct mbuf *m, void *buf, void *args) 265116491Sharti{ 266116491Sharti struct hatm_softc *sc = args; 267116491Sharti struct mbuf0_chunk *c = buf; 268116491Sharti 269121729Sharti KASSERT((c->hdr.flags & (MBUF_USED | MBUF_CARD)) == MBUF_USED, 270121729Sharti ("freeing unused mbuf %x", c->hdr.flags)); 271121729Sharti c->hdr.flags &= ~MBUF_USED; 272121675Sharti hatm_ext_free(&sc->mbuf_list[0], (struct mbufx_free *)c); 273254842Sandre return (EXT_FREE_OK); 274116491Sharti} 275254842Sandrestatic int 276254799Sandrehatm_mbuf1_free(struct mbuf *m, void *buf, void *args) 277116491Sharti{ 278116491Sharti struct hatm_softc *sc = args; 279116491Sharti struct mbuf1_chunk *c = buf; 280116491Sharti 281121729Sharti KASSERT((c->hdr.flags & (MBUF_USED | MBUF_CARD)) == MBUF_USED, 282121729Sharti ("freeing unused mbuf %x", c->hdr.flags)); 283121729Sharti c->hdr.flags &= ~MBUF_USED; 284121675Sharti hatm_ext_free(&sc->mbuf_list[1], (struct mbufx_free *)c); 285254842Sandre return (EXT_FREE_OK); 286116491Sharti} 287116491Sharti 288116491Shartistatic void 289116491Shartihatm_mbuf_helper(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 290116491Sharti{ 291116491Sharti uint32_t *ptr = (uint32_t *)arg; 292116491Sharti 293116491Sharti if (nsegs == 0) { 294116491Sharti printf("%s: error=%d\n", __func__, error); 295116491Sharti return; 296116491Sharti } 297116491Sharti KASSERT(nsegs == 1, ("too many segments for mbuf: %d", nsegs)); 298116491Sharti KASSERT(segs[0].ds_addr <= 0xffffffffLU, 299116491Sharti ("phys addr too large %lx", (u_long)segs[0].ds_addr)); 300116491Sharti 301116491Sharti *ptr = segs[0].ds_addr; 302116491Sharti} 303116491Sharti 304116491Sharti/* 305116491Sharti * Receive buffer pool interrupt. This means the number of entries in the 306116491Sharti * queue has dropped below the threshold. Try to supply new buffers. 307116491Sharti */ 308116491Shartistatic void 309116491Shartihe_intr_rbp(struct hatm_softc *sc, struct herbp *rbp, u_int large, 310116491Sharti u_int group) 311116491Sharti{ 312121617Sharti u_int ntail; 313116491Sharti struct mbuf *m; 314116491Sharti int error; 315121680Sharti struct mbufx_free *cf; 316121680Sharti struct mbuf_page *pg; 317121680Sharti struct mbuf0_chunk *buf0; 318121680Sharti struct mbuf1_chunk *buf1; 319116491Sharti 320116491Sharti DBG(sc, INTR, ("%s buffer supply threshold crossed for group %u", 321116491Sharti large ? "large" : "small", group)); 322116491Sharti 323116491Sharti rbp->head = (READ4(sc, HE_REGO_RBP_S(large, group)) >> HE_REGS_RBP_HEAD) 324116491Sharti & (rbp->size - 1); 325116491Sharti 326116491Sharti for (;;) { 327116491Sharti if ((ntail = rbp->tail + 1) == rbp->size) 328116491Sharti ntail = 0; 329116491Sharti if (ntail == rbp->head) 330116491Sharti break; 331121680Sharti m = NULL; 332116491Sharti 333116491Sharti if (large) { 334121676Sharti /* allocate the MBUF */ 335243857Sglebius if ((m = m_getcl(M_NOWAIT, MT_DATA, 336116491Sharti M_PKTHDR)) == NULL) { 337147256Sbrooks if_printf(sc->ifp, 338116491Sharti "no mbuf clusters\n"); 339116491Sharti break; 340116491Sharti } 341116491Sharti m->m_data += MBUFL_OFFSET; 342116491Sharti 343116491Sharti if (sc->lbufs[sc->lbufs_next] != NULL) 344116491Sharti panic("hatm: lbufs full %u", sc->lbufs_next); 345116491Sharti sc->lbufs[sc->lbufs_next] = m; 346116491Sharti 347116491Sharti if ((error = bus_dmamap_load(sc->mbuf_tag, 348116491Sharti sc->rmaps[sc->lbufs_next], 349116491Sharti m->m_data, rbp->bsize, hatm_mbuf_helper, 350123810Salfred &rbp->rbp[rbp->tail].phys, BUS_DMA_NOWAIT)) != 0) 351116491Sharti panic("hatm: mbuf mapping failed %d", error); 352116491Sharti 353116491Sharti bus_dmamap_sync(sc->mbuf_tag, 354116491Sharti sc->rmaps[sc->lbufs_next], 355116491Sharti BUS_DMASYNC_PREREAD); 356116491Sharti 357121680Sharti rbp->rbp[rbp->tail].handle = 358121680Sharti MBUF_MAKE_LHANDLE(sc->lbufs_next); 359116491Sharti 360116491Sharti if (++sc->lbufs_next == sc->lbufs_size) 361116491Sharti sc->lbufs_next = 0; 362116491Sharti 363121680Sharti } else if (group == 0) { 364121680Sharti /* 365121680Sharti * Allocate small buffer in group 0 366121680Sharti */ 367121680Sharti if ((cf = hatm_ext_alloc(sc, 0)) == NULL) 368116491Sharti break; 369121680Sharti buf0 = (struct mbuf0_chunk *)cf; 370121680Sharti pg = sc->mbuf_pages[buf0->hdr.pageno]; 371121729Sharti buf0->hdr.flags |= MBUF_CARD; 372121680Sharti rbp->rbp[rbp->tail].phys = pg->hdr.phys + 373121680Sharti buf0->hdr.chunkno * MBUF0_CHUNK + MBUF0_OFFSET; 374121680Sharti rbp->rbp[rbp->tail].handle = 375121680Sharti MBUF_MAKE_HANDLE(buf0->hdr.pageno, 376121680Sharti buf0->hdr.chunkno); 377121680Sharti 378121680Sharti bus_dmamap_sync(sc->mbuf_tag, pg->hdr.map, 379121680Sharti BUS_DMASYNC_PREREAD); 380121680Sharti 381121680Sharti } else if (group == 1) { 382121680Sharti /* 383121680Sharti * Allocate small buffer in group 1 384121680Sharti */ 385121680Sharti if ((cf = hatm_ext_alloc(sc, 1)) == NULL) 386121680Sharti break; 387121680Sharti buf1 = (struct mbuf1_chunk *)cf; 388121680Sharti pg = sc->mbuf_pages[buf1->hdr.pageno]; 389121729Sharti buf1->hdr.flags |= MBUF_CARD; 390121680Sharti rbp->rbp[rbp->tail].phys = pg->hdr.phys + 391121680Sharti buf1->hdr.chunkno * MBUF1_CHUNK + MBUF1_OFFSET; 392121680Sharti rbp->rbp[rbp->tail].handle = 393121680Sharti MBUF_MAKE_HANDLE(buf1->hdr.pageno, 394121680Sharti buf1->hdr.chunkno); 395121680Sharti 396121680Sharti bus_dmamap_sync(sc->mbuf_tag, pg->hdr.map, 397121680Sharti BUS_DMASYNC_PREREAD); 398121680Sharti 399121680Sharti } else 400121680Sharti /* ups */ 401121680Sharti break; 402121680Sharti 403116491Sharti DBG(sc, DMA, ("MBUF loaded: handle=%x m=%p phys=%x", 404116491Sharti rbp->rbp[rbp->tail].handle, m, rbp->rbp[rbp->tail].phys)); 405116491Sharti 406116491Sharti rbp->tail = ntail; 407116491Sharti } 408121617Sharti WRITE4(sc, HE_REGO_RBP_T(large, group), 409121617Sharti (rbp->tail << HE_REGS_RBP_TAIL)); 410116491Sharti} 411116491Sharti 412116491Sharti/* 413116491Sharti * Extract the buffer and hand it to the receive routine 414116491Sharti */ 415116491Shartistatic struct mbuf * 416116491Shartihatm_rx_buffer(struct hatm_softc *sc, u_int group, u_int handle) 417116491Sharti{ 418116491Sharti u_int pageno; 419116491Sharti u_int chunkno; 420116491Sharti struct mbuf *m; 421116491Sharti 422116491Sharti if (handle & MBUF_LARGE_FLAG) { 423116491Sharti /* large buffer - sync and unload */ 424121680Sharti MBUF_PARSE_LHANDLE(handle, handle); 425116491Sharti DBG(sc, RX, ("RX large handle=%x", handle)); 426116491Sharti 427116491Sharti bus_dmamap_sync(sc->mbuf_tag, sc->rmaps[handle], 428116491Sharti BUS_DMASYNC_POSTREAD); 429116491Sharti bus_dmamap_unload(sc->mbuf_tag, sc->rmaps[handle]); 430116491Sharti 431116491Sharti m = sc->lbufs[handle]; 432116491Sharti sc->lbufs[handle] = NULL; 433116491Sharti 434116491Sharti return (m); 435116491Sharti } 436116491Sharti 437116491Sharti MBUF_PARSE_HANDLE(handle, pageno, chunkno); 438116491Sharti 439116491Sharti DBG(sc, RX, ("RX group=%u handle=%x page=%u chunk=%u", group, handle, 440116491Sharti pageno, chunkno)); 441116491Sharti 442243857Sglebius MGETHDR(m, M_NOWAIT, MT_DATA); 443121676Sharti 444116491Sharti if (group == 0) { 445116491Sharti struct mbuf0_chunk *c0; 446116491Sharti 447116491Sharti c0 = (struct mbuf0_chunk *)sc->mbuf_pages[pageno] + chunkno; 448116491Sharti KASSERT(c0->hdr.pageno == pageno, ("pageno = %u/%u", 449116491Sharti c0->hdr.pageno, pageno)); 450116491Sharti KASSERT(c0->hdr.chunkno == chunkno, ("chunkno = %u/%u", 451116491Sharti c0->hdr.chunkno, chunkno)); 452121729Sharti KASSERT(c0->hdr.flags & MBUF_CARD, ("mbuf not on card %u/%u", 453121729Sharti pageno, chunkno)); 454121729Sharti KASSERT(!(c0->hdr.flags & MBUF_USED), ("used mbuf %u/%u", 455121729Sharti pageno, chunkno)); 456116491Sharti 457121729Sharti c0->hdr.flags |= MBUF_USED; 458121729Sharti c0->hdr.flags &= ~MBUF_CARD; 459121729Sharti 460121676Sharti if (m != NULL) { 461121677Sharti m->m_ext.ref_cnt = &c0->hdr.ref_cnt; 462150347Sandre MEXTADD(m, (void *)c0, MBUF0_SIZE, 463175872Sphk hatm_mbuf0_free, c0, sc, M_PKTHDR, EXT_EXTREF); 464121676Sharti m->m_data += MBUF0_OFFSET; 465121676Sharti } else 466254842Sandre (void)hatm_mbuf0_free(NULL, c0, sc); 467116491Sharti 468116491Sharti } else { 469116491Sharti struct mbuf1_chunk *c1; 470116491Sharti 471116491Sharti c1 = (struct mbuf1_chunk *)sc->mbuf_pages[pageno] + chunkno; 472116491Sharti KASSERT(c1->hdr.pageno == pageno, ("pageno = %u/%u", 473116491Sharti c1->hdr.pageno, pageno)); 474116491Sharti KASSERT(c1->hdr.chunkno == chunkno, ("chunkno = %u/%u", 475116491Sharti c1->hdr.chunkno, chunkno)); 476121729Sharti KASSERT(c1->hdr.flags & MBUF_CARD, ("mbuf not on card %u/%u", 477121729Sharti pageno, chunkno)); 478121729Sharti KASSERT(!(c1->hdr.flags & MBUF_USED), ("used mbuf %u/%u", 479121729Sharti pageno, chunkno)); 480116491Sharti 481121729Sharti c1->hdr.flags |= MBUF_USED; 482121729Sharti c1->hdr.flags &= ~MBUF_CARD; 483121729Sharti 484121676Sharti if (m != NULL) { 485121677Sharti m->m_ext.ref_cnt = &c1->hdr.ref_cnt; 486150347Sandre MEXTADD(m, (void *)c1, MBUF1_SIZE, 487175872Sphk hatm_mbuf1_free, c1, sc, M_PKTHDR, EXT_EXTREF); 488121676Sharti m->m_data += MBUF1_OFFSET; 489121676Sharti } else 490254842Sandre (void)hatm_mbuf1_free(NULL, c1, sc); 491116491Sharti } 492116491Sharti 493116491Sharti return (m); 494116491Sharti} 495116491Sharti 496116491Sharti/* 497116491Sharti * Interrupt because of receive buffer returned. 498116491Sharti */ 499116491Shartistatic void 500116491Shartihe_intr_rbrq(struct hatm_softc *sc, struct herbrq *rq, u_int group) 501116491Sharti{ 502116491Sharti struct he_rbrqen *e; 503116491Sharti uint32_t flags, tail; 504116491Sharti u_int cid, len; 505116491Sharti struct mbuf *m; 506116491Sharti 507116491Sharti for (;;) { 508116491Sharti tail = sc->hsp->group[group].rbrq_tail >> 3; 509116491Sharti 510116491Sharti if (rq->head == tail) 511116491Sharti break; 512116491Sharti 513116491Sharti e = &rq->rbrq[rq->head]; 514116491Sharti 515116491Sharti flags = e->addr & HE_REGM_RBRQ_FLAGS; 516116491Sharti if (!(flags & HE_REGM_RBRQ_HBUF_ERROR)) 517121680Sharti m = hatm_rx_buffer(sc, group, e->addr); 518116491Sharti else 519116491Sharti m = NULL; 520116491Sharti 521116491Sharti cid = (e->len & HE_REGM_RBRQ_CID) >> HE_REGS_RBRQ_CID; 522116491Sharti len = 4 * (e->len & HE_REGM_RBRQ_LEN); 523116491Sharti 524116491Sharti hatm_rx(sc, cid, flags, m, len); 525116491Sharti 526116491Sharti if (++rq->head == rq->size) 527116491Sharti rq->head = 0; 528116491Sharti } 529116491Sharti WRITE4(sc, HE_REGO_RBRQ_H(group), rq->head << 3); 530116491Sharti} 531116491Sharti 532116491Shartivoid 533116491Shartihatm_intr(void *p) 534116491Sharti{ 535116491Sharti struct heirq *q = p; 536116491Sharti struct hatm_softc *sc = q->sc; 537116491Sharti u_int status; 538116491Sharti u_int tail; 539116491Sharti 540116491Sharti /* if we have a stray interrupt with a non-initialized card, 541116491Sharti * we cannot even lock before looking at the flag */ 542148887Srwatson if (!(sc->ifp->if_drv_flags & IFF_DRV_RUNNING)) 543116491Sharti return; 544116491Sharti 545116491Sharti mtx_lock(&sc->mtx); 546116491Sharti (void)READ4(sc, HE_REGO_INT_FIFO); 547116491Sharti 548116491Sharti tail = *q->tailp; 549116491Sharti if (q->head == tail) { 550116491Sharti /* workaround for tail pointer not updated bug (8.1.1) */ 551116491Sharti DBG(sc, INTR, ("hatm: intr tailq not updated bug triggered")); 552116491Sharti 553116491Sharti /* read the tail pointer from the card */ 554116491Sharti tail = READ4(sc, HE_REGO_IRQ_BASE(q->group)) & 555116491Sharti HE_REGM_IRQ_BASE_TAIL; 556116491Sharti BARRIER_R(sc); 557116491Sharti 558116491Sharti sc->istats.bug_no_irq_upd++; 559116491Sharti } 560116491Sharti 561116491Sharti /* clear the interrupt */ 562116491Sharti WRITE4(sc, HE_REGO_INT_FIFO, HE_REGM_INT_FIFO_CLRA); 563116491Sharti BARRIER_W(sc); 564116491Sharti 565116491Sharti while (q->head != tail) { 566116491Sharti status = q->irq[q->head]; 567116491Sharti q->irq[q->head] = HE_REGM_ITYPE_INVALID; 568116491Sharti if (++q->head == (q->size - 1)) 569116491Sharti q->head = 0; 570116491Sharti 571116491Sharti switch (status & HE_REGM_ITYPE) { 572116491Sharti 573116491Sharti case HE_REGM_ITYPE_TBRQ: 574116491Sharti DBG(sc, INTR, ("TBRQ treshold %u", status & HE_REGM_IGROUP)); 575116491Sharti sc->istats.itype_tbrq++; 576116491Sharti he_intr_tbrq(sc, &sc->tbrq, status & HE_REGM_IGROUP); 577116491Sharti break; 578116491Sharti 579116491Sharti case HE_REGM_ITYPE_TPD: 580116491Sharti DBG(sc, INTR, ("TPD ready %u", status & HE_REGM_IGROUP)); 581116491Sharti sc->istats.itype_tpd++; 582116491Sharti he_intr_tbrq(sc, &sc->tbrq, status & HE_REGM_IGROUP); 583116491Sharti break; 584116491Sharti 585116491Sharti case HE_REGM_ITYPE_RBPS: 586116491Sharti sc->istats.itype_rbps++; 587116491Sharti switch (status & HE_REGM_IGROUP) { 588116491Sharti 589116491Sharti case 0: 590116491Sharti he_intr_rbp(sc, &sc->rbp_s0, 0, 0); 591116491Sharti break; 592116491Sharti 593116491Sharti case 1: 594116491Sharti he_intr_rbp(sc, &sc->rbp_s1, 0, 1); 595116491Sharti break; 596116491Sharti 597116491Sharti default: 598147256Sbrooks if_printf(sc->ifp, "bad INTR RBPS%u\n", 599116491Sharti status & HE_REGM_IGROUP); 600116491Sharti break; 601116491Sharti } 602116491Sharti break; 603116491Sharti 604116491Sharti case HE_REGM_ITYPE_RBPL: 605116491Sharti sc->istats.itype_rbpl++; 606116491Sharti switch (status & HE_REGM_IGROUP) { 607116491Sharti 608116491Sharti case 0: 609116491Sharti he_intr_rbp(sc, &sc->rbp_l0, 1, 0); 610116491Sharti break; 611116491Sharti 612116491Sharti default: 613147256Sbrooks if_printf(sc->ifp, "bad INTR RBPL%u\n", 614116491Sharti status & HE_REGM_IGROUP); 615116491Sharti break; 616116491Sharti } 617116491Sharti break; 618116491Sharti 619116491Sharti case HE_REGM_ITYPE_RBRQ: 620116491Sharti DBG(sc, INTR, ("INTERRUPT RBRQ %u", status & HE_REGM_IGROUP)); 621116491Sharti sc->istats.itype_rbrq++; 622116491Sharti switch (status & HE_REGM_IGROUP) { 623116491Sharti 624116491Sharti case 0: 625116491Sharti he_intr_rbrq(sc, &sc->rbrq_0, 0); 626116491Sharti break; 627116491Sharti 628116491Sharti case 1: 629116491Sharti if (sc->rbrq_1.size > 0) { 630116491Sharti he_intr_rbrq(sc, &sc->rbrq_1, 1); 631116491Sharti break; 632116491Sharti } 633116491Sharti /* FALLTHRU */ 634116491Sharti 635116491Sharti default: 636147256Sbrooks if_printf(sc->ifp, "bad INTR RBRQ%u\n", 637116491Sharti status & HE_REGM_IGROUP); 638116491Sharti break; 639116491Sharti } 640116491Sharti break; 641116491Sharti 642116491Sharti case HE_REGM_ITYPE_RBRQT: 643116491Sharti DBG(sc, INTR, ("INTERRUPT RBRQT %u", status & HE_REGM_IGROUP)); 644116491Sharti sc->istats.itype_rbrqt++; 645116491Sharti switch (status & HE_REGM_IGROUP) { 646116491Sharti 647116491Sharti case 0: 648116491Sharti he_intr_rbrq(sc, &sc->rbrq_0, 0); 649116491Sharti break; 650116491Sharti 651116491Sharti case 1: 652116491Sharti if (sc->rbrq_1.size > 0) { 653116491Sharti he_intr_rbrq(sc, &sc->rbrq_1, 1); 654116491Sharti break; 655116491Sharti } 656116491Sharti /* FALLTHRU */ 657116491Sharti 658116491Sharti default: 659147256Sbrooks if_printf(sc->ifp, "bad INTR RBRQT%u\n", 660116491Sharti status & HE_REGM_IGROUP); 661116491Sharti break; 662116491Sharti } 663116491Sharti break; 664116491Sharti 665116491Sharti case HE_REGM_ITYPE_PHYS: 666116491Sharti sc->istats.itype_phys++; 667116491Sharti utopia_intr(&sc->utopia); 668116491Sharti break; 669116491Sharti 670116491Sharti#if HE_REGM_ITYPE_UNKNOWN != HE_REGM_ITYPE_INVALID 671116491Sharti case HE_REGM_ITYPE_UNKNOWN: 672116491Sharti sc->istats.itype_unknown++; 673147256Sbrooks if_printf(sc->ifp, "bad interrupt\n"); 674116491Sharti break; 675116491Sharti#endif 676116491Sharti 677116491Sharti case HE_REGM_ITYPE_ERR: 678116491Sharti sc->istats.itype_err++; 679116491Sharti switch (status) { 680116491Sharti 681116491Sharti case HE_REGM_ITYPE_PERR: 682147256Sbrooks if_printf(sc->ifp, "parity error\n"); 683116491Sharti break; 684116491Sharti 685116491Sharti case HE_REGM_ITYPE_ABORT: 686147256Sbrooks if_printf(sc->ifp, "abort interrupt " 687116491Sharti "addr=0x%08x\n", 688116491Sharti READ4(sc, HE_REGO_ABORT_ADDR)); 689116491Sharti break; 690116491Sharti 691116491Sharti default: 692147256Sbrooks if_printf(sc->ifp, 693116491Sharti "bad interrupt type %08x\n", status); 694116491Sharti break; 695116491Sharti } 696116491Sharti break; 697116491Sharti 698116491Sharti case HE_REGM_ITYPE_INVALID: 699116491Sharti /* this is the documented fix for the ISW bug 8.1.1 700116491Sharti * Note, that the documented fix is partly wrong: 701116491Sharti * the ISWs should be intialized to 0xf8 not 0xff */ 702116491Sharti sc->istats.bug_bad_isw++; 703116491Sharti DBG(sc, INTR, ("hatm: invalid ISW bug triggered")); 704116491Sharti he_intr_tbrq(sc, &sc->tbrq, 0); 705116491Sharti he_intr_rbp(sc, &sc->rbp_s0, 0, 0); 706116491Sharti he_intr_rbp(sc, &sc->rbp_l0, 1, 0); 707116491Sharti he_intr_rbp(sc, &sc->rbp_s1, 0, 1); 708116491Sharti he_intr_rbrq(sc, &sc->rbrq_0, 0); 709116491Sharti he_intr_rbrq(sc, &sc->rbrq_1, 1); 710116491Sharti utopia_intr(&sc->utopia); 711116491Sharti break; 712116491Sharti 713116491Sharti default: 714147256Sbrooks if_printf(sc->ifp, "bad interrupt type %08x\n", 715116491Sharti status); 716116491Sharti break; 717116491Sharti } 718116491Sharti } 719116491Sharti 720116491Sharti /* write back head to clear queue */ 721116491Sharti WRITE4(sc, HE_REGO_IRQ_HEAD(0), 722116491Sharti ((q->size - 1) << HE_REGS_IRQ_HEAD_SIZE) | 723116491Sharti (q->thresh << HE_REGS_IRQ_HEAD_THRESH) | 724116491Sharti (q->head << HE_REGS_IRQ_HEAD_HEAD)); 725116491Sharti BARRIER_W(sc); 726116491Sharti 727116491Sharti /* workaround the back-to-back irq access problem (8.1.2) */ 728116491Sharti (void)READ4(sc, HE_REGO_INT_FIFO); 729116491Sharti BARRIER_R(sc); 730116491Sharti 731116491Sharti mtx_unlock(&sc->mtx); 732116491Sharti} 733