1364973Szec/*- 2364973Szec * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3364973Szec * 4364973Szec * Copyright (c) 2015 Bjoern A. Zeeb 5364973Szec * Copyright (c) 2020 Denis Salopek 6364973Szec * 7364973Szec * This software was developed by SRI International and the University of 8364973Szec * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 9364973Szec * ("MRC2"), as part of the DARPA MRC research programme. 10364973Szec * 11364973Szec * Redistribution and use in source and binary forms, with or without 12364973Szec * modification, are permitted provided that the following conditions 13364973Szec * are met: 14364973Szec * 1. Redistributions of source code must retain the above copyright 15364973Szec * notice, this list of conditions and the following disclaimer. 16364973Szec * 2. Redistributions in binary form must reproduce the above copyright 17364973Szec * notice, this list of conditions and the following disclaimer in the 18364973Szec * documentation and/or other materials provided with the distribution. 19364973Szec * 20364973Szec * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21364973Szec * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22364973Szec * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23364973Szec * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24364973Szec * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25364973Szec * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26364973Szec * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27364973Szec * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28364973Szec * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29364973Szec * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30364973Szec * POSSIBILITY OF SUCH DAMAGE. 31364973Szec */ 32364973Szec 33364973Szec#include <sys/cdefs.h> 34364973Szec__FBSDID("$FreeBSD: stable/11/sys/dev/sume/if_sume.c 364973 2020-08-30 07:34:32Z zec $"); 35364973Szec 36364973Szec#include <sys/param.h> 37364973Szec#include <sys/bus.h> 38364973Szec#include <sys/endian.h> 39364973Szec#include <sys/kernel.h> 40364973Szec#include <sys/limits.h> 41364973Szec#include <sys/module.h> 42364973Szec#include <sys/rman.h> 43364973Szec#include <sys/socket.h> 44364973Szec#include <sys/sockio.h> 45364973Szec#include <sys/sysctl.h> 46364973Szec#include <sys/taskqueue.h> 47364973Szec 48364973Szec#include <net/if.h> 49364973Szec#include <net/if_media.h> 50364973Szec#include <net/if_types.h> 51364973Szec#include <net/if_var.h> 52364973Szec 53364973Szec#include <netinet/in.h> 54364973Szec#include <netinet/if_ether.h> 55364973Szec 56364973Szec#include <dev/pci/pcivar.h> 57364973Szec#include <dev/pci/pcireg.h> 58364973Szec 59364973Szec#include <machine/bus.h> 60364973Szec 61364973Szec#include "adapter.h" 62364973Szec 63364973Szec#define PCI_VENDOR_ID_XILINX 0x10ee 64364973Szec#define PCI_DEVICE_ID_SUME 0x7028 65364973Szec 66364973Szec/* SUME bus driver interface */ 67364973Szecstatic int sume_probe(device_t); 68364973Szecstatic int sume_attach(device_t); 69364973Szecstatic int sume_detach(device_t); 70364973Szec 71364973Szecstatic device_method_t sume_methods[] = { 72364973Szec DEVMETHOD(device_probe, sume_probe), 73364973Szec DEVMETHOD(device_attach, sume_attach), 74364973Szec DEVMETHOD(device_detach, sume_detach), 75364973Szec DEVMETHOD_END 76364973Szec}; 77364973Szec 78364973Szecstatic driver_t sume_driver = { 79364973Szec "sume", 80364973Szec sume_methods, 81364973Szec sizeof(struct sume_adapter) 82364973Szec}; 83364973Szec 84364973Szec/* 85364973Szec * The DMA engine for SUME generates interrupts for each RX/TX transaction. 86364973Szec * Depending on the channel (0 if packet transaction, 1 if register transaction) 87364973Szec * the used bits of the interrupt vector will be the lowest or the second lowest 88364973Szec * 5 bits. 89364973Szec * 90364973Szec * When receiving packets from SUME (RX): 91364973Szec * (1) SUME received a packet on one of the interfaces. 92364973Szec * (2) SUME generates an interrupt vector, bit 00001 is set (channel 0 - new RX 93364973Szec * transaction). 94364973Szec * (3) We read the length of the incoming packet and the offset along with the 95364973Szec * 'last' flag from the SUME registers. 96364973Szec * (4) We prepare for the DMA transaction by setting the bouncebuffer on the 97364973Szec * address buf_addr. For now, this is how it's done: 98364973Szec * - First 3*sizeof(uint32_t) bytes are: lower and upper 32 bits of physical 99364973Szec * address where we want the data to arrive (buf_addr[0] and buf_addr[1]), 100364973Szec * and length of incoming data (buf_addr[2]). 101364973Szec * - Data will start right after, at buf_addr+3*sizeof(uint32_t). The 102364973Szec * physical address buf_hw_addr is a block of contiguous memory mapped to 103364973Szec * buf_addr, so we can set the incoming data's physical address (buf_addr[0] 104364973Szec * and buf_addr[1]) to buf_hw_addr+3*sizeof(uint32_t). 105364973Szec * (5) We notify SUME that the bouncebuffer is ready for the transaction by 106364973Szec * writing the lower/upper physical address buf_hw_addr to the SUME 107364973Szec * registers RIFFA_TX_SG_ADDR_LO_REG_OFF and RIFFA_TX_SG_ADDR_HI_REG_OFF as 108364973Szec * well as the number of segments to the register RIFFA_TX_SG_LEN_REG_OFF. 109364973Szec * (6) SUME generates an interrupt vector, bit 00010 is set (channel 0 - 110364973Szec * bouncebuffer received). 111364973Szec * (7) SUME generates an interrupt vector, bit 00100 is set (channel 0 - 112364973Szec * transaction is done). 113364973Szec * (8) SUME can do both steps (6) and (7) using the same interrupt. 114364973Szec * (8) We read the first 16 bytes (metadata) of the received data and note the 115364973Szec * incoming interface so we can later forward it to the right one in the OS 116364973Szec * (sume0, sume1, sume2 or sume3). 117364973Szec * (10) We create an mbuf and copy the data from the bouncebuffer to the mbuf 118364973Szec * and set the mbuf rcvif to the incoming interface. 119364973Szec * (11) We forward the mbuf to the appropriate interface via ifp->if_input. 120364973Szec * 121364973Szec * When sending packets to SUME (TX): 122364973Szec * (1) The OS calls sume_if_start() function on TX. 123364973Szec * (2) We get the mbuf packet data and copy it to the 124364973Szec * buf_addr+3*sizeof(uint32_t) + metadata 16 bytes. 125364973Szec * (3) We create the metadata based on the output interface and copy it to the 126364973Szec * buf_addr+3*sizeof(uint32_t). 127364973Szec * (4) We write the offset/last and length of the packet to the SUME registers 128364973Szec * RIFFA_RX_OFFLAST_REG_OFF and RIFFA_RX_LEN_REG_OFF. 129364973Szec * (5) We fill the bouncebuffer by filling the first 3*sizeof(uint32_t) bytes 130364973Szec * with the physical address and length just as in RX step (4). 131364973Szec * (6) We notify SUME that the bouncebuffer is ready by writing to SUME 132364973Szec * registers RIFFA_RX_SG_ADDR_LO_REG_OFF, RIFFA_RX_SG_ADDR_HI_REG_OFF and 133364973Szec * RIFFA_RX_SG_LEN_REG_OFF just as in RX step (5). 134364973Szec * (7) SUME generates an interrupt vector, bit 01000 is set (channel 0 - 135364973Szec * bouncebuffer is read). 136364973Szec * (8) SUME generates an interrupt vector, bit 10000 is set (channel 0 - 137364973Szec * transaction is done). 138364973Szec * (9) SUME can do both steps (7) and (8) using the same interrupt. 139364973Szec * 140364973Szec * Internal registers 141364973Szec * Every module in the SUME hardware has its own set of internal registers 142364973Szec * (IDs, for debugging and statistic purposes, etc.). Their base addresses are 143364973Szec * defined in 'projects/reference_nic/hw/tcl/reference_nic_defines.tcl' and the 144364973Szec * offsets to different memory locations of every module are defined in their 145364973Szec * corresponding folder inside the library. These registers can be RO/RW and 146364973Szec * there is a special method to fetch/change this data over 1 or 2 DMA 147364973Szec * transactions. For writing, by calling the sume_module_reg_write(). For 148364973Szec * reading, by calling the sume_module_reg_write() and then 149364973Szec * sume_module_reg_read(). Check those functions for more information. 150364973Szec */ 151364973Szec 152364973SzecMALLOC_DECLARE(M_SUME); 153364973SzecMALLOC_DEFINE(M_SUME, "sume", "NetFPGA SUME device driver"); 154364973Szec 155364973Szecstatic void check_tx_queues(struct sume_adapter *); 156364973Szecstatic void sume_fill_bb_desc(struct sume_adapter *, struct riffa_chnl_dir *, 157364973Szec uint64_t); 158364973Szec 159364973Szecstatic struct unrhdr *unr; 160364973Szec 161364973Szecstatic struct { 162364973Szec uint16_t device; 163364973Szec char *desc; 164364973Szec} sume_pciids[] = { 165364973Szec {PCI_DEVICE_ID_SUME, "NetFPGA SUME reference NIC"}, 166364973Szec}; 167364973Szec 168364973Szecstatic inline uint32_t 169364973Szecread_reg(struct sume_adapter *adapter, int offset) 170364973Szec{ 171364973Szec 172364973Szec return (bus_space_read_4(adapter->bt, adapter->bh, offset << 2)); 173364973Szec} 174364973Szec 175364973Szecstatic inline void 176364973Szecwrite_reg(struct sume_adapter *adapter, int offset, uint32_t val) 177364973Szec{ 178364973Szec 179364973Szec bus_space_write_4(adapter->bt, adapter->bh, offset << 2, val); 180364973Szec} 181364973Szec 182364973Szecstatic int 183364973Szecsume_probe(device_t dev) 184364973Szec{ 185364973Szec int i; 186364973Szec uint16_t v = pci_get_vendor(dev); 187364973Szec uint16_t d = pci_get_device(dev); 188364973Szec 189364973Szec if (v != PCI_VENDOR_ID_XILINX) 190364973Szec return (ENXIO); 191364973Szec 192364973Szec for (i = 0; i < nitems(sume_pciids); i++) { 193364973Szec if (d == sume_pciids[i].device) { 194364973Szec device_set_desc(dev, sume_pciids[i].desc); 195364973Szec return (BUS_PROBE_DEFAULT); 196364973Szec } 197364973Szec } 198364973Szec 199364973Szec return (ENXIO); 200364973Szec} 201364973Szec 202364973Szec/* 203364973Szec * Building mbuf for packet received from SUME. We expect to receive 'len' 204364973Szec * bytes of data (including metadata) written from the bouncebuffer address 205364973Szec * buf_addr+3*sizeof(uint32_t). Metadata will tell us which SUME interface 206364973Szec * received the packet (sport will be 1, 2, 4 or 8), the packet length (plen), 207364973Szec * and the magic word needs to be 0xcafe. When we have the packet data, we 208364973Szec * create an mbuf and copy the data to it using m_copyback() function, set the 209364973Szec * correct interface to rcvif and return the mbuf to be later sent to the OS 210364973Szec * with if_input. 211364973Szec */ 212364973Szecstatic struct mbuf * 213364973Szecsume_rx_build_mbuf(struct sume_adapter *adapter, uint32_t len) 214364973Szec{ 215364973Szec struct nf_priv *nf_priv; 216364973Szec struct mbuf *m; 217364973Szec struct ifnet *ifp = NULL; 218364973Szec int np; 219364973Szec uint16_t dport, plen, magic; 220364973Szec device_t dev = adapter->dev; 221364973Szec uint8_t *indata = (uint8_t *) 222364973Szec adapter->recv[SUME_RIFFA_CHANNEL_DATA]->buf_addr + 223364973Szec sizeof(struct nf_bb_desc); 224364973Szec struct nf_metadata *mdata = (struct nf_metadata *) indata; 225364973Szec 226364973Szec /* The metadata header is 16 bytes. */ 227364973Szec if (len < sizeof(struct nf_metadata)) { 228364973Szec device_printf(dev, "short frame (%d)\n", len); 229364973Szec adapter->packets_err++; 230364973Szec adapter->bytes_err += len; 231364973Szec return (NULL); 232364973Szec } 233364973Szec 234364973Szec dport = le16toh(mdata->dport); 235364973Szec plen = le16toh(mdata->plen); 236364973Szec magic = le16toh(mdata->magic); 237364973Szec 238364973Szec if (sizeof(struct nf_metadata) + plen > len || 239364973Szec magic != SUME_RIFFA_MAGIC) { 240364973Szec device_printf(dev, "corrupted packet (%zd + %d > %d || magic " 241364973Szec "0x%04x != 0x%04x)\n", sizeof(struct nf_metadata), plen, 242364973Szec len, magic, SUME_RIFFA_MAGIC); 243364973Szec return (NULL); 244364973Szec } 245364973Szec 246364973Szec /* We got the packet from one of the even bits */ 247364973Szec np = (ffs(dport & SUME_DPORT_MASK) >> 1) - 1; 248364973Szec if (np > SUME_NPORTS) { 249364973Szec device_printf(dev, "invalid destination port 0x%04x (%d)\n", 250364973Szec dport, np); 251364973Szec adapter->packets_err++; 252364973Szec adapter->bytes_err += plen; 253364973Szec return (NULL); 254364973Szec } 255364973Szec ifp = adapter->ifp[np]; 256364973Szec nf_priv = ifp->if_softc; 257364973Szec nf_priv->stats.rx_packets++; 258364973Szec nf_priv->stats.rx_bytes += plen; 259364973Szec 260364973Szec /* If the interface is down, well, we are done. */ 261364973Szec if (!(ifp->if_flags & IFF_UP)) { 262364973Szec nf_priv->stats.ifc_down_packets++; 263364973Szec nf_priv->stats.ifc_down_bytes += plen; 264364973Szec return (NULL); 265364973Szec } 266364973Szec 267364973Szec if (adapter->sume_debug) 268364973Szec printf("Building mbuf with length: %d\n", plen); 269364973Szec 270364973Szec m = m_getm(NULL, plen, M_NOWAIT, MT_DATA); 271364973Szec if (m == NULL) { 272364973Szec adapter->packets_err++; 273364973Szec adapter->bytes_err += plen; 274364973Szec return (NULL); 275364973Szec } 276364973Szec 277364973Szec /* Copy the data in at the right offset. */ 278364973Szec m_copyback(m, 0, plen, (void *) (indata + sizeof(struct nf_metadata))); 279364973Szec m->m_pkthdr.rcvif = ifp; 280364973Szec 281364973Szec return (m); 282364973Szec} 283364973Szec 284364973Szec/* 285364973Szec * SUME interrupt handler for when we get a valid interrupt from the board. 286364973Szec * Theoretically, we can receive interrupt for any of the available channels, 287364973Szec * but RIFFA DMA uses only 2: 0 and 1, so we use only vect0. The vector is a 32 288364973Szec * bit number, using 5 bits for every channel, the least significant bits 289364973Szec * correspond to channel 0 and the next 5 bits correspond to channel 1. Vector 290364973Szec * bits for RX/TX are: 291364973Szec * RX 292364973Szec * bit 0 - new transaction from SUME 293364973Szec * bit 1 - SUME received our bouncebuffer address 294364973Szec * bit 2 - SUME copied the received data to our bouncebuffer, transaction done 295364973Szec * TX 296364973Szec * bit 3 - SUME received our bouncebuffer address 297364973Szec * bit 4 - SUME copied the data from our bouncebuffer, transaction done 298364973Szec * 299364973Szec * There are two finite state machines (one for TX, one for RX). We loop 300364973Szec * through channels 0 and 1 to check and our current state and which interrupt 301364973Szec * bit is set. 302364973Szec * TX 303364973Szec * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the first TX transaction. 304364973Szec * SUME_RIFFA_CHAN_STATE_READY: we prepared (filled with data) the bouncebuffer 305364973Szec * and triggered the SUME for the TX transaction. Waiting for interrupt bit 3 306364973Szec * to go to the next state. 307364973Szec * SUME_RIFFA_CHAN_STATE_READ: waiting for interrupt bit 4 (for SUME to send 308364973Szec * our packet). Then we get the length of the sent data and go back to the 309364973Szec * IDLE state. 310364973Szec * RX 311364973Szec * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the interrupt bit 0 (new RX 312364973Szec * transaction). When we get it, we prepare our bouncebuffer for reading and 313364973Szec * trigger the SUME to start the transaction. Go to the next state. 314364973Szec * SUME_RIFFA_CHAN_STATE_READY: waiting for the interrupt bit 1 (SUME got our 315364973Szec * bouncebuffer). Go to the next state. 316364973Szec * SUME_RIFFA_CHAN_STATE_READ: SUME copied data and our bouncebuffer is ready, 317364973Szec * we can build the mbuf and go back to the IDLE state. 318364973Szec */ 319364973Szecstatic void 320364973Szecsume_intr_handler(void *arg) 321364973Szec{ 322364973Szec struct sume_adapter *adapter = arg; 323364973Szec uint32_t vect, vect0, len; 324364973Szec int ch, loops; 325364973Szec device_t dev = adapter->dev; 326364973Szec struct mbuf *m = NULL; 327364973Szec struct ifnet *ifp = NULL; 328364973Szec struct riffa_chnl_dir *send, *recv; 329364973Szec 330364973Szec SUME_LOCK(adapter); 331364973Szec 332364973Szec vect0 = read_reg(adapter, RIFFA_IRQ_REG0_OFF); 333364973Szec if ((vect0 & SUME_INVALID_VECT) != 0) { 334364973Szec SUME_UNLOCK(adapter); 335364973Szec return; 336364973Szec } 337364973Szec 338364973Szec /* 339364973Szec * We only have one interrupt for all channels and no way 340364973Szec * to quickly lookup for which channel(s) we got an interrupt? 341364973Szec */ 342364973Szec for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 343364973Szec vect = vect0 >> (5 * ch); 344364973Szec send = adapter->send[ch]; 345364973Szec recv = adapter->recv[ch]; 346364973Szec 347364973Szec loops = 0; 348364973Szec while ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) && 349364973Szec loops <= 5) { 350364973Szec if (adapter->sume_debug) 351364973Szec device_printf(dev, "TX ch %d state %u vect = " 352364973Szec "0x%08x\n", ch, send->state, vect); 353364973Szec switch (send->state) { 354364973Szec case SUME_RIFFA_CHAN_STATE_IDLE: 355364973Szec break; 356364973Szec case SUME_RIFFA_CHAN_STATE_READY: 357364973Szec if (!(vect & SUME_MSI_TXBUF)) { 358364973Szec device_printf(dev, "ch %d unexpected " 359364973Szec "interrupt in send+3 state %u: " 360364973Szec "vect = 0x%08x\n", ch, send->state, 361364973Szec vect); 362364973Szec send->recovery = 1; 363364973Szec break; 364364973Szec } 365364973Szec send->state = SUME_RIFFA_CHAN_STATE_READ; 366364973Szec vect &= ~SUME_MSI_TXBUF; 367364973Szec break; 368364973Szec case SUME_RIFFA_CHAN_STATE_READ: 369364973Szec if (!(vect & SUME_MSI_TXDONE)) { 370364973Szec device_printf(dev, "ch %d unexpected " 371364973Szec "interrupt in send+4 state %u: " 372364973Szec "vect = 0x%08x\n", ch, send->state, 373364973Szec vect); 374364973Szec send->recovery = 1; 375364973Szec break; 376364973Szec } 377364973Szec send->state = SUME_RIFFA_CHAN_STATE_LEN; 378364973Szec 379364973Szec len = read_reg(adapter, RIFFA_CHNL_REG(ch, 380364973Szec RIFFA_RX_TNFR_LEN_REG_OFF)); 381364973Szec if (ch == SUME_RIFFA_CHANNEL_DATA) { 382364973Szec send->state = 383364973Szec SUME_RIFFA_CHAN_STATE_IDLE; 384364973Szec check_tx_queues(adapter); 385364973Szec } else if (ch == SUME_RIFFA_CHANNEL_REG) 386364973Szec wakeup(&send->event); 387364973Szec else { 388364973Szec device_printf(dev, "ch %d unexpected " 389364973Szec "interrupt in send+4 state %u: " 390364973Szec "vect = 0x%08x\n", ch, send->state, 391364973Szec vect); 392364973Szec send->recovery = 1; 393364973Szec } 394364973Szec vect &= ~SUME_MSI_TXDONE; 395364973Szec break; 396364973Szec case SUME_RIFFA_CHAN_STATE_LEN: 397364973Szec break; 398364973Szec default: 399364973Szec device_printf(dev, "unknown TX state!\n"); 400364973Szec } 401364973Szec loops++; 402364973Szec } 403364973Szec 404364973Szec if ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) && 405364973Szec send->recovery) 406364973Szec device_printf(dev, "ch %d ignoring vect = 0x%08x " 407364973Szec "during TX; not in recovery; state = %d loops = " 408364973Szec "%d\n", ch, vect, send->state, loops); 409364973Szec 410364973Szec loops = 0; 411364973Szec while ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF | 412364973Szec SUME_MSI_RXDONE)) && loops < 5) { 413364973Szec if (adapter->sume_debug) 414364973Szec device_printf(dev, "RX ch %d state %u vect = " 415364973Szec "0x%08x\n", ch, recv->state, vect); 416364973Szec switch (recv->state) { 417364973Szec case SUME_RIFFA_CHAN_STATE_IDLE: 418364973Szec if (!(vect & SUME_MSI_RXQUE)) { 419364973Szec device_printf(dev, "ch %d unexpected " 420364973Szec "interrupt in recv+0 state %u: " 421364973Szec "vect = 0x%08x\n", ch, recv->state, 422364973Szec vect); 423364973Szec recv->recovery = 1; 424364973Szec break; 425364973Szec } 426364973Szec uint32_t max_ptr; 427364973Szec 428364973Szec /* Clear recovery state. */ 429364973Szec recv->recovery = 0; 430364973Szec 431364973Szec /* Get offset and length. */ 432364973Szec recv->offlast = read_reg(adapter, 433364973Szec RIFFA_CHNL_REG(ch, 434364973Szec RIFFA_TX_OFFLAST_REG_OFF)); 435364973Szec recv->len = read_reg(adapter, RIFFA_CHNL_REG(ch, 436364973Szec RIFFA_TX_LEN_REG_OFF)); 437364973Szec 438364973Szec /* Boundary checks. */ 439364973Szec max_ptr = (uint32_t)((uintptr_t)recv->buf_addr 440364973Szec + SUME_RIFFA_OFFSET(recv->offlast) 441364973Szec + SUME_RIFFA_LEN(recv->len) - 1); 442364973Szec if (max_ptr < 443364973Szec (uint32_t)((uintptr_t)recv->buf_addr)) 444364973Szec device_printf(dev, "receive buffer " 445364973Szec "wrap-around overflow.\n"); 446364973Szec if (SUME_RIFFA_OFFSET(recv->offlast) + 447364973Szec SUME_RIFFA_LEN(recv->len) > 448364973Szec adapter->sg_buf_size) 449364973Szec device_printf(dev, "receive buffer too" 450364973Szec " small.\n"); 451364973Szec 452364973Szec /* Fill the bouncebuf "descriptor". */ 453364973Szec sume_fill_bb_desc(adapter, recv, 454364973Szec SUME_RIFFA_LEN(recv->len)); 455364973Szec 456364973Szec bus_dmamap_sync(recv->ch_tag, recv->ch_map, 457364973Szec BUS_DMASYNC_PREREAD | 458364973Szec BUS_DMASYNC_PREWRITE); 459364973Szec write_reg(adapter, RIFFA_CHNL_REG(ch, 460364973Szec RIFFA_TX_SG_ADDR_LO_REG_OFF), 461364973Szec SUME_RIFFA_LO_ADDR(recv->buf_hw_addr)); 462364973Szec write_reg(adapter, RIFFA_CHNL_REG(ch, 463364973Szec RIFFA_TX_SG_ADDR_HI_REG_OFF), 464364973Szec SUME_RIFFA_HI_ADDR(recv->buf_hw_addr)); 465364973Szec write_reg(adapter, RIFFA_CHNL_REG(ch, 466364973Szec RIFFA_TX_SG_LEN_REG_OFF), 467364973Szec 4 * recv->num_sg); 468364973Szec bus_dmamap_sync(recv->ch_tag, recv->ch_map, 469364973Szec BUS_DMASYNC_POSTREAD | 470364973Szec BUS_DMASYNC_POSTWRITE); 471364973Szec 472364973Szec recv->state = SUME_RIFFA_CHAN_STATE_READY; 473364973Szec vect &= ~SUME_MSI_RXQUE; 474364973Szec break; 475364973Szec case SUME_RIFFA_CHAN_STATE_READY: 476364973Szec if (!(vect & SUME_MSI_RXBUF)) { 477364973Szec device_printf(dev, "ch %d unexpected " 478364973Szec "interrupt in recv+1 state %u: " 479364973Szec "vect = 0x%08x\n", ch, recv->state, 480364973Szec vect); 481364973Szec recv->recovery = 1; 482364973Szec break; 483364973Szec } 484364973Szec recv->state = SUME_RIFFA_CHAN_STATE_READ; 485364973Szec vect &= ~SUME_MSI_RXBUF; 486364973Szec break; 487364973Szec case SUME_RIFFA_CHAN_STATE_READ: 488364973Szec if (!(vect & SUME_MSI_RXDONE)) { 489364973Szec device_printf(dev, "ch %d unexpected " 490364973Szec "interrupt in recv+2 state %u: " 491364973Szec "vect = 0x%08x\n", ch, recv->state, 492364973Szec vect); 493364973Szec recv->recovery = 1; 494364973Szec break; 495364973Szec } 496364973Szec len = read_reg(adapter, RIFFA_CHNL_REG(ch, 497364973Szec RIFFA_TX_TNFR_LEN_REG_OFF)); 498364973Szec 499364973Szec /* Remember, len and recv->len are words. */ 500364973Szec if (ch == SUME_RIFFA_CHANNEL_DATA) { 501364973Szec m = sume_rx_build_mbuf(adapter, 502364973Szec len << 2); 503364973Szec recv->state = 504364973Szec SUME_RIFFA_CHAN_STATE_IDLE; 505364973Szec } else if (ch == SUME_RIFFA_CHANNEL_REG) 506364973Szec wakeup(&recv->event); 507364973Szec else { 508364973Szec device_printf(dev, "ch %d unexpected " 509364973Szec "interrupt in recv+2 state %u: " 510364973Szec "vect = 0x%08x\n", ch, recv->state, 511364973Szec vect); 512364973Szec recv->recovery = 1; 513364973Szec } 514364973Szec vect &= ~SUME_MSI_RXDONE; 515364973Szec break; 516364973Szec case SUME_RIFFA_CHAN_STATE_LEN: 517364973Szec break; 518364973Szec default: 519364973Szec device_printf(dev, "unknown RX state!\n"); 520364973Szec } 521364973Szec loops++; 522364973Szec } 523364973Szec 524364973Szec if ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF | 525364973Szec SUME_MSI_RXDONE)) && recv->recovery) { 526364973Szec device_printf(dev, "ch %d ignoring vect = 0x%08x " 527364973Szec "during RX; not in recovery; state = %d, loops = " 528364973Szec "%d\n", ch, vect, recv->state, loops); 529364973Szec 530364973Szec /* Clean the unfinished transaction. */ 531364973Szec if (ch == SUME_RIFFA_CHANNEL_REG && 532364973Szec vect & SUME_MSI_RXDONE) { 533364973Szec read_reg(adapter, RIFFA_CHNL_REG(ch, 534364973Szec RIFFA_TX_TNFR_LEN_REG_OFF)); 535364973Szec recv->recovery = 0; 536364973Szec } 537364973Szec } 538364973Szec } 539364973Szec SUME_UNLOCK(adapter); 540364973Szec 541364973Szec if (m != NULL) { 542364973Szec ifp = m->m_pkthdr.rcvif; 543364973Szec (*ifp->if_input)(ifp, m); 544364973Szec } 545364973Szec} 546364973Szec 547364973Szec/* 548364973Szec * As we cannot disable interrupt generation, ignore early interrupts by waiting 549364973Szec * for the adapter to go into the 'running' state. 550364973Szec */ 551364973Szecstatic int 552364973Szecsume_intr_filter(void *arg) 553364973Szec{ 554364973Szec struct sume_adapter *adapter = arg; 555364973Szec 556364973Szec if (adapter->running == 0) 557364973Szec return (FILTER_STRAY); 558364973Szec 559364973Szec return (FILTER_SCHEDULE_THREAD); 560364973Szec} 561364973Szec 562364973Szecstatic int 563364973Szecsume_probe_riffa_pci(struct sume_adapter *adapter) 564364973Szec{ 565364973Szec device_t dev = adapter->dev; 566364973Szec int error, count, capmem; 567364973Szec uint32_t reg, devctl, linkctl; 568364973Szec 569364973Szec pci_enable_busmaster(dev); 570364973Szec 571364973Szec adapter->rid = PCIR_BAR(0); 572364973Szec adapter->bar0_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 573364973Szec &adapter->rid, RF_ACTIVE); 574364973Szec if (adapter->bar0_addr == NULL) { 575364973Szec device_printf(dev, "unable to allocate bus resource: " 576364973Szec "BAR0 address\n"); 577364973Szec return (ENXIO); 578364973Szec } 579364973Szec adapter->bt = rman_get_bustag(adapter->bar0_addr); 580364973Szec adapter->bh = rman_get_bushandle(adapter->bar0_addr); 581364973Szec adapter->bar0_len = rman_get_size(adapter->bar0_addr); 582364973Szec if (adapter->bar0_len != 1024) { 583364973Szec device_printf(dev, "BAR0 resource length %lu != 1024\n", 584364973Szec adapter->bar0_len); 585364973Szec return (ENXIO); 586364973Szec } 587364973Szec 588364973Szec count = pci_msi_count(dev); 589364973Szec error = pci_alloc_msi(dev, &count); 590364973Szec if (error) { 591364973Szec device_printf(dev, "unable to allocate bus resource: PCI " 592364973Szec "MSI\n"); 593364973Szec return (error); 594364973Szec } 595364973Szec 596364973Szec adapter->irq.rid = 1; /* Should be 1, thus says pci_alloc_msi() */ 597364973Szec adapter->irq.res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 598364973Szec &adapter->irq.rid, RF_SHAREABLE | RF_ACTIVE); 599364973Szec if (adapter->irq.res == NULL) { 600364973Szec device_printf(dev, "unable to allocate bus resource: IRQ " 601364973Szec "memory\n"); 602364973Szec return (ENXIO); 603364973Szec } 604364973Szec 605364973Szec error = bus_setup_intr(dev, adapter->irq.res, INTR_MPSAFE | 606364973Szec INTR_TYPE_NET, sume_intr_filter, sume_intr_handler, adapter, 607364973Szec &adapter->irq.tag); 608364973Szec if (error) { 609364973Szec device_printf(dev, "failed to setup interrupt for rid %d, name" 610364973Szec " %s: %d\n", adapter->irq.rid, "SUME_INTR", error); 611364973Szec return (ENXIO); 612364973Szec } 613364973Szec 614364973Szec if (pci_find_cap(dev, PCIY_EXPRESS, &capmem) != 0) { 615364973Szec device_printf(dev, "PCI not PCIe capable\n"); 616364973Szec return (ENXIO); 617364973Szec } 618364973Szec 619364973Szec devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL, 2); 620364973Szec pci_write_config(dev, capmem + PCIER_DEVICE_CTL, (devctl | 621364973Szec PCIEM_CTL_EXT_TAG_FIELD), 2); 622364973Szec 623364973Szec devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL2, 2); 624364973Szec pci_write_config(dev, capmem + PCIER_DEVICE_CTL2, (devctl | 625364973Szec PCIEM_CTL2_ID_ORDERED_REQ_EN), 2); 626364973Szec 627364973Szec linkctl = pci_read_config(dev, capmem + PCIER_LINK_CTL, 2); 628364973Szec pci_write_config(dev, capmem + PCIER_LINK_CTL, (linkctl | 629364973Szec PCIEM_LINK_CTL_RCB), 2); 630364973Szec 631364973Szec reg = read_reg(adapter, RIFFA_INFO_REG_OFF); 632364973Szec adapter->num_sg = RIFFA_SG_ELEMS * ((reg >> 19) & 0xf); 633364973Szec adapter->sg_buf_size = RIFFA_SG_BUF_SIZE * ((reg >> 19) & 0xf); 634364973Szec 635364973Szec error = ENODEV; 636364973Szec /* Check bus master is enabled. */ 637364973Szec if (((reg >> 4) & 0x1) != 1) { 638364973Szec device_printf(dev, "bus master not enabled: %d\n", 639364973Szec (reg >> 4) & 0x1); 640364973Szec return (error); 641364973Szec } 642364973Szec /* Check link parameters are valid. */ 643364973Szec if (((reg >> 5) & 0x3f) == 0 || ((reg >> 11) & 0x3) == 0) { 644364973Szec device_printf(dev, "link parameters not valid: %d %d\n", 645364973Szec (reg >> 5) & 0x3f, (reg >> 11) & 0x3); 646364973Szec return (error); 647364973Szec } 648364973Szec /* Check # of channels are within valid range. */ 649364973Szec if ((reg & 0xf) == 0 || (reg & 0xf) > RIFFA_MAX_CHNLS) { 650364973Szec device_printf(dev, "number of channels out of range: %d\n", 651364973Szec reg & 0xf); 652364973Szec return (error); 653364973Szec } 654364973Szec /* Check bus width. */ 655364973Szec if (((reg >> 19) & 0xf) == 0 || 656364973Szec ((reg >> 19) & 0xf) > RIFFA_MAX_BUS_WIDTH_PARAM) { 657364973Szec device_printf(dev, "bus width out of range: %d\n", 658364973Szec (reg >> 19) & 0xf); 659364973Szec return (error); 660364973Szec } 661364973Szec 662364973Szec device_printf(dev, "[riffa] # of channels: %d\n", 663364973Szec reg & 0xf); 664364973Szec device_printf(dev, "[riffa] bus interface width: %d\n", 665364973Szec ((reg >> 19) & 0xf) << 5); 666364973Szec device_printf(dev, "[riffa] bus master enabled: %d\n", 667364973Szec (reg >> 4) & 0x1); 668364973Szec device_printf(dev, "[riffa] negotiated link width: %d\n", 669364973Szec (reg >> 5) & 0x3f); 670364973Szec device_printf(dev, "[riffa] negotiated rate width: %d MTs\n", 671364973Szec ((reg >> 11) & 0x3) * 2500); 672364973Szec device_printf(dev, "[riffa] max downstream payload: %d B\n", 673364973Szec 128 << ((reg >> 13) & 0x7)); 674364973Szec device_printf(dev, "[riffa] max upstream payload: %d B\n", 675364973Szec 128 << ((reg >> 16) & 0x7)); 676364973Szec 677364973Szec return (0); 678364973Szec} 679364973Szec 680364973Szec/* If there is no sume_if_init, the ether_ioctl panics. */ 681364973Szecstatic void 682364973Szecsume_if_init(void *sc) 683364973Szec{ 684364973Szec} 685364973Szec 686364973Szec/* Write the address and length for our incoming / outgoing transaction. */ 687364973Szecstatic void 688364973Szecsume_fill_bb_desc(struct sume_adapter *adapter, struct riffa_chnl_dir *p, 689364973Szec uint64_t len) 690364973Szec{ 691364973Szec struct nf_bb_desc *bouncebuf = (struct nf_bb_desc *) p->buf_addr; 692364973Szec 693364973Szec bouncebuf->lower = (p->buf_hw_addr + sizeof(struct nf_bb_desc)); 694364973Szec bouncebuf->upper = (p->buf_hw_addr + sizeof(struct nf_bb_desc)) >> 32; 695364973Szec bouncebuf->len = len >> 2; 696364973Szec} 697364973Szec 698364973Szec/* Module register locked write. */ 699364973Szecstatic int 700364973Szecsume_modreg_write_locked(struct sume_adapter *adapter) 701364973Szec{ 702364973Szec struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 703364973Szec 704364973Szec /* Let the FPGA know about the transfer. */ 705364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 706364973Szec RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST); 707364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 708364973Szec RIFFA_RX_LEN_REG_OFF), send->len); /* words */ 709364973Szec 710364973Szec /* Fill the bouncebuf "descriptor". */ 711364973Szec sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len)); 712364973Szec 713364973Szec /* Update the state before intiating the DMA to avoid races. */ 714364973Szec send->state = SUME_RIFFA_CHAN_STATE_READY; 715364973Szec 716364973Szec bus_dmamap_sync(send->ch_tag, send->ch_map, 717364973Szec BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 718364973Szec /* DMA. */ 719364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 720364973Szec RIFFA_RX_SG_ADDR_LO_REG_OFF), 721364973Szec SUME_RIFFA_LO_ADDR(send->buf_hw_addr)); 722364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 723364973Szec RIFFA_RX_SG_ADDR_HI_REG_OFF), 724364973Szec SUME_RIFFA_HI_ADDR(send->buf_hw_addr)); 725364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 726364973Szec RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg); 727364973Szec bus_dmamap_sync(send->ch_tag, send->ch_map, 728364973Szec BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 729364973Szec 730364973Szec return (0); 731364973Szec} 732364973Szec 733364973Szec/* 734364973Szec * Request a register read or write (depending on optype). 735364973Szec * If optype is set (0x1f) this will result in a register write, 736364973Szec * otherwise this will result in a register read request at the given 737364973Szec * address and the result will need to be DMAed back. 738364973Szec */ 739364973Szecstatic int 740364973Szecsume_module_reg_write(struct nf_priv *nf_priv, struct sume_ifreq *sifr, 741364973Szec uint32_t optype) 742364973Szec{ 743364973Szec struct sume_adapter *adapter = nf_priv->adapter; 744364973Szec struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 745364973Szec struct nf_regop_data *data; 746364973Szec int error; 747364973Szec 748364973Szec /* 749364973Szec * 1. Make sure the channel is free; otherwise return EBUSY. 750364973Szec * 2. Prepare the memory in the bounce buffer (which we always 751364973Szec * use for regs). 752364973Szec * 3. Start the DMA process. 753364973Szec * 4. Sleep and wait for result and return success or error. 754364973Szec */ 755364973Szec SUME_LOCK(adapter); 756364973Szec 757364973Szec if (send->state != SUME_RIFFA_CHAN_STATE_IDLE) { 758364973Szec SUME_UNLOCK(adapter); 759364973Szec return (EBUSY); 760364973Szec } 761364973Szec 762364973Szec data = (struct nf_regop_data *) (send->buf_addr + 763364973Szec sizeof(struct nf_bb_desc)); 764364973Szec data->addr = htole32(sifr->addr); 765364973Szec data->val = htole32(sifr->val); 766364973Szec /* Tag to indentify request. */ 767364973Szec data->rtag = htole32(++send->rtag); 768364973Szec data->optype = htole32(optype); 769364973Szec send->len = sizeof(struct nf_regop_data) / 4; /* words */ 770364973Szec 771364973Szec error = sume_modreg_write_locked(adapter); 772364973Szec if (error) { 773364973Szec SUME_UNLOCK(adapter); 774364973Szec return (EFAULT); 775364973Szec } 776364973Szec 777364973Szec /* Timeout after 1s. */ 778364973Szec if (send->state != SUME_RIFFA_CHAN_STATE_LEN) 779364973Szec error = msleep(&send->event, &adapter->lock, 0, 780364973Szec "Waiting recv finish", 1 * hz); 781364973Szec 782364973Szec /* This was a write so we are done; were interrupted, or timed out. */ 783364973Szec if (optype != SUME_MR_READ || error != 0 || error == EWOULDBLOCK) { 784364973Szec send->state = SUME_RIFFA_CHAN_STATE_IDLE; 785364973Szec if (optype == SUME_MR_READ) 786364973Szec error = EWOULDBLOCK; 787364973Szec else 788364973Szec error = 0; 789364973Szec } else 790364973Szec error = 0; 791364973Szec 792364973Szec /* 793364973Szec * For read requests we will update state once we are done 794364973Szec * having read the result to avoid any two outstanding 795364973Szec * transactions, or we need a queue and validate tags, 796364973Szec * which is a lot of work for a low priority, infrequent 797364973Szec * event. 798364973Szec */ 799364973Szec 800364973Szec SUME_UNLOCK(adapter); 801364973Szec 802364973Szec return (error); 803364973Szec} 804364973Szec 805364973Szec/* Module register read. */ 806364973Szecstatic int 807364973Szecsume_module_reg_read(struct nf_priv *nf_priv, struct sume_ifreq *sifr) 808364973Szec{ 809364973Szec struct sume_adapter *adapter = nf_priv->adapter; 810364973Szec struct riffa_chnl_dir *recv = adapter->recv[SUME_RIFFA_CHANNEL_REG]; 811364973Szec struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 812364973Szec struct nf_regop_data *data; 813364973Szec int error = 0; 814364973Szec 815364973Szec /* 816364973Szec * 0. Sleep waiting for result if needed (unless condition is 817364973Szec * true already). 818364973Szec * 1. Read DMA results. 819364973Szec * 2. Update state on *TX* to IDLE to allow next read to start. 820364973Szec */ 821364973Szec SUME_LOCK(adapter); 822364973Szec 823364973Szec bus_dmamap_sync(recv->ch_tag, recv->ch_map, 824364973Szec BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 825364973Szec /* 826364973Szec * We only need to be woken up at the end of the transaction. 827364973Szec * Timeout after 1s. 828364973Szec */ 829364973Szec if (recv->state != SUME_RIFFA_CHAN_STATE_READ) 830364973Szec error = msleep(&recv->event, &adapter->lock, 0, 831364973Szec "Waiting transaction finish", 1 * hz); 832364973Szec 833364973Szec if (recv->state != SUME_RIFFA_CHAN_STATE_READ || error == EWOULDBLOCK) { 834364973Szec SUME_UNLOCK(adapter); 835364973Szec device_printf(adapter->dev, "wait error: %d\n", error); 836364973Szec return (EWOULDBLOCK); 837364973Szec } 838364973Szec 839364973Szec bus_dmamap_sync(recv->ch_tag, recv->ch_map, 840364973Szec BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 841364973Szec 842364973Szec /* 843364973Szec * Read reply data and validate address and tag. 844364973Szec * Note: we do access the send side without lock but the state 845364973Szec * machine does prevent the data from changing. 846364973Szec */ 847364973Szec data = (struct nf_regop_data *) (recv->buf_addr + 848364973Szec sizeof(struct nf_bb_desc)); 849364973Szec 850364973Szec if (le32toh(data->rtag) != send->rtag) 851364973Szec device_printf(adapter->dev, "rtag error: 0x%08x 0x%08x\n", 852364973Szec le32toh(data->rtag), send->rtag); 853364973Szec 854364973Szec sifr->val = le32toh(data->val); 855364973Szec recv->state = SUME_RIFFA_CHAN_STATE_IDLE; 856364973Szec 857364973Szec /* We are done. */ 858364973Szec send->state = SUME_RIFFA_CHAN_STATE_IDLE; 859364973Szec 860364973Szec SUME_UNLOCK(adapter); 861364973Szec 862364973Szec return (0); 863364973Szec} 864364973Szec 865364973Szec/* Read value from a module register and return it to a sume_ifreq. */ 866364973Szecstatic int 867364973Szecget_modreg_value(struct nf_priv *nf_priv, struct sume_ifreq *sifr) 868364973Szec{ 869364973Szec int error; 870364973Szec 871364973Szec error = sume_module_reg_write(nf_priv, sifr, SUME_MR_READ); 872364973Szec if (!error) 873364973Szec error = sume_module_reg_read(nf_priv, sifr); 874364973Szec 875364973Szec return (error); 876364973Szec} 877364973Szec 878364973Szecstatic int 879364973Szecsume_if_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 880364973Szec{ 881364973Szec struct ifreq *ifr = (struct ifreq *) data; 882364973Szec struct nf_priv *nf_priv = ifp->if_softc; 883364973Szec struct sume_ifreq sifr; 884364973Szec int error = 0; 885364973Szec 886364973Szec switch (cmd) { 887364973Szec case SIOCGIFMEDIA: 888364973Szec case SIOCGIFXMEDIA: 889364973Szec error = ifmedia_ioctl(ifp, ifr, &nf_priv->media, cmd); 890364973Szec break; 891364973Szec 892364973Szec case SUME_IOCTL_CMD_WRITE_REG: 893364973Szec error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr)); 894364973Szec if (error) { 895364973Szec error = EINVAL; 896364973Szec break; 897364973Szec } 898364973Szec error = sume_module_reg_write(nf_priv, &sifr, SUME_MR_WRITE); 899364973Szec break; 900364973Szec 901364973Szec case SUME_IOCTL_CMD_READ_REG: 902364973Szec error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr)); 903364973Szec if (error) { 904364973Szec error = EINVAL; 905364973Szec break; 906364973Szec } 907364973Szec 908364973Szec error = get_modreg_value(nf_priv, &sifr); 909364973Szec if (error) 910364973Szec break; 911364973Szec 912364973Szec error = copyout(&sifr, ifr_data_get_ptr(ifr), sizeof(sifr)); 913364973Szec if (error) 914364973Szec error = EINVAL; 915364973Szec 916364973Szec break; 917364973Szec 918364973Szec case SIOCSIFFLAGS: 919364973Szec /* Silence tcpdump 'promisc mode not supported' warning. */ 920364973Szec if (ifp->if_flags & IFF_PROMISC) 921364973Szec break; 922364973Szec 923364973Szec default: 924364973Szec error = ether_ioctl(ifp, cmd, data); 925364973Szec break; 926364973Szec } 927364973Szec 928364973Szec return (error); 929364973Szec} 930364973Szec 931364973Szecstatic int 932364973Szecsume_media_change(struct ifnet *ifp) 933364973Szec{ 934364973Szec struct nf_priv *nf_priv = ifp->if_softc; 935364973Szec struct ifmedia *ifm = &nf_priv->media; 936364973Szec 937364973Szec if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 938364973Szec return (EINVAL); 939364973Szec 940364973Szec if (IFM_SUBTYPE(ifm->ifm_media) == IFM_10G_SR) 941364973Szec ifp->if_baudrate = ifmedia_baudrate(IFM_ETHER | IFM_10G_SR); 942364973Szec else 943364973Szec ifp->if_baudrate = ifmedia_baudrate(ifm->ifm_media); 944364973Szec 945364973Szec return (0); 946364973Szec} 947364973Szec 948364973Szecstatic void 949364973Szecsume_update_link_status(struct ifnet *ifp) 950364973Szec{ 951364973Szec struct nf_priv *nf_priv = ifp->if_softc; 952364973Szec struct sume_adapter *adapter = nf_priv->adapter; 953364973Szec struct sume_ifreq sifr; 954364973Szec int link_status; 955364973Szec 956364973Szec sifr.addr = SUME_STATUS_ADDR(nf_priv->port); 957364973Szec sifr.val = 0; 958364973Szec 959364973Szec if (get_modreg_value(nf_priv, &sifr)) 960364973Szec return; 961364973Szec 962364973Szec link_status = SUME_LINK_STATUS(sifr.val); 963364973Szec 964364973Szec if (!link_status && nf_priv->link_up) { 965364973Szec if_link_state_change(ifp, LINK_STATE_DOWN); 966364973Szec nf_priv->link_up = 0; 967364973Szec if (adapter->sume_debug) 968364973Szec device_printf(adapter->dev, "port %d link state " 969364973Szec "changed to DOWN\n", nf_priv->unit); 970364973Szec } else if (link_status && !nf_priv->link_up) { 971364973Szec nf_priv->link_up = 1; 972364973Szec if_link_state_change(ifp, LINK_STATE_UP); 973364973Szec if (adapter->sume_debug) 974364973Szec device_printf(adapter->dev, "port %d link state " 975364973Szec "changed to UP\n", nf_priv->unit); 976364973Szec } 977364973Szec} 978364973Szec 979364973Szecstatic void 980364973Szecsume_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 981364973Szec{ 982364973Szec struct nf_priv *nf_priv = ifp->if_softc; 983364973Szec struct ifmedia *ifm = &nf_priv->media; 984364973Szec 985364973Szec if (ifm->ifm_cur->ifm_media == (IFM_ETHER | IFM_10G_SR) && 986364973Szec (ifp->if_flags & IFF_UP)) 987364973Szec ifmr->ifm_active = IFM_ETHER | IFM_10G_SR; 988364973Szec else 989364973Szec ifmr->ifm_active = ifm->ifm_cur->ifm_media; 990364973Szec 991364973Szec ifmr->ifm_status |= IFM_AVALID; 992364973Szec 993364973Szec sume_update_link_status(ifp); 994364973Szec 995364973Szec if (nf_priv->link_up) 996364973Szec ifmr->ifm_status |= IFM_ACTIVE; 997364973Szec} 998364973Szec 999364973Szec/* 1000364973Szec * Packet to transmit. We take the packet data from the mbuf and copy it to the 1001364973Szec * bouncebuffer address buf_addr+3*sizeof(uint32_t)+16. The 16 bytes before the 1002364973Szec * packet data are for metadata: sport/dport (depending on our source 1003364973Szec * interface), packet length and magic 0xcafe. We tell the SUME about the 1004364973Szec * transfer, fill the first 3*sizeof(uint32_t) bytes of the bouncebuffer with 1005364973Szec * the information about the start and length of the packet and trigger the 1006364973Szec * transaction. 1007364973Szec */ 1008364973Szecstatic int 1009364973Szecsume_if_start_locked(struct ifnet *ifp) 1010364973Szec{ 1011364973Szec struct mbuf *m; 1012364973Szec struct nf_priv *nf_priv = ifp->if_softc; 1013364973Szec struct sume_adapter *adapter = nf_priv->adapter; 1014364973Szec struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_DATA]; 1015364973Szec uint8_t *outbuf; 1016364973Szec struct nf_metadata *mdata; 1017364973Szec int plen = SUME_MIN_PKT_SIZE; 1018364973Szec 1019364973Szec KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned")); 1020364973Szec KASSERT(send->state == SUME_RIFFA_CHAN_STATE_IDLE, 1021364973Szec ("SUME not in IDLE state")); 1022364973Szec 1023364973Szec IFQ_DEQUEUE(&ifp->if_snd, m); 1024364973Szec if (m == NULL) 1025364973Szec return (EINVAL); 1026364973Szec 1027364973Szec /* Packets large enough do not need to be padded */ 1028364973Szec if (m->m_pkthdr.len > SUME_MIN_PKT_SIZE) 1029364973Szec plen = m->m_pkthdr.len; 1030364973Szec 1031364973Szec if (adapter->sume_debug) 1032364973Szec device_printf(adapter->dev, "sending %d bytes to %s%d\n", plen, 1033364973Szec SUME_ETH_DEVICE_NAME, nf_priv->unit); 1034364973Szec 1035364973Szec outbuf = (uint8_t *) send->buf_addr + sizeof(struct nf_bb_desc); 1036364973Szec mdata = (struct nf_metadata *) outbuf; 1037364973Szec 1038364973Szec /* Clear the recovery flag. */ 1039364973Szec send->recovery = 0; 1040364973Szec 1041364973Szec /* Make sure we fit with the 16 bytes nf_metadata. */ 1042364973Szec if (m->m_pkthdr.len + sizeof(struct nf_metadata) > 1043364973Szec adapter->sg_buf_size) { 1044364973Szec device_printf(adapter->dev, "packet too big for bounce buffer " 1045364973Szec "(%d)\n", m->m_pkthdr.len); 1046364973Szec m_freem(m); 1047364973Szec nf_priv->stats.tx_dropped++; 1048364973Szec return (ENOMEM); 1049364973Szec } 1050364973Szec 1051364973Szec bus_dmamap_sync(send->ch_tag, send->ch_map, 1052364973Szec BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1053364973Szec 1054364973Szec /* Zero out the padded data */ 1055364973Szec if (m->m_pkthdr.len < SUME_MIN_PKT_SIZE) 1056364973Szec bzero(outbuf + sizeof(struct nf_metadata), SUME_MIN_PKT_SIZE); 1057364973Szec /* Skip the first 16 bytes for the metadata. */ 1058364973Szec m_copydata(m, 0, m->m_pkthdr.len, outbuf + sizeof(struct nf_metadata)); 1059364973Szec send->len = (sizeof(struct nf_metadata) + plen + 3) / 4; 1060364973Szec 1061364973Szec /* Fill in the metadata: CPU(DMA) ports are odd, MAC ports are even. */ 1062364973Szec mdata->sport = htole16(1 << (nf_priv->port * 2 + 1)); 1063364973Szec mdata->dport = htole16(1 << (nf_priv->port * 2)); 1064364973Szec mdata->plen = htole16(plen); 1065364973Szec mdata->magic = htole16(SUME_RIFFA_MAGIC); 1066364973Szec mdata->t1 = htole32(0); 1067364973Szec mdata->t2 = htole32(0); 1068364973Szec 1069364973Szec /* Let the FPGA know about the transfer. */ 1070364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1071364973Szec RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST); 1072364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1073364973Szec RIFFA_RX_LEN_REG_OFF), send->len); 1074364973Szec 1075364973Szec /* Fill the bouncebuf "descriptor". */ 1076364973Szec sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len)); 1077364973Szec 1078364973Szec /* Update the state before intiating the DMA to avoid races. */ 1079364973Szec send->state = SUME_RIFFA_CHAN_STATE_READY; 1080364973Szec 1081364973Szec /* DMA. */ 1082364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1083364973Szec RIFFA_RX_SG_ADDR_LO_REG_OFF), 1084364973Szec SUME_RIFFA_LO_ADDR(send->buf_hw_addr)); 1085364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1086364973Szec RIFFA_RX_SG_ADDR_HI_REG_OFF), 1087364973Szec SUME_RIFFA_HI_ADDR(send->buf_hw_addr)); 1088364973Szec write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1089364973Szec RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg); 1090364973Szec 1091364973Szec bus_dmamap_sync(send->ch_tag, send->ch_map, 1092364973Szec BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1093364973Szec 1094364973Szec nf_priv->stats.tx_packets++; 1095364973Szec nf_priv->stats.tx_bytes += plen; 1096364973Szec 1097364973Szec /* We can free as long as we use the bounce buffer. */ 1098364973Szec m_freem(m); 1099364973Szec 1100364973Szec adapter->last_ifc = nf_priv->port; 1101364973Szec 1102364973Szec /* Reset watchdog counter. */ 1103364973Szec adapter->wd_counter = 0; 1104364973Szec 1105364973Szec return (0); 1106364973Szec} 1107364973Szec 1108364973Szecstatic void 1109364973Szecsume_if_start(struct ifnet *ifp) 1110364973Szec{ 1111364973Szec struct nf_priv *nf_priv = ifp->if_softc; 1112364973Szec struct sume_adapter *adapter = nf_priv->adapter; 1113364973Szec 1114364973Szec if (!adapter->running || !(ifp->if_flags & IFF_UP)) 1115364973Szec return; 1116364973Szec 1117364973Szec SUME_LOCK(adapter); 1118364973Szec if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state == 1119364973Szec SUME_RIFFA_CHAN_STATE_IDLE) 1120364973Szec sume_if_start_locked(ifp); 1121364973Szec SUME_UNLOCK(adapter); 1122364973Szec} 1123364973Szec 1124364973Szec/* 1125364973Szec * We call this function at the end of every TX transaction to check for 1126364973Szec * remaining packets in the TX queues for every UP interface. 1127364973Szec */ 1128364973Szecstatic void 1129364973Szeccheck_tx_queues(struct sume_adapter *adapter) 1130364973Szec{ 1131364973Szec int i, last_ifc; 1132364973Szec 1133364973Szec KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned")); 1134364973Szec 1135364973Szec last_ifc = adapter->last_ifc; 1136364973Szec 1137364973Szec /* Check all interfaces */ 1138364973Szec for (i = last_ifc + 1; i < last_ifc + SUME_NPORTS + 1; i++) { 1139364973Szec struct ifnet *ifp = adapter->ifp[i % SUME_NPORTS]; 1140364973Szec 1141364973Szec if (!(ifp->if_flags & IFF_UP)) 1142364973Szec continue; 1143364973Szec 1144364973Szec if (!sume_if_start_locked(ifp)) 1145364973Szec break; 1146364973Szec } 1147364973Szec} 1148364973Szec 1149364973Szecstatic int 1150364973Szecsume_ifp_alloc(struct sume_adapter *adapter, uint32_t port) 1151364973Szec{ 1152364973Szec struct ifnet *ifp; 1153364973Szec struct nf_priv *nf_priv = malloc(sizeof(struct nf_priv), M_SUME, 1154364973Szec M_ZERO | M_WAITOK); 1155364973Szec 1156364973Szec ifp = if_alloc(IFT_ETHER); 1157364973Szec if (ifp == NULL) { 1158364973Szec device_printf(adapter->dev, "cannot allocate ifnet\n"); 1159364973Szec return (ENOMEM); 1160364973Szec } 1161364973Szec 1162364973Szec adapter->ifp[port] = ifp; 1163364973Szec ifp->if_softc = nf_priv; 1164364973Szec 1165364973Szec nf_priv->adapter = adapter; 1166364973Szec nf_priv->unit = alloc_unr(unr); 1167364973Szec nf_priv->port = port; 1168364973Szec nf_priv->link_up = 0; 1169364973Szec 1170364973Szec if_initname(ifp, SUME_ETH_DEVICE_NAME, nf_priv->unit); 1171364973Szec ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1172364973Szec 1173364973Szec ifp->if_init = sume_if_init; 1174364973Szec ifp->if_start = sume_if_start; 1175364973Szec ifp->if_ioctl = sume_if_ioctl; 1176364973Szec 1177364973Szec uint8_t hw_addr[ETHER_ADDR_LEN] = DEFAULT_ETHER_ADDRESS; 1178364973Szec hw_addr[ETHER_ADDR_LEN-1] = nf_priv->unit; 1179364973Szec ether_ifattach(ifp, hw_addr); 1180364973Szec 1181364973Szec ifmedia_init(&nf_priv->media, IFM_IMASK, sume_media_change, 1182364973Szec sume_media_status); 1183364973Szec ifmedia_add(&nf_priv->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1184364973Szec ifmedia_set(&nf_priv->media, IFM_ETHER | IFM_10G_SR); 1185364973Szec 1186364973Szec ifp->if_drv_flags |= IFF_DRV_RUNNING; 1187364973Szec 1188364973Szec return (0); 1189364973Szec} 1190364973Szec 1191364973Szecstatic void 1192364973Szeccallback_dma(void *arg, bus_dma_segment_t *segs, int nseg, int err) 1193364973Szec{ 1194364973Szec if (err) 1195364973Szec return; 1196364973Szec 1197364973Szec KASSERT(nseg == 1, ("%d segments returned!", nseg)); 1198364973Szec 1199364973Szec *(bus_addr_t *) arg = segs[0].ds_addr; 1200364973Szec} 1201364973Szec 1202364973Szecstatic int 1203364973Szecsume_probe_riffa_buffer(const struct sume_adapter *adapter, 1204364973Szec struct riffa_chnl_dir ***p, const char *dir) 1205364973Szec{ 1206364973Szec struct riffa_chnl_dir **rp; 1207364973Szec bus_addr_t hw_addr; 1208364973Szec int error, ch; 1209364973Szec device_t dev = adapter->dev; 1210364973Szec 1211364973Szec error = ENOMEM; 1212364973Szec *p = malloc(SUME_RIFFA_CHANNELS * sizeof(struct riffa_chnl_dir *), 1213364973Szec M_SUME, M_ZERO | M_WAITOK); 1214364973Szec if (*p == NULL) { 1215364973Szec device_printf(dev, "malloc(%s) failed.\n", dir); 1216364973Szec return (error); 1217364973Szec } 1218364973Szec 1219364973Szec rp = *p; 1220364973Szec /* Allocate the chnl_dir structs themselves. */ 1221364973Szec for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 1222364973Szec /* One direction. */ 1223364973Szec rp[ch] = malloc(sizeof(struct riffa_chnl_dir), M_SUME, 1224364973Szec M_ZERO | M_WAITOK); 1225364973Szec if (rp[ch] == NULL) { 1226364973Szec device_printf(dev, "malloc(%s[%d]) riffa_chnl_dir " 1227364973Szec "failed.\n", dir, ch); 1228364973Szec return (error); 1229364973Szec } 1230364973Szec 1231364973Szec int err = bus_dma_tag_create(bus_get_dma_tag(dev), 1232364973Szec 4, 0, 1233364973Szec BUS_SPACE_MAXADDR, 1234364973Szec BUS_SPACE_MAXADDR, 1235364973Szec NULL, NULL, 1236364973Szec adapter->sg_buf_size, 1237364973Szec 1, 1238364973Szec adapter->sg_buf_size, 1239364973Szec 0, 1240364973Szec NULL, 1241364973Szec NULL, 1242364973Szec &rp[ch]->ch_tag); 1243364973Szec 1244364973Szec if (err) { 1245364973Szec device_printf(dev, "bus_dma_tag_create(%s[%d]) " 1246364973Szec "failed.\n", dir, ch); 1247364973Szec return (err); 1248364973Szec } 1249364973Szec 1250364973Szec err = bus_dmamem_alloc(rp[ch]->ch_tag, (void **) 1251364973Szec &rp[ch]->buf_addr, BUS_DMA_WAITOK | BUS_DMA_COHERENT | 1252364973Szec BUS_DMA_ZERO, &rp[ch]->ch_map); 1253364973Szec if (err) { 1254364973Szec device_printf(dev, "bus_dmamem_alloc(%s[%d]) failed.\n", 1255364973Szec dir, ch); 1256364973Szec return (err); 1257364973Szec } 1258364973Szec 1259364973Szec bzero(rp[ch]->buf_addr, adapter->sg_buf_size); 1260364973Szec 1261364973Szec err = bus_dmamap_load(rp[ch]->ch_tag, rp[ch]->ch_map, 1262364973Szec rp[ch]->buf_addr, adapter->sg_buf_size, callback_dma, 1263364973Szec &hw_addr, BUS_DMA_NOWAIT); 1264364973Szec if (err) { 1265364973Szec device_printf(dev, "bus_dmamap_load(%s[%d]) failed.\n", 1266364973Szec dir, ch); 1267364973Szec return (err); 1268364973Szec } 1269364973Szec rp[ch]->buf_hw_addr = hw_addr; 1270364973Szec rp[ch]->num_sg = 1; 1271364973Szec rp[ch]->state = SUME_RIFFA_CHAN_STATE_IDLE; 1272364973Szec 1273364973Szec rp[ch]->rtag = SUME_INIT_RTAG; 1274364973Szec } 1275364973Szec 1276364973Szec return (0); 1277364973Szec} 1278364973Szec 1279364973Szecstatic int 1280364973Szecsume_probe_riffa_buffers(struct sume_adapter *adapter) 1281364973Szec{ 1282364973Szec int error; 1283364973Szec 1284364973Szec error = sume_probe_riffa_buffer(adapter, &adapter->recv, "recv"); 1285364973Szec if (error) 1286364973Szec return (error); 1287364973Szec 1288364973Szec error = sume_probe_riffa_buffer(adapter, &adapter->send, "send"); 1289364973Szec 1290364973Szec return (error); 1291364973Szec} 1292364973Szec 1293364973Szecstatic void 1294364973Szecsume_sysctl_init(struct sume_adapter *adapter) 1295364973Szec{ 1296364973Szec device_t dev = adapter->dev; 1297364973Szec struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 1298364973Szec struct sysctl_oid *tree = device_get_sysctl_tree(dev); 1299364973Szec struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 1300364973Szec struct sysctl_oid *tmp_tree; 1301364973Szec char namebuf[MAX_IFC_NAME_LEN]; 1302364973Szec int i; 1303364973Szec 1304364973Szec tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "sume", CTLFLAG_RW, 1305364973Szec 0, "SUME top-level tree"); 1306364973Szec if (tree == NULL) { 1307364973Szec device_printf(dev, "SYSCTL_ADD_NODE failed.\n"); 1308364973Szec return; 1309364973Szec } 1310364973Szec SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug", CTLFLAG_RW, 1311364973Szec &adapter->sume_debug, 0, "debug int leaf"); 1312364973Szec 1313364973Szec /* total RX error stats */ 1314364973Szec SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_epkts", 1315364973Szec CTLFLAG_RD, &adapter->packets_err, 0, "rx errors"); 1316364973Szec SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_ebytes", 1317364973Szec CTLFLAG_RD, &adapter->bytes_err, 0, "rx error bytes"); 1318364973Szec 1319364973Szec for (i = SUME_NPORTS - 1; i >= 0; i--) { 1320364973Szec struct ifnet *ifp = adapter->ifp[i]; 1321364973Szec if (ifp == NULL) 1322364973Szec continue; 1323364973Szec 1324364973Szec struct nf_priv *nf_priv = ifp->if_softc; 1325364973Szec 1326364973Szec snprintf(namebuf, MAX_IFC_NAME_LEN, "%s%d", 1327364973Szec SUME_ETH_DEVICE_NAME, nf_priv->unit); 1328364973Szec tmp_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 1329364973Szec CTLFLAG_RW, 0, "SUME ifc tree"); 1330364973Szec if (tmp_tree == NULL) { 1331364973Szec device_printf(dev, "SYSCTL_ADD_NODE failed.\n"); 1332364973Szec return; 1333364973Szec } 1334364973Szec 1335364973Szec /* Packets dropped by down interface. */ 1336364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1337364973Szec "ifc_down_bytes", CTLFLAG_RD, 1338364973Szec &nf_priv->stats.ifc_down_bytes, 0, "ifc_down bytes"); 1339364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1340364973Szec "ifc_down_packets", CTLFLAG_RD, 1341364973Szec &nf_priv->stats.ifc_down_packets, 0, "ifc_down packets"); 1342364973Szec 1343364973Szec /* HW RX stats */ 1344364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1345364973Szec "hw_rx_packets", CTLFLAG_RD, &nf_priv->stats.hw_rx_packets, 1346364973Szec 0, "hw_rx packets"); 1347364973Szec 1348364973Szec /* HW TX stats */ 1349364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1350364973Szec "hw_tx_packets", CTLFLAG_RD, &nf_priv->stats.hw_tx_packets, 1351364973Szec 0, "hw_tx packets"); 1352364973Szec 1353364973Szec /* RX stats */ 1354364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1355364973Szec "rx_bytes", CTLFLAG_RD, &nf_priv->stats.rx_bytes, 0, 1356364973Szec "rx bytes"); 1357364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1358364973Szec "rx_dropped", CTLFLAG_RD, &nf_priv->stats.rx_dropped, 0, 1359364973Szec "rx dropped"); 1360364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1361364973Szec "rx_packets", CTLFLAG_RD, &nf_priv->stats.rx_packets, 0, 1362364973Szec "rx packets"); 1363364973Szec 1364364973Szec /* TX stats */ 1365364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1366364973Szec "tx_bytes", CTLFLAG_RD, &nf_priv->stats.tx_bytes, 0, 1367364973Szec "tx bytes"); 1368364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1369364973Szec "tx_dropped", CTLFLAG_RD, &nf_priv->stats.tx_dropped, 0, 1370364973Szec "tx dropped"); 1371364973Szec SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1372364973Szec "tx_packets", CTLFLAG_RD, &nf_priv->stats.tx_packets, 0, 1373364973Szec "tx packets"); 1374364973Szec } 1375364973Szec} 1376364973Szec 1377364973Szecstatic void 1378364973Szecsume_local_timer(void *arg) 1379364973Szec{ 1380364973Szec struct sume_adapter *adapter = arg; 1381364973Szec 1382364973Szec if (!adapter->running) 1383364973Szec return; 1384364973Szec 1385364973Szec taskqueue_enqueue(adapter->tq, &adapter->stat_task); 1386364973Szec 1387364973Szec SUME_LOCK(adapter); 1388364973Szec if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state != 1389364973Szec SUME_RIFFA_CHAN_STATE_IDLE && ++adapter->wd_counter >= 3) { 1390364973Szec /* Resetting interfaces if stuck for 3 seconds. */ 1391364973Szec device_printf(adapter->dev, "TX stuck, resetting adapter.\n"); 1392364973Szec read_reg(adapter, RIFFA_INFO_REG_OFF); 1393364973Szec 1394364973Szec adapter->send[SUME_RIFFA_CHANNEL_DATA]->state = 1395364973Szec SUME_RIFFA_CHAN_STATE_IDLE; 1396364973Szec adapter->wd_counter = 0; 1397364973Szec 1398364973Szec check_tx_queues(adapter); 1399364973Szec } 1400364973Szec SUME_UNLOCK(adapter); 1401364973Szec 1402364973Szec callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter); 1403364973Szec} 1404364973Szec 1405364973Szecstatic void 1406364973Szecsume_get_stats(void *context, int pending) 1407364973Szec{ 1408364973Szec struct sume_adapter *adapter = context; 1409364973Szec int i; 1410364973Szec 1411364973Szec for (i = 0; i < SUME_NPORTS; i++) { 1412364973Szec struct ifnet *ifp = adapter->ifp[i]; 1413364973Szec 1414364973Szec if (ifp->if_flags & IFF_UP) { 1415364973Szec struct nf_priv *nf_priv = ifp->if_softc; 1416364973Szec struct sume_ifreq sifr; 1417364973Szec 1418364973Szec sume_update_link_status(ifp); 1419364973Szec 1420364973Szec /* Get RX counter. */ 1421364973Szec sifr.addr = SUME_STAT_RX_ADDR(nf_priv->port); 1422364973Szec sifr.val = 0; 1423364973Szec 1424364973Szec if (!get_modreg_value(nf_priv, &sifr)) 1425364973Szec nf_priv->stats.hw_rx_packets += sifr.val; 1426364973Szec 1427364973Szec /* Get TX counter. */ 1428364973Szec sifr.addr = SUME_STAT_TX_ADDR(nf_priv->port); 1429364973Szec sifr.val = 0; 1430364973Szec 1431364973Szec if (!get_modreg_value(nf_priv, &sifr)) 1432364973Szec nf_priv->stats.hw_tx_packets += sifr.val; 1433364973Szec } 1434364973Szec } 1435364973Szec} 1436364973Szec 1437364973Szecstatic int 1438364973Szecsume_attach(device_t dev) 1439364973Szec{ 1440364973Szec struct sume_adapter *adapter = device_get_softc(dev); 1441364973Szec adapter->dev = dev; 1442364973Szec int error, i; 1443364973Szec 1444364973Szec mtx_init(&adapter->lock, "Global lock", NULL, MTX_DEF); 1445364973Szec 1446364973Szec adapter->running = 0; 1447364973Szec 1448364973Szec /* OK finish up RIFFA. */ 1449364973Szec error = sume_probe_riffa_pci(adapter); 1450364973Szec if (error != 0) 1451364973Szec goto error; 1452364973Szec 1453364973Szec error = sume_probe_riffa_buffers(adapter); 1454364973Szec if (error != 0) 1455364973Szec goto error; 1456364973Szec 1457364973Szec /* Now do the network interfaces. */ 1458364973Szec for (i = 0; i < SUME_NPORTS; i++) { 1459364973Szec error = sume_ifp_alloc(adapter, i); 1460364973Szec if (error != 0) 1461364973Szec goto error; 1462364973Szec } 1463364973Szec 1464364973Szec /* Register stats and register sysctls. */ 1465364973Szec sume_sysctl_init(adapter); 1466364973Szec 1467364973Szec /* Reset the HW. */ 1468364973Szec read_reg(adapter, RIFFA_INFO_REG_OFF); 1469364973Szec 1470364973Szec /* Ready to go, "enable" IRQ. */ 1471364973Szec adapter->running = 1; 1472364973Szec 1473364973Szec callout_init(&adapter->timer, 1); 1474364973Szec TASK_INIT(&adapter->stat_task, 0, sume_get_stats, adapter); 1475364973Szec 1476364973Szec adapter->tq = taskqueue_create("sume_stats", M_NOWAIT, 1477364973Szec taskqueue_thread_enqueue, &adapter->tq); 1478364973Szec taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s stattaskq", 1479364973Szec device_get_nameunit(adapter->dev)); 1480364973Szec 1481364973Szec callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter); 1482364973Szec 1483364973Szec return (0); 1484364973Szec 1485364973Szecerror: 1486364973Szec sume_detach(dev); 1487364973Szec 1488364973Szec return (error); 1489364973Szec} 1490364973Szec 1491364973Szecstatic void 1492364973Szecsume_remove_riffa_buffer(const struct sume_adapter *adapter, 1493364973Szec struct riffa_chnl_dir **pp) 1494364973Szec{ 1495364973Szec int ch; 1496364973Szec 1497364973Szec for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 1498364973Szec if (pp[ch] == NULL) 1499364973Szec continue; 1500364973Szec 1501364973Szec if (pp[ch]->buf_hw_addr != 0) { 1502364973Szec bus_dmamem_free(pp[ch]->ch_tag, pp[ch]->buf_addr, 1503364973Szec pp[ch]->ch_map); 1504364973Szec pp[ch]->buf_hw_addr = 0; 1505364973Szec } 1506364973Szec 1507364973Szec free(pp[ch], M_SUME); 1508364973Szec } 1509364973Szec} 1510364973Szec 1511364973Szecstatic void 1512364973Szecsume_remove_riffa_buffers(struct sume_adapter *adapter) 1513364973Szec{ 1514364973Szec if (adapter->send != NULL) { 1515364973Szec sume_remove_riffa_buffer(adapter, adapter->send); 1516364973Szec free(adapter->send, M_SUME); 1517364973Szec adapter->send = NULL; 1518364973Szec } 1519364973Szec if (adapter->recv != NULL) { 1520364973Szec sume_remove_riffa_buffer(adapter, adapter->recv); 1521364973Szec free(adapter->recv, M_SUME); 1522364973Szec adapter->recv = NULL; 1523364973Szec } 1524364973Szec} 1525364973Szec 1526364973Szecstatic int 1527364973Szecsume_detach(device_t dev) 1528364973Szec{ 1529364973Szec struct sume_adapter *adapter = device_get_softc(dev); 1530364973Szec int i; 1531364973Szec struct nf_priv *nf_priv; 1532364973Szec 1533364973Szec KASSERT(mtx_initialized(&adapter->lock), ("SUME mutex not " 1534364973Szec "initialized")); 1535364973Szec adapter->running = 0; 1536364973Szec 1537364973Szec /* Drain the stats callout and task queue. */ 1538364973Szec callout_drain(&adapter->timer); 1539364973Szec 1540364973Szec if (adapter->tq) { 1541364973Szec taskqueue_drain(adapter->tq, &adapter->stat_task); 1542364973Szec taskqueue_free(adapter->tq); 1543364973Szec } 1544364973Szec 1545364973Szec for (i = 0; i < SUME_NPORTS; i++) { 1546364973Szec struct ifnet *ifp = adapter->ifp[i]; 1547364973Szec if (ifp == NULL) 1548364973Szec continue; 1549364973Szec 1550364973Szec ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1551364973Szec nf_priv = ifp->if_softc; 1552364973Szec 1553364973Szec if (ifp->if_flags & IFF_UP) 1554364973Szec if_down(ifp); 1555364973Szec ifmedia_removeall(&nf_priv->media); 1556364973Szec free_unr(unr, nf_priv->unit); 1557364973Szec 1558364973Szec ifp->if_flags &= ~IFF_UP; 1559364973Szec ether_ifdetach(ifp); 1560364973Szec if_free(ifp); 1561364973Szec 1562364973Szec free(nf_priv, M_SUME); 1563364973Szec } 1564364973Szec 1565364973Szec sume_remove_riffa_buffers(adapter); 1566364973Szec 1567364973Szec if (adapter->irq.tag) 1568364973Szec bus_teardown_intr(dev, adapter->irq.res, adapter->irq.tag); 1569364973Szec if (adapter->irq.res) 1570364973Szec bus_release_resource(dev, SYS_RES_IRQ, adapter->irq.rid, 1571364973Szec adapter->irq.res); 1572364973Szec 1573364973Szec pci_release_msi(dev); 1574364973Szec 1575364973Szec if (adapter->bar0_addr) 1576364973Szec bus_release_resource(dev, SYS_RES_MEMORY, adapter->rid, 1577364973Szec adapter->bar0_addr); 1578364973Szec 1579364973Szec mtx_destroy(&adapter->lock); 1580364973Szec 1581364973Szec return (0); 1582364973Szec} 1583364973Szec 1584364973Szecstatic int 1585364973Szecmod_event(module_t mod, int cmd, void *arg) 1586364973Szec{ 1587364973Szec switch (cmd) { 1588364973Szec case MOD_LOAD: 1589364973Szec unr = new_unrhdr(0, INT_MAX, NULL); 1590364973Szec break; 1591364973Szec 1592364973Szec case MOD_UNLOAD: 1593364973Szec delete_unrhdr(unr); 1594364973Szec break; 1595364973Szec } 1596364973Szec 1597364973Szec return (0); 1598364973Szec} 1599364973Szecstatic devclass_t sume_devclass; 1600364973Szec 1601364973SzecDRIVER_MODULE(sume, pci, sume_driver, sume_devclass, mod_event, 0); 1602364973SzecMODULE_VERSION(sume, 1); 1603