1159571Sgallatin/****************************************************************************** 2155852Sgallatin 3247160SgallatinCopyright (c) 2006-2013, Myricom Inc. 4155852SgallatinAll rights reserved. 5155852Sgallatin 6155852SgallatinRedistribution and use in source and binary forms, with or without 7155852Sgallatinmodification, are permitted provided that the following conditions are met: 8155852Sgallatin 9155852Sgallatin 1. Redistributions of source code must retain the above copyright notice, 10155852Sgallatin this list of conditions and the following disclaimer. 11155852Sgallatin 12171405Sgallatin 2. Neither the name of the Myricom Inc, nor the names of its 13155852Sgallatin contributors may be used to endorse or promote products derived from 14155852Sgallatin this software without specific prior written permission. 15155852Sgallatin 16155852SgallatinTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17155852SgallatinAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18155852SgallatinIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19155852SgallatinARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20155852SgallatinLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21155852SgallatinCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22155852SgallatinSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23155852SgallatinINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24155852SgallatinCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25155852SgallatinARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26155852SgallatinPOSSIBILITY OF SUCH DAMAGE. 27155852Sgallatin 28155852Sgallatin***************************************************************************/ 29155852Sgallatin 30155852Sgallatin#include <sys/cdefs.h> 31155852Sgallatin__FBSDID("$FreeBSD: stable/10/sys/dev/mxge/if_mxge.c 329834 2018-02-22 19:40:03Z rpokala $"); 32155852Sgallatin 33155852Sgallatin#include <sys/param.h> 34155852Sgallatin#include <sys/systm.h> 35155852Sgallatin#include <sys/linker.h> 36155852Sgallatin#include <sys/firmware.h> 37155852Sgallatin#include <sys/endian.h> 38155852Sgallatin#include <sys/sockio.h> 39155852Sgallatin#include <sys/mbuf.h> 40155852Sgallatin#include <sys/malloc.h> 41155852Sgallatin#include <sys/kdb.h> 42155852Sgallatin#include <sys/kernel.h> 43168191Sjhb#include <sys/lock.h> 44155852Sgallatin#include <sys/module.h> 45155852Sgallatin#include <sys/socket.h> 46155852Sgallatin#include <sys/sysctl.h> 47155852Sgallatin#include <sys/sx.h> 48198250Sgallatin#include <sys/taskqueue.h> 49155852Sgallatin 50155852Sgallatin#include <net/if.h> 51155852Sgallatin#include <net/if_arp.h> 52155852Sgallatin#include <net/ethernet.h> 53155852Sgallatin#include <net/if_dl.h> 54155852Sgallatin#include <net/if_media.h> 55155852Sgallatin 56155852Sgallatin#include <net/bpf.h> 57155852Sgallatin 58155852Sgallatin#include <net/if_types.h> 59155852Sgallatin#include <net/if_vlan_var.h> 60155852Sgallatin#include <net/zlib.h> 61155852Sgallatin 62155852Sgallatin#include <netinet/in_systm.h> 63155852Sgallatin#include <netinet/in.h> 64155852Sgallatin#include <netinet/ip.h> 65247011Sgallatin#include <netinet/ip6.h> 66162322Sgallatin#include <netinet/tcp.h> 67247133Sgallatin#include <netinet/tcp_lro.h> 68247011Sgallatin#include <netinet6/ip6_var.h> 69155852Sgallatin 70155852Sgallatin#include <machine/bus.h> 71169840Sgallatin#include <machine/in_cksum.h> 72155852Sgallatin#include <machine/resource.h> 73155852Sgallatin#include <sys/bus.h> 74155852Sgallatin#include <sys/rman.h> 75175365Sgallatin#include <sys/smp.h> 76155852Sgallatin 77155852Sgallatin#include <dev/pci/pcireg.h> 78155852Sgallatin#include <dev/pci/pcivar.h> 79180567Sgallatin#include <dev/pci/pci_private.h> /* XXX for pci_cfg_restore */ 80155852Sgallatin 81155852Sgallatin#include <vm/vm.h> /* for pmap_mapdev() */ 82155852Sgallatin#include <vm/pmap.h> 83155852Sgallatin 84170330Sgallatin#if defined(__i386) || defined(__amd64) 85170330Sgallatin#include <machine/specialreg.h> 86170330Sgallatin#endif 87170330Sgallatin 88159571Sgallatin#include <dev/mxge/mxge_mcp.h> 89159571Sgallatin#include <dev/mxge/mcp_gen_header.h> 90175365Sgallatin/*#define MXGE_FAKE_IFP*/ 91159571Sgallatin#include <dev/mxge/if_mxge_var.h> 92193311Sgallatin#ifdef IFNET_BUF_RING 93193311Sgallatin#include <sys/buf_ring.h> 94193311Sgallatin#endif 95155852Sgallatin 96194743Sgallatin#include "opt_inet.h" 97247011Sgallatin#include "opt_inet6.h" 98194743Sgallatin 99155852Sgallatin/* tunable params */ 100159571Sgallatinstatic int mxge_nvidia_ecrc_enable = 1; 101164513Sgallatinstatic int mxge_force_firmware = 0; 102159571Sgallatinstatic int mxge_intr_coal_delay = 30; 103159612Sgallatinstatic int mxge_deassert_wait = 1; 104159571Sgallatinstatic int mxge_flow_control = 1; 105159612Sgallatinstatic int mxge_verbose = 0; 106166373Sgallatinstatic int mxge_ticks; 107175365Sgallatinstatic int mxge_max_slices = 1; 108202121Sgallatinstatic int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 109175365Sgallatinstatic int mxge_always_promisc = 0; 110194836Sgallatinstatic int mxge_initial_mtu = ETHERMTU_JUMBO; 111197391Sgallatinstatic int mxge_throttle = 0; 112159571Sgallatinstatic char *mxge_fw_unaligned = "mxge_ethp_z8e"; 113159571Sgallatinstatic char *mxge_fw_aligned = "mxge_eth_z8e"; 114175365Sgallatinstatic char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e"; 115175365Sgallatinstatic char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e"; 116155852Sgallatin 117159571Sgallatinstatic int mxge_probe(device_t dev); 118159571Sgallatinstatic int mxge_attach(device_t dev); 119159571Sgallatinstatic int mxge_detach(device_t dev); 120159571Sgallatinstatic int mxge_shutdown(device_t dev); 121159571Sgallatinstatic void mxge_intr(void *arg); 122155852Sgallatin 123159571Sgallatinstatic device_method_t mxge_methods[] = 124155852Sgallatin{ 125155852Sgallatin /* Device interface */ 126159571Sgallatin DEVMETHOD(device_probe, mxge_probe), 127159571Sgallatin DEVMETHOD(device_attach, mxge_attach), 128159571Sgallatin DEVMETHOD(device_detach, mxge_detach), 129159571Sgallatin DEVMETHOD(device_shutdown, mxge_shutdown), 130246128Ssbz 131246128Ssbz DEVMETHOD_END 132155852Sgallatin}; 133155852Sgallatin 134159571Sgallatinstatic driver_t mxge_driver = 135155852Sgallatin{ 136159571Sgallatin "mxge", 137159571Sgallatin mxge_methods, 138159571Sgallatin sizeof(mxge_softc_t), 139155852Sgallatin}; 140155852Sgallatin 141159571Sgallatinstatic devclass_t mxge_devclass; 142155852Sgallatin 143155852Sgallatin/* Declare ourselves to be a child of the PCI bus.*/ 144159571SgallatinDRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0); 145159571SgallatinMODULE_DEPEND(mxge, firmware, 1, 1, 1); 146171500SgallatinMODULE_DEPEND(mxge, zlib, 1, 1, 1); 147155852Sgallatin 148175365Sgallatinstatic int mxge_load_firmware(mxge_softc_t *sc, int adopt); 149169376Sgallatinstatic int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data); 150197395Sgallatinstatic int mxge_close(mxge_softc_t *sc, int down); 151170559Sgallatinstatic int mxge_open(mxge_softc_t *sc); 152170559Sgallatinstatic void mxge_tick(void *arg); 153169376Sgallatin 154155852Sgallatinstatic int 155159571Sgallatinmxge_probe(device_t dev) 156155852Sgallatin{ 157188736Sgallatin int rev; 158188736Sgallatin 159188736Sgallatin 160188736Sgallatin if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) && 161188736Sgallatin ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) || 162188736Sgallatin (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) { 163188736Sgallatin rev = pci_get_revid(dev); 164188736Sgallatin switch (rev) { 165188736Sgallatin case MXGE_PCI_REV_Z8E: 166188736Sgallatin device_set_desc(dev, "Myri10G-PCIE-8A"); 167188736Sgallatin break; 168188736Sgallatin case MXGE_PCI_REV_Z8ES: 169188736Sgallatin device_set_desc(dev, "Myri10G-PCIE-8B"); 170188736Sgallatin break; 171188736Sgallatin default: 172188736Sgallatin device_set_desc(dev, "Myri10G-PCIE-8??"); 173188736Sgallatin device_printf(dev, "Unrecognized rev %d NIC\n", 174188736Sgallatin rev); 175188736Sgallatin break; 176188736Sgallatin } 177188736Sgallatin return 0; 178188736Sgallatin } 179188736Sgallatin return ENXIO; 180155852Sgallatin} 181155852Sgallatin 182155852Sgallatinstatic void 183159571Sgallatinmxge_enable_wc(mxge_softc_t *sc) 184155852Sgallatin{ 185171500Sgallatin#if defined(__i386) || defined(__amd64) 186155852Sgallatin vm_offset_t len; 187177104Sgallatin int err; 188155852Sgallatin 189170853Sgallatin sc->wc = 1; 190170330Sgallatin len = rman_get_size(sc->mem_res); 191170330Sgallatin err = pmap_change_attr((vm_offset_t) sc->sram, 192170330Sgallatin len, PAT_WRITE_COMBINING); 193177104Sgallatin if (err != 0) { 194170330Sgallatin device_printf(sc->dev, "pmap_change_attr failed, %d\n", 195170330Sgallatin err); 196170853Sgallatin sc->wc = 0; 197155852Sgallatin } 198171500Sgallatin#endif 199155852Sgallatin} 200155852Sgallatin 201155852Sgallatin 202155852Sgallatin/* callback to get our DMA address */ 203155852Sgallatinstatic void 204159571Sgallatinmxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, 205155852Sgallatin int error) 206155852Sgallatin{ 207155852Sgallatin if (error == 0) { 208155852Sgallatin *(bus_addr_t *) arg = segs->ds_addr; 209155852Sgallatin } 210155852Sgallatin} 211155852Sgallatin 212155852Sgallatinstatic int 213159571Sgallatinmxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes, 214155852Sgallatin bus_size_t alignment) 215155852Sgallatin{ 216155852Sgallatin int err; 217155852Sgallatin device_t dev = sc->dev; 218175365Sgallatin bus_size_t boundary, maxsegsize; 219155852Sgallatin 220175365Sgallatin if (bytes > 4096 && alignment == 4096) { 221175365Sgallatin boundary = 0; 222175365Sgallatin maxsegsize = bytes; 223175365Sgallatin } else { 224175365Sgallatin boundary = 4096; 225175365Sgallatin maxsegsize = 4096; 226175365Sgallatin } 227175365Sgallatin 228155852Sgallatin /* allocate DMAable memory tags */ 229155852Sgallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 230155852Sgallatin alignment, /* alignment */ 231175365Sgallatin boundary, /* boundary */ 232155852Sgallatin BUS_SPACE_MAXADDR, /* low */ 233155852Sgallatin BUS_SPACE_MAXADDR, /* high */ 234155852Sgallatin NULL, NULL, /* filter */ 235155852Sgallatin bytes, /* maxsize */ 236155852Sgallatin 1, /* num segs */ 237175365Sgallatin maxsegsize, /* maxsegsize */ 238155852Sgallatin BUS_DMA_COHERENT, /* flags */ 239155852Sgallatin NULL, NULL, /* lock */ 240155852Sgallatin &dma->dmat); /* tag */ 241155852Sgallatin if (err != 0) { 242155852Sgallatin device_printf(dev, "couldn't alloc tag (err = %d)\n", err); 243155852Sgallatin return err; 244155852Sgallatin } 245155852Sgallatin 246155852Sgallatin /* allocate DMAable memory & map */ 247155852Sgallatin err = bus_dmamem_alloc(dma->dmat, &dma->addr, 248155852Sgallatin (BUS_DMA_WAITOK | BUS_DMA_COHERENT 249155852Sgallatin | BUS_DMA_ZERO), &dma->map); 250155852Sgallatin if (err != 0) { 251155852Sgallatin device_printf(dev, "couldn't alloc mem (err = %d)\n", err); 252155852Sgallatin goto abort_with_dmat; 253155852Sgallatin } 254155852Sgallatin 255155852Sgallatin /* load the memory */ 256155852Sgallatin err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes, 257159571Sgallatin mxge_dmamap_callback, 258155852Sgallatin (void *)&dma->bus_addr, 0); 259155852Sgallatin if (err != 0) { 260155852Sgallatin device_printf(dev, "couldn't load map (err = %d)\n", err); 261155852Sgallatin goto abort_with_mem; 262155852Sgallatin } 263155852Sgallatin return 0; 264155852Sgallatin 265155852Sgallatinabort_with_mem: 266155852Sgallatin bus_dmamem_free(dma->dmat, dma->addr, dma->map); 267155852Sgallatinabort_with_dmat: 268155852Sgallatin (void)bus_dma_tag_destroy(dma->dmat); 269155852Sgallatin return err; 270155852Sgallatin} 271155852Sgallatin 272155852Sgallatin 273155852Sgallatinstatic void 274159571Sgallatinmxge_dma_free(mxge_dma_t *dma) 275155852Sgallatin{ 276155852Sgallatin bus_dmamap_unload(dma->dmat, dma->map); 277155852Sgallatin bus_dmamem_free(dma->dmat, dma->addr, dma->map); 278155852Sgallatin (void)bus_dma_tag_destroy(dma->dmat); 279155852Sgallatin} 280155852Sgallatin 281155852Sgallatin/* 282155852Sgallatin * The eeprom strings on the lanaiX have the format 283155852Sgallatin * SN=x\0 284155852Sgallatin * MAC=x:x:x:x:x:x\0 285155852Sgallatin * PC=text\0 286155852Sgallatin */ 287155852Sgallatin 288155852Sgallatinstatic int 289159571Sgallatinmxge_parse_strings(mxge_softc_t *sc) 290155852Sgallatin{ 291247268Sgallatin char *ptr; 292247159Sgallatin int i, found_mac, found_sn2; 293247268Sgallatin char *endptr; 294155852Sgallatin 295155852Sgallatin ptr = sc->eeprom_strings; 296155852Sgallatin found_mac = 0; 297247159Sgallatin found_sn2 = 0; 298247268Sgallatin while (*ptr != '\0') { 299247268Sgallatin if (strncmp(ptr, "MAC=", 4) == 0) { 300247268Sgallatin ptr += 4; 301247268Sgallatin for (i = 0;;) { 302247268Sgallatin sc->mac_addr[i] = strtoul(ptr, &endptr, 16); 303247268Sgallatin if (endptr - ptr != 2) 304155852Sgallatin goto abort; 305247268Sgallatin ptr = endptr; 306247268Sgallatin if (++i == 6) 307247268Sgallatin break; 308247268Sgallatin if (*ptr++ != ':') 309247268Sgallatin goto abort; 310155852Sgallatin } 311247268Sgallatin found_mac = 1; 312247268Sgallatin } else if (strncmp(ptr, "PC=", 3) == 0) { 313159612Sgallatin ptr += 3; 314247268Sgallatin strlcpy(sc->product_code_string, ptr, 315247268Sgallatin sizeof(sc->product_code_string)); 316247268Sgallatin } else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) { 317159612Sgallatin ptr += 3; 318247268Sgallatin strlcpy(sc->serial_number_string, ptr, 319247268Sgallatin sizeof(sc->serial_number_string)); 320247268Sgallatin } else if (strncmp(ptr, "SN2=", 4) == 0) { 321247159Sgallatin /* SN2 takes precedence over SN */ 322247159Sgallatin ptr += 4; 323247159Sgallatin found_sn2 = 1; 324247268Sgallatin strlcpy(sc->serial_number_string, ptr, 325247268Sgallatin sizeof(sc->serial_number_string)); 326155852Sgallatin } 327247268Sgallatin while (*ptr++ != '\0') {} 328155852Sgallatin } 329155852Sgallatin 330155852Sgallatin if (found_mac) 331155852Sgallatin return 0; 332155852Sgallatin 333155852Sgallatin abort: 334155852Sgallatin device_printf(sc->dev, "failed to parse eeprom_strings\n"); 335155852Sgallatin 336155852Sgallatin return ENXIO; 337155852Sgallatin} 338155852Sgallatin 339188531Srdivacky#if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__ 340169376Sgallatinstatic void 341169376Sgallatinmxge_enable_nvidia_ecrc(mxge_softc_t *sc) 342155852Sgallatin{ 343155852Sgallatin uint32_t val; 344169376Sgallatin unsigned long base, off; 345155852Sgallatin char *va, *cfgptr; 346169376Sgallatin device_t pdev, mcp55; 347169376Sgallatin uint16_t vendor_id, device_id, word; 348155852Sgallatin uintptr_t bus, slot, func, ivend, idev; 349155852Sgallatin uint32_t *ptr32; 350155852Sgallatin 351169376Sgallatin 352169376Sgallatin if (!mxge_nvidia_ecrc_enable) 353169376Sgallatin return; 354169376Sgallatin 355169376Sgallatin pdev = device_get_parent(device_get_parent(sc->dev)); 356169376Sgallatin if (pdev == NULL) { 357169376Sgallatin device_printf(sc->dev, "could not find parent?\n"); 358169376Sgallatin return; 359169376Sgallatin } 360169376Sgallatin vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2); 361169376Sgallatin device_id = pci_read_config(pdev, PCIR_DEVICE, 2); 362169376Sgallatin 363169376Sgallatin if (vendor_id != 0x10de) 364169376Sgallatin return; 365169376Sgallatin 366169376Sgallatin base = 0; 367169376Sgallatin 368169376Sgallatin if (device_id == 0x005d) { 369169376Sgallatin /* ck804, base address is magic */ 370169376Sgallatin base = 0xe0000000UL; 371169376Sgallatin } else if (device_id >= 0x0374 && device_id <= 0x378) { 372169376Sgallatin /* mcp55, base address stored in chipset */ 373169376Sgallatin mcp55 = pci_find_bsf(0, 0, 0); 374169376Sgallatin if (mcp55 && 375169376Sgallatin 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) && 376169376Sgallatin 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) { 377169376Sgallatin word = pci_read_config(mcp55, 0x90, 2); 378169376Sgallatin base = ((unsigned long)word & 0x7ffeU) << 25; 379169376Sgallatin } 380169376Sgallatin } 381169376Sgallatin if (!base) 382169376Sgallatin return; 383169376Sgallatin 384155852Sgallatin /* XXXX 385155852Sgallatin Test below is commented because it is believed that doing 386155852Sgallatin config read/write beyond 0xff will access the config space 387155852Sgallatin for the next larger function. Uncomment this and remove 388155852Sgallatin the hacky pmap_mapdev() way of accessing config space when 389155852Sgallatin FreeBSD grows support for extended pcie config space access 390155852Sgallatin */ 391155852Sgallatin#if 0 392155852Sgallatin /* See if we can, by some miracle, access the extended 393155852Sgallatin config space */ 394155852Sgallatin val = pci_read_config(pdev, 0x178, 4); 395155852Sgallatin if (val != 0xffffffff) { 396155852Sgallatin val |= 0x40; 397155852Sgallatin pci_write_config(pdev, 0x178, val, 4); 398169376Sgallatin return; 399155852Sgallatin } 400155852Sgallatin#endif 401155852Sgallatin /* Rather than using normal pci config space writes, we must 402155852Sgallatin * map the Nvidia config space ourselves. This is because on 403155852Sgallatin * opteron/nvidia class machine the 0xe000000 mapping is 404155852Sgallatin * handled by the nvidia chipset, that means the internal PCI 405155852Sgallatin * device (the on-chip northbridge), or the amd-8131 bridge 406155852Sgallatin * and things behind them are not visible by this method. 407155852Sgallatin */ 408155852Sgallatin 409155852Sgallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 410155852Sgallatin PCI_IVAR_BUS, &bus); 411155852Sgallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 412155852Sgallatin PCI_IVAR_SLOT, &slot); 413155852Sgallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 414155852Sgallatin PCI_IVAR_FUNCTION, &func); 415155852Sgallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 416155852Sgallatin PCI_IVAR_VENDOR, &ivend); 417155852Sgallatin BUS_READ_IVAR(device_get_parent(pdev), pdev, 418155852Sgallatin PCI_IVAR_DEVICE, &idev); 419155852Sgallatin 420169376Sgallatin off = base 421155852Sgallatin + 0x00100000UL * (unsigned long)bus 422155852Sgallatin + 0x00001000UL * (unsigned long)(func 423155852Sgallatin + 8 * slot); 424155852Sgallatin 425155852Sgallatin /* map it into the kernel */ 426155852Sgallatin va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE); 427155852Sgallatin 428155852Sgallatin 429155852Sgallatin if (va == NULL) { 430155852Sgallatin device_printf(sc->dev, "pmap_kenter_temporary didn't\n"); 431169376Sgallatin return; 432155852Sgallatin } 433155852Sgallatin /* get a pointer to the config space mapped into the kernel */ 434155852Sgallatin cfgptr = va + (off & PAGE_MASK); 435155852Sgallatin 436155852Sgallatin /* make sure that we can really access it */ 437155852Sgallatin vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR); 438155852Sgallatin device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE); 439155852Sgallatin if (! (vendor_id == ivend && device_id == idev)) { 440155852Sgallatin device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n", 441155852Sgallatin vendor_id, device_id); 442155852Sgallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 443169376Sgallatin return; 444155852Sgallatin } 445155852Sgallatin 446155852Sgallatin ptr32 = (uint32_t*)(cfgptr + 0x178); 447155852Sgallatin val = *ptr32; 448155852Sgallatin 449155852Sgallatin if (val == 0xffffffff) { 450155852Sgallatin device_printf(sc->dev, "extended mapping failed\n"); 451155852Sgallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 452169376Sgallatin return; 453155852Sgallatin } 454155852Sgallatin *ptr32 = val | 0x40; 455155852Sgallatin pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); 456159612Sgallatin if (mxge_verbose) 457159612Sgallatin device_printf(sc->dev, 458159612Sgallatin "Enabled ECRC on upstream Nvidia bridge " 459159612Sgallatin "at %d:%d:%d\n", 460159612Sgallatin (int)bus, (int)slot, (int)func); 461169376Sgallatin return; 462155852Sgallatin} 463155852Sgallatin#else 464169376Sgallatinstatic void 465171500Sgallatinmxge_enable_nvidia_ecrc(mxge_softc_t *sc) 466155852Sgallatin{ 467155852Sgallatin device_printf(sc->dev, 468155852Sgallatin "Nforce 4 chipset on non-x86/amd64!?!?!\n"); 469169376Sgallatin return; 470155852Sgallatin} 471155852Sgallatin#endif 472169376Sgallatin 473169376Sgallatin 474169376Sgallatinstatic int 475169376Sgallatinmxge_dma_test(mxge_softc_t *sc, int test_type) 476169376Sgallatin{ 477169376Sgallatin mxge_cmd_t cmd; 478169376Sgallatin bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr; 479169376Sgallatin int status; 480169376Sgallatin uint32_t len; 481169376Sgallatin char *test = " "; 482169376Sgallatin 483169376Sgallatin 484169376Sgallatin /* Run a small DMA test. 485169376Sgallatin * The magic multipliers to the length tell the firmware 486169376Sgallatin * to do DMA read, write, or read+write tests. The 487169376Sgallatin * results are returned in cmd.data0. The upper 16 488169376Sgallatin * bits of the return is the number of transfers completed. 489169376Sgallatin * The lower 16 bits is the time in 0.5us ticks that the 490169376Sgallatin * transfers took to complete. 491169376Sgallatin */ 492169376Sgallatin 493175365Sgallatin len = sc->tx_boundary; 494169376Sgallatin 495169376Sgallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 496169376Sgallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 497169376Sgallatin cmd.data2 = len * 0x10000; 498169376Sgallatin status = mxge_send_cmd(sc, test_type, &cmd); 499169376Sgallatin if (status != 0) { 500169376Sgallatin test = "read"; 501169376Sgallatin goto abort; 502169376Sgallatin } 503169376Sgallatin sc->read_dma = ((cmd.data0>>16) * len * 2) / 504169376Sgallatin (cmd.data0 & 0xffff); 505169376Sgallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 506169376Sgallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 507169376Sgallatin cmd.data2 = len * 0x1; 508169376Sgallatin status = mxge_send_cmd(sc, test_type, &cmd); 509169376Sgallatin if (status != 0) { 510169376Sgallatin test = "write"; 511169376Sgallatin goto abort; 512169376Sgallatin } 513169376Sgallatin sc->write_dma = ((cmd.data0>>16) * len * 2) / 514169376Sgallatin (cmd.data0 & 0xffff); 515169376Sgallatin 516169376Sgallatin cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); 517169376Sgallatin cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); 518169376Sgallatin cmd.data2 = len * 0x10001; 519169376Sgallatin status = mxge_send_cmd(sc, test_type, &cmd); 520169376Sgallatin if (status != 0) { 521169376Sgallatin test = "read/write"; 522169376Sgallatin goto abort; 523169376Sgallatin } 524169376Sgallatin sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) / 525169376Sgallatin (cmd.data0 & 0xffff); 526169376Sgallatin 527169376Sgallatinabort: 528169376Sgallatin if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 529169376Sgallatin device_printf(sc->dev, "DMA %s benchmark failed: %d\n", 530169376Sgallatin test, status); 531169376Sgallatin 532169376Sgallatin return status; 533169376Sgallatin} 534169376Sgallatin 535155852Sgallatin/* 536155852Sgallatin * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 537155852Sgallatin * when the PCI-E Completion packets are aligned on an 8-byte 538155852Sgallatin * boundary. Some PCI-E chip sets always align Completion packets; on 539155852Sgallatin * the ones that do not, the alignment can be enforced by enabling 540155852Sgallatin * ECRC generation (if supported). 541155852Sgallatin * 542155852Sgallatin * When PCI-E Completion packets are not aligned, it is actually more 543155852Sgallatin * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 544155852Sgallatin * 545155852Sgallatin * If the driver can neither enable ECRC nor verify that it has 546155852Sgallatin * already been enabled, then it must use a firmware image which works 547155852Sgallatin * around unaligned completion packets (ethp_z8e.dat), and it should 548155852Sgallatin * also ensure that it never gives the device a Read-DMA which is 549175365Sgallatin * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 550155852Sgallatin * enabled, then the driver should use the aligned (eth_z8e.dat) 551175365Sgallatin * firmware image, and set tx_boundary to 4KB. 552155852Sgallatin */ 553155852Sgallatin 554169376Sgallatinstatic int 555169376Sgallatinmxge_firmware_probe(mxge_softc_t *sc) 556169376Sgallatin{ 557169376Sgallatin device_t dev = sc->dev; 558169376Sgallatin int reg, status; 559169376Sgallatin uint16_t pectl; 560169376Sgallatin 561175365Sgallatin sc->tx_boundary = 4096; 562169376Sgallatin /* 563169376Sgallatin * Verify the max read request size was set to 4KB 564169376Sgallatin * before trying the test with 4KB. 565169376Sgallatin */ 566219902Sjhb if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 567169376Sgallatin pectl = pci_read_config(dev, reg + 0x8, 2); 568169376Sgallatin if ((pectl & (5 << 12)) != (5 << 12)) { 569169376Sgallatin device_printf(dev, "Max Read Req. size != 4k (0x%x\n", 570169376Sgallatin pectl); 571175365Sgallatin sc->tx_boundary = 2048; 572169376Sgallatin } 573169376Sgallatin } 574169376Sgallatin 575169376Sgallatin /* 576169376Sgallatin * load the optimized firmware (which assumes aligned PCIe 577169376Sgallatin * completions) in order to see if it works on this host. 578169376Sgallatin */ 579169376Sgallatin sc->fw_name = mxge_fw_aligned; 580175365Sgallatin status = mxge_load_firmware(sc, 1); 581169376Sgallatin if (status != 0) { 582169376Sgallatin return status; 583169376Sgallatin } 584169376Sgallatin 585169376Sgallatin /* 586169376Sgallatin * Enable ECRC if possible 587169376Sgallatin */ 588169376Sgallatin mxge_enable_nvidia_ecrc(sc); 589169376Sgallatin 590169376Sgallatin /* 591169376Sgallatin * Run a DMA test which watches for unaligned completions and 592247159Sgallatin * aborts on the first one seen. Not required on Z8ES or newer. 593169376Sgallatin */ 594247159Sgallatin if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES) 595247159Sgallatin return 0; 596169376Sgallatin status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST); 597169376Sgallatin if (status == 0) 598169376Sgallatin return 0; /* keep the aligned firmware */ 599169376Sgallatin 600169376Sgallatin if (status != E2BIG) 601169376Sgallatin device_printf(dev, "DMA test failed: %d\n", status); 602169376Sgallatin if (status == ENOSYS) 603169376Sgallatin device_printf(dev, "Falling back to ethp! " 604169376Sgallatin "Please install up to date fw\n"); 605169376Sgallatin return status; 606169376Sgallatin} 607169376Sgallatin 608169376Sgallatinstatic int 609159571Sgallatinmxge_select_firmware(mxge_softc_t *sc) 610155852Sgallatin{ 611169376Sgallatin int aligned = 0; 612197391Sgallatin int force_firmware = mxge_force_firmware; 613155852Sgallatin 614197391Sgallatin if (sc->throttle) 615197391Sgallatin force_firmware = sc->throttle; 616164513Sgallatin 617197391Sgallatin if (force_firmware != 0) { 618197391Sgallatin if (force_firmware == 1) 619164513Sgallatin aligned = 1; 620164513Sgallatin else 621164513Sgallatin aligned = 0; 622164513Sgallatin if (mxge_verbose) 623164513Sgallatin device_printf(sc->dev, 624164513Sgallatin "Assuming %s completions (forced)\n", 625164513Sgallatin aligned ? "aligned" : "unaligned"); 626164513Sgallatin goto abort; 627164513Sgallatin } 628164513Sgallatin 629164513Sgallatin /* if the PCIe link width is 4 or less, we can use the aligned 630164513Sgallatin firmware and skip any checks */ 631164513Sgallatin if (sc->link_width != 0 && sc->link_width <= 4) { 632164513Sgallatin device_printf(sc->dev, 633164513Sgallatin "PCIe x%d Link, expect reduced performance\n", 634164513Sgallatin sc->link_width); 635164513Sgallatin aligned = 1; 636164513Sgallatin goto abort; 637164513Sgallatin } 638164513Sgallatin 639169376Sgallatin if (0 == mxge_firmware_probe(sc)) 640169376Sgallatin return 0; 641155852Sgallatin 642155852Sgallatinabort: 643155852Sgallatin if (aligned) { 644159571Sgallatin sc->fw_name = mxge_fw_aligned; 645175365Sgallatin sc->tx_boundary = 4096; 646155852Sgallatin } else { 647159571Sgallatin sc->fw_name = mxge_fw_unaligned; 648175365Sgallatin sc->tx_boundary = 2048; 649155852Sgallatin } 650175365Sgallatin return (mxge_load_firmware(sc, 0)); 651155852Sgallatin} 652155852Sgallatin 653160456Sgallatinstatic int 654160456Sgallatinmxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr) 655160456Sgallatin{ 656155852Sgallatin 657166875Sgallatin 658160456Sgallatin if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) { 659160456Sgallatin device_printf(sc->dev, "Bad firmware type: 0x%x\n", 660160456Sgallatin be32toh(hdr->mcp_type)); 661160456Sgallatin return EIO; 662160456Sgallatin } 663160456Sgallatin 664160456Sgallatin /* save firmware version for sysctl */ 665247268Sgallatin strlcpy(sc->fw_version, hdr->version, sizeof(sc->fw_version)); 666160456Sgallatin if (mxge_verbose) 667160456Sgallatin device_printf(sc->dev, "firmware id: %s\n", hdr->version); 668160456Sgallatin 669166875Sgallatin sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major, 670166875Sgallatin &sc->fw_ver_minor, &sc->fw_ver_tiny); 671160456Sgallatin 672166875Sgallatin if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR 673166875Sgallatin && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 674160456Sgallatin device_printf(sc->dev, "Found firmware version %s\n", 675160456Sgallatin sc->fw_version); 676160456Sgallatin device_printf(sc->dev, "Driver needs %d.%d\n", 677160456Sgallatin MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR); 678160456Sgallatin return EINVAL; 679160456Sgallatin } 680160456Sgallatin return 0; 681160456Sgallatin 682160456Sgallatin} 683160456Sgallatin 684171500Sgallatinstatic void * 685171500Sgallatinz_alloc(void *nil, u_int items, u_int size) 686171500Sgallatin{ 687171500Sgallatin void *ptr; 688171500Sgallatin 689171500Sgallatin ptr = malloc(items * size, M_TEMP, M_NOWAIT); 690171500Sgallatin return ptr; 691171500Sgallatin} 692171500Sgallatin 693171500Sgallatinstatic void 694171500Sgallatinz_free(void *nil, void *ptr) 695171500Sgallatin{ 696171500Sgallatin free(ptr, M_TEMP); 697171500Sgallatin} 698171500Sgallatin 699171500Sgallatin 700155852Sgallatinstatic int 701159571Sgallatinmxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit) 702155852Sgallatin{ 703171500Sgallatin z_stream zs; 704171500Sgallatin char *inflate_buffer; 705166756Sluigi const struct firmware *fw; 706155852Sgallatin const mcp_gen_header_t *hdr; 707155852Sgallatin unsigned hdr_offset; 708155852Sgallatin int status; 709160456Sgallatin unsigned int i; 710160456Sgallatin char dummy; 711171500Sgallatin size_t fw_len; 712155852Sgallatin 713155852Sgallatin fw = firmware_get(sc->fw_name); 714155852Sgallatin if (fw == NULL) { 715155852Sgallatin device_printf(sc->dev, "Could not find firmware image %s\n", 716155852Sgallatin sc->fw_name); 717155852Sgallatin return ENOENT; 718155852Sgallatin } 719171500Sgallatin 720171500Sgallatin 721171500Sgallatin 722171500Sgallatin /* setup zlib and decompress f/w */ 723171500Sgallatin bzero(&zs, sizeof (zs)); 724171500Sgallatin zs.zalloc = z_alloc; 725171500Sgallatin zs.zfree = z_free; 726171500Sgallatin status = inflateInit(&zs); 727171500Sgallatin if (status != Z_OK) { 728171500Sgallatin status = EIO; 729155852Sgallatin goto abort_with_fw; 730155852Sgallatin } 731155852Sgallatin 732171500Sgallatin /* the uncompressed size is stored as the firmware version, 733171500Sgallatin which would otherwise go unused */ 734171500Sgallatin fw_len = (size_t) fw->version; 735171500Sgallatin inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT); 736171500Sgallatin if (inflate_buffer == NULL) 737171500Sgallatin goto abort_with_zs; 738171500Sgallatin zs.avail_in = fw->datasize; 739171500Sgallatin zs.next_in = __DECONST(char *, fw->data); 740171500Sgallatin zs.avail_out = fw_len; 741171500Sgallatin zs.next_out = inflate_buffer; 742171500Sgallatin status = inflate(&zs, Z_FINISH); 743171500Sgallatin if (status != Z_STREAM_END) { 744171500Sgallatin device_printf(sc->dev, "zlib %d\n", status); 745171500Sgallatin status = EIO; 746171500Sgallatin goto abort_with_buffer; 747171500Sgallatin } 748171500Sgallatin 749155852Sgallatin /* check id */ 750155852Sgallatin hdr_offset = htobe32(*(const uint32_t *) 751171500Sgallatin (inflate_buffer + MCP_HEADER_PTR_OFFSET)); 752171500Sgallatin if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) { 753155852Sgallatin device_printf(sc->dev, "Bad firmware file"); 754155852Sgallatin status = EIO; 755171500Sgallatin goto abort_with_buffer; 756155852Sgallatin } 757171500Sgallatin hdr = (const void*)(inflate_buffer + hdr_offset); 758160456Sgallatin 759160456Sgallatin status = mxge_validate_firmware(sc, hdr); 760160456Sgallatin if (status != 0) 761171500Sgallatin goto abort_with_buffer; 762155852Sgallatin 763155852Sgallatin /* Copy the inflated firmware to NIC SRAM. */ 764171500Sgallatin for (i = 0; i < fw_len; i += 256) { 765160456Sgallatin mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, 766171500Sgallatin inflate_buffer + i, 767171500Sgallatin min(256U, (unsigned)(fw_len - i))); 768185255Sgallatin wmb(); 769160456Sgallatin dummy = *sc->sram; 770185255Sgallatin wmb(); 771160456Sgallatin } 772155852Sgallatin 773171500Sgallatin *limit = fw_len; 774155852Sgallatin status = 0; 775171500Sgallatinabort_with_buffer: 776171500Sgallatin free(inflate_buffer, M_TEMP); 777171500Sgallatinabort_with_zs: 778171500Sgallatin inflateEnd(&zs); 779155852Sgallatinabort_with_fw: 780155852Sgallatin firmware_put(fw, FIRMWARE_UNLOAD); 781155852Sgallatin return status; 782155852Sgallatin} 783155852Sgallatin 784155852Sgallatin/* 785155852Sgallatin * Enable or disable periodic RDMAs from the host to make certain 786155852Sgallatin * chipsets resend dropped PCIe messages 787155852Sgallatin */ 788155852Sgallatin 789155852Sgallatinstatic void 790159571Sgallatinmxge_dummy_rdma(mxge_softc_t *sc, int enable) 791155852Sgallatin{ 792155852Sgallatin char buf_bytes[72]; 793155852Sgallatin volatile uint32_t *confirm; 794155852Sgallatin volatile char *submit; 795155852Sgallatin uint32_t *buf, dma_low, dma_high; 796155852Sgallatin int i; 797155852Sgallatin 798155852Sgallatin buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 799155852Sgallatin 800155852Sgallatin /* clear confirmation addr */ 801155852Sgallatin confirm = (volatile uint32_t *)sc->cmd; 802155852Sgallatin *confirm = 0; 803185255Sgallatin wmb(); 804155852Sgallatin 805155852Sgallatin /* send an rdma command to the PCIe engine, and wait for the 806155852Sgallatin response in the confirmation address. The firmware should 807155852Sgallatin write a -1 there to indicate it is alive and well 808155852Sgallatin */ 809155852Sgallatin 810159571Sgallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 811159571Sgallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 812155852Sgallatin buf[0] = htobe32(dma_high); /* confirm addr MSW */ 813155852Sgallatin buf[1] = htobe32(dma_low); /* confirm addr LSW */ 814155852Sgallatin buf[2] = htobe32(0xffffffff); /* confirm data */ 815159571Sgallatin dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr); 816159571Sgallatin dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr); 817155852Sgallatin buf[3] = htobe32(dma_high); /* dummy addr MSW */ 818155852Sgallatin buf[4] = htobe32(dma_low); /* dummy addr LSW */ 819155852Sgallatin buf[5] = htobe32(enable); /* enable? */ 820155852Sgallatin 821155852Sgallatin 822162328Sgallatin submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); 823155852Sgallatin 824159571Sgallatin mxge_pio_copy(submit, buf, 64); 825185255Sgallatin wmb(); 826155852Sgallatin DELAY(1000); 827185255Sgallatin wmb(); 828155852Sgallatin i = 0; 829155852Sgallatin while (*confirm != 0xffffffff && i < 20) { 830155852Sgallatin DELAY(1000); 831155852Sgallatin i++; 832155852Sgallatin } 833155852Sgallatin if (*confirm != 0xffffffff) { 834155852Sgallatin device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)", 835155852Sgallatin (enable ? "enable" : "disable"), confirm, 836155852Sgallatin *confirm); 837155852Sgallatin } 838155852Sgallatin return; 839155852Sgallatin} 840155852Sgallatin 841155852Sgallatinstatic int 842159571Sgallatinmxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data) 843155852Sgallatin{ 844155852Sgallatin mcp_cmd_t *buf; 845155852Sgallatin char buf_bytes[sizeof(*buf) + 8]; 846155852Sgallatin volatile mcp_cmd_response_t *response = sc->cmd; 847162328Sgallatin volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD; 848155852Sgallatin uint32_t dma_low, dma_high; 849169384Sgallatin int err, sleep_total = 0; 850155852Sgallatin 851155852Sgallatin /* ensure buf is aligned to 8 bytes */ 852155852Sgallatin buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 853155852Sgallatin 854155852Sgallatin buf->data0 = htobe32(data->data0); 855155852Sgallatin buf->data1 = htobe32(data->data1); 856155852Sgallatin buf->data2 = htobe32(data->data2); 857155852Sgallatin buf->cmd = htobe32(cmd); 858159571Sgallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 859159571Sgallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 860155852Sgallatin 861155852Sgallatin buf->response_addr.low = htobe32(dma_low); 862155852Sgallatin buf->response_addr.high = htobe32(dma_high); 863166370Sgallatin mtx_lock(&sc->cmd_mtx); 864155852Sgallatin response->result = 0xffffffff; 865185255Sgallatin wmb(); 866159571Sgallatin mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); 867155852Sgallatin 868159612Sgallatin /* wait up to 20ms */ 869169384Sgallatin err = EAGAIN; 870159612Sgallatin for (sleep_total = 0; sleep_total < 20; sleep_total++) { 871155852Sgallatin bus_dmamap_sync(sc->cmd_dma.dmat, 872155852Sgallatin sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 873185255Sgallatin wmb(); 874169384Sgallatin switch (be32toh(response->result)) { 875169384Sgallatin case 0: 876169384Sgallatin data->data0 = be32toh(response->data); 877169384Sgallatin err = 0; 878169384Sgallatin break; 879169384Sgallatin case 0xffffffff: 880169384Sgallatin DELAY(1000); 881169384Sgallatin break; 882169384Sgallatin case MXGEFW_CMD_UNKNOWN: 883169384Sgallatin err = ENOSYS; 884169384Sgallatin break; 885169384Sgallatin case MXGEFW_CMD_ERROR_UNALIGNED: 886169384Sgallatin err = E2BIG; 887169384Sgallatin break; 888171917Sgallatin case MXGEFW_CMD_ERROR_BUSY: 889171917Sgallatin err = EBUSY; 890171917Sgallatin break; 891206662Sgallatin case MXGEFW_CMD_ERROR_I2C_ABSENT: 892206662Sgallatin err = ENXIO; 893206662Sgallatin break; 894169384Sgallatin default: 895169384Sgallatin device_printf(sc->dev, 896169384Sgallatin "mxge: command %d " 897169384Sgallatin "failed, result = %d\n", 898169384Sgallatin cmd, be32toh(response->result)); 899169384Sgallatin err = ENXIO; 900169384Sgallatin break; 901155852Sgallatin } 902169384Sgallatin if (err != EAGAIN) 903169384Sgallatin break; 904155852Sgallatin } 905169384Sgallatin if (err == EAGAIN) 906169384Sgallatin device_printf(sc->dev, "mxge: command %d timed out" 907169384Sgallatin "result = %d\n", 908169384Sgallatin cmd, be32toh(response->result)); 909166370Sgallatin mtx_unlock(&sc->cmd_mtx); 910169384Sgallatin return err; 911155852Sgallatin} 912155852Sgallatin 913160456Sgallatinstatic int 914160456Sgallatinmxge_adopt_running_firmware(mxge_softc_t *sc) 915160456Sgallatin{ 916160456Sgallatin struct mcp_gen_header *hdr; 917160456Sgallatin const size_t bytes = sizeof (struct mcp_gen_header); 918160456Sgallatin size_t hdr_offset; 919160456Sgallatin int status; 920155852Sgallatin 921160456Sgallatin /* find running firmware header */ 922160456Sgallatin hdr_offset = htobe32(*(volatile uint32_t *) 923160456Sgallatin (sc->sram + MCP_HEADER_PTR_OFFSET)); 924160456Sgallatin 925160456Sgallatin if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) { 926160456Sgallatin device_printf(sc->dev, 927160456Sgallatin "Running firmware has bad header offset (%d)\n", 928160456Sgallatin (int)hdr_offset); 929160456Sgallatin return EIO; 930160456Sgallatin } 931160456Sgallatin 932160456Sgallatin /* copy header of running firmware from SRAM to host memory to 933160456Sgallatin * validate firmware */ 934160456Sgallatin hdr = malloc(bytes, M_DEVBUF, M_NOWAIT); 935160456Sgallatin if (hdr == NULL) { 936160456Sgallatin device_printf(sc->dev, "could not malloc firmware hdr\n"); 937160456Sgallatin return ENOMEM; 938160456Sgallatin } 939160456Sgallatin bus_space_read_region_1(rman_get_bustag(sc->mem_res), 940160456Sgallatin rman_get_bushandle(sc->mem_res), 941160456Sgallatin hdr_offset, (char *)hdr, bytes); 942160456Sgallatin status = mxge_validate_firmware(sc, hdr); 943160456Sgallatin free(hdr, M_DEVBUF); 944166875Sgallatin 945166875Sgallatin /* 946166875Sgallatin * check to see if adopted firmware has bug where adopting 947166875Sgallatin * it will cause broadcasts to be filtered unless the NIC 948166875Sgallatin * is kept in ALLMULTI mode 949166875Sgallatin */ 950166875Sgallatin if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 951166875Sgallatin sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) { 952166875Sgallatin sc->adopted_rx_filter_bug = 1; 953166875Sgallatin device_printf(sc->dev, "Adopting fw %d.%d.%d: " 954166875Sgallatin "working around rx filter bug\n", 955166875Sgallatin sc->fw_ver_major, sc->fw_ver_minor, 956166875Sgallatin sc->fw_ver_tiny); 957166875Sgallatin } 958166875Sgallatin 959160456Sgallatin return status; 960160456Sgallatin} 961160456Sgallatin 962160456Sgallatin 963155852Sgallatinstatic int 964175365Sgallatinmxge_load_firmware(mxge_softc_t *sc, int adopt) 965155852Sgallatin{ 966155852Sgallatin volatile uint32_t *confirm; 967155852Sgallatin volatile char *submit; 968155852Sgallatin char buf_bytes[72]; 969155852Sgallatin uint32_t *buf, size, dma_low, dma_high; 970155852Sgallatin int status, i; 971155852Sgallatin 972155852Sgallatin buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); 973155852Sgallatin 974155852Sgallatin size = sc->sram_size; 975159571Sgallatin status = mxge_load_firmware_helper(sc, &size); 976155852Sgallatin if (status) { 977175365Sgallatin if (!adopt) 978175365Sgallatin return status; 979160456Sgallatin /* Try to use the currently running firmware, if 980160456Sgallatin it is new enough */ 981160456Sgallatin status = mxge_adopt_running_firmware(sc); 982160456Sgallatin if (status) { 983160456Sgallatin device_printf(sc->dev, 984160456Sgallatin "failed to adopt running firmware\n"); 985160456Sgallatin return status; 986160456Sgallatin } 987160456Sgallatin device_printf(sc->dev, 988160456Sgallatin "Successfully adopted running firmware\n"); 989175365Sgallatin if (sc->tx_boundary == 4096) { 990160456Sgallatin device_printf(sc->dev, 991160456Sgallatin "Using firmware currently running on NIC" 992160456Sgallatin ". For optimal\n"); 993160456Sgallatin device_printf(sc->dev, 994160456Sgallatin "performance consider loading optimized " 995160456Sgallatin "firmware\n"); 996160456Sgallatin } 997164513Sgallatin sc->fw_name = mxge_fw_unaligned; 998175365Sgallatin sc->tx_boundary = 2048; 999164513Sgallatin return 0; 1000155852Sgallatin } 1001155852Sgallatin /* clear confirmation addr */ 1002155852Sgallatin confirm = (volatile uint32_t *)sc->cmd; 1003155852Sgallatin *confirm = 0; 1004185255Sgallatin wmb(); 1005155852Sgallatin /* send a reload command to the bootstrap MCP, and wait for the 1006155852Sgallatin response in the confirmation address. The firmware should 1007155852Sgallatin write a -1 there to indicate it is alive and well 1008155852Sgallatin */ 1009155852Sgallatin 1010159571Sgallatin dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); 1011159571Sgallatin dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); 1012155852Sgallatin 1013155852Sgallatin buf[0] = htobe32(dma_high); /* confirm addr MSW */ 1014155852Sgallatin buf[1] = htobe32(dma_low); /* confirm addr LSW */ 1015155852Sgallatin buf[2] = htobe32(0xffffffff); /* confirm data */ 1016155852Sgallatin 1017155852Sgallatin /* FIX: All newest firmware should un-protect the bottom of 1018155852Sgallatin the sram before handoff. However, the very first interfaces 1019155852Sgallatin do not. Therefore the handoff copy must skip the first 8 bytes 1020155852Sgallatin */ 1021155852Sgallatin /* where the code starts*/ 1022159571Sgallatin buf[3] = htobe32(MXGE_FW_OFFSET + 8); 1023155852Sgallatin buf[4] = htobe32(size - 8); /* length of code */ 1024155852Sgallatin buf[5] = htobe32(8); /* where to copy to */ 1025155852Sgallatin buf[6] = htobe32(0); /* where to jump to */ 1026155852Sgallatin 1027162328Sgallatin submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); 1028159571Sgallatin mxge_pio_copy(submit, buf, 64); 1029185255Sgallatin wmb(); 1030155852Sgallatin DELAY(1000); 1031185255Sgallatin wmb(); 1032155852Sgallatin i = 0; 1033155852Sgallatin while (*confirm != 0xffffffff && i < 20) { 1034155852Sgallatin DELAY(1000*10); 1035155852Sgallatin i++; 1036155852Sgallatin bus_dmamap_sync(sc->cmd_dma.dmat, 1037155852Sgallatin sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); 1038155852Sgallatin } 1039155852Sgallatin if (*confirm != 0xffffffff) { 1040155852Sgallatin device_printf(sc->dev,"handoff failed (%p = 0x%x)", 1041155852Sgallatin confirm, *confirm); 1042155852Sgallatin 1043155852Sgallatin return ENXIO; 1044155852Sgallatin } 1045155852Sgallatin return 0; 1046155852Sgallatin} 1047155852Sgallatin 1048155852Sgallatinstatic int 1049159571Sgallatinmxge_update_mac_address(mxge_softc_t *sc) 1050155852Sgallatin{ 1051159571Sgallatin mxge_cmd_t cmd; 1052155852Sgallatin uint8_t *addr = sc->mac_addr; 1053155852Sgallatin int status; 1054155852Sgallatin 1055155852Sgallatin 1056155852Sgallatin cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 1057155852Sgallatin | (addr[2] << 8) | addr[3]); 1058155852Sgallatin 1059155852Sgallatin cmd.data1 = ((addr[4] << 8) | (addr[5])); 1060155852Sgallatin 1061159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd); 1062155852Sgallatin return status; 1063155852Sgallatin} 1064155852Sgallatin 1065155852Sgallatinstatic int 1066159571Sgallatinmxge_change_pause(mxge_softc_t *sc, int pause) 1067155852Sgallatin{ 1068159571Sgallatin mxge_cmd_t cmd; 1069155852Sgallatin int status; 1070155852Sgallatin 1071155852Sgallatin if (pause) 1072159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL, 1073159571Sgallatin &cmd); 1074155852Sgallatin else 1075159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL, 1076159571Sgallatin &cmd); 1077155852Sgallatin 1078155852Sgallatin if (status) { 1079155852Sgallatin device_printf(sc->dev, "Failed to set flow control mode\n"); 1080155852Sgallatin return ENXIO; 1081155852Sgallatin } 1082155852Sgallatin sc->pause = pause; 1083155852Sgallatin return 0; 1084155852Sgallatin} 1085155852Sgallatin 1086155852Sgallatinstatic void 1087159571Sgallatinmxge_change_promisc(mxge_softc_t *sc, int promisc) 1088155852Sgallatin{ 1089159571Sgallatin mxge_cmd_t cmd; 1090155852Sgallatin int status; 1091155852Sgallatin 1092175365Sgallatin if (mxge_always_promisc) 1093175365Sgallatin promisc = 1; 1094175365Sgallatin 1095155852Sgallatin if (promisc) 1096159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC, 1097159571Sgallatin &cmd); 1098155852Sgallatin else 1099159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC, 1100159571Sgallatin &cmd); 1101155852Sgallatin 1102155852Sgallatin if (status) { 1103155852Sgallatin device_printf(sc->dev, "Failed to set promisc mode\n"); 1104155852Sgallatin } 1105155852Sgallatin} 1106155852Sgallatin 1107162328Sgallatinstatic void 1108162328Sgallatinmxge_set_multicast_list(mxge_softc_t *sc) 1109162328Sgallatin{ 1110162328Sgallatin mxge_cmd_t cmd; 1111162328Sgallatin struct ifmultiaddr *ifma; 1112162328Sgallatin struct ifnet *ifp = sc->ifp; 1113162328Sgallatin int err; 1114162328Sgallatin 1115162328Sgallatin /* This firmware is known to not support multicast */ 1116162328Sgallatin if (!sc->fw_multicast_support) 1117162328Sgallatin return; 1118162328Sgallatin 1119162328Sgallatin /* Disable multicast filtering while we play with the lists*/ 1120162328Sgallatin err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd); 1121162328Sgallatin if (err != 0) { 1122162328Sgallatin device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI," 1123162328Sgallatin " error status: %d\n", err); 1124162328Sgallatin return; 1125162328Sgallatin } 1126162328Sgallatin 1127166875Sgallatin if (sc->adopted_rx_filter_bug) 1128166875Sgallatin return; 1129166875Sgallatin 1130162328Sgallatin if (ifp->if_flags & IFF_ALLMULTI) 1131162328Sgallatin /* request to disable multicast filtering, so quit here */ 1132162328Sgallatin return; 1133162328Sgallatin 1134162328Sgallatin /* Flush all the filters */ 1135162328Sgallatin 1136162328Sgallatin err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd); 1137162328Sgallatin if (err != 0) { 1138162328Sgallatin device_printf(sc->dev, 1139162328Sgallatin "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS" 1140162328Sgallatin ", error status: %d\n", err); 1141162328Sgallatin return; 1142162328Sgallatin } 1143162328Sgallatin 1144162328Sgallatin /* Walk the multicast list, and add each address */ 1145162328Sgallatin 1146195049Srwatson if_maddr_rlock(ifp); 1147162328Sgallatin TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1148162328Sgallatin if (ifma->ifma_addr->sa_family != AF_LINK) 1149162328Sgallatin continue; 1150162328Sgallatin bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1151162328Sgallatin &cmd.data0, 4); 1152162328Sgallatin bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4, 1153162328Sgallatin &cmd.data1, 2); 1154162328Sgallatin cmd.data0 = htonl(cmd.data0); 1155162328Sgallatin cmd.data1 = htonl(cmd.data1); 1156162328Sgallatin err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd); 1157162328Sgallatin if (err != 0) { 1158162328Sgallatin device_printf(sc->dev, "Failed " 1159162328Sgallatin "MXGEFW_JOIN_MULTICAST_GROUP, error status:" 1160162328Sgallatin "%d\t", err); 1161162328Sgallatin /* abort, leaving multicast filtering off */ 1162195049Srwatson if_maddr_runlock(ifp); 1163162328Sgallatin return; 1164162328Sgallatin } 1165162328Sgallatin } 1166195049Srwatson if_maddr_runlock(ifp); 1167162328Sgallatin /* Enable multicast filtering */ 1168162328Sgallatin err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd); 1169162328Sgallatin if (err != 0) { 1170162328Sgallatin device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI" 1171162328Sgallatin ", error status: %d\n", err); 1172162328Sgallatin } 1173162328Sgallatin} 1174162328Sgallatin 1175155852Sgallatinstatic int 1176169840Sgallatinmxge_max_mtu(mxge_softc_t *sc) 1177169840Sgallatin{ 1178169840Sgallatin mxge_cmd_t cmd; 1179169840Sgallatin int status; 1180169840Sgallatin 1181169905Sgallatin if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU) 1182169905Sgallatin return MXGEFW_MAX_MTU - MXGEFW_PAD; 1183169840Sgallatin 1184169840Sgallatin /* try to set nbufs to see if it we can 1185169840Sgallatin use virtually contiguous jumbos */ 1186169840Sgallatin cmd.data0 = 0; 1187169840Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 1188169840Sgallatin &cmd); 1189169840Sgallatin if (status == 0) 1190169905Sgallatin return MXGEFW_MAX_MTU - MXGEFW_PAD; 1191169840Sgallatin 1192169840Sgallatin /* otherwise, we're limited to MJUMPAGESIZE */ 1193169840Sgallatin return MJUMPAGESIZE - MXGEFW_PAD; 1194169840Sgallatin} 1195169840Sgallatin 1196169840Sgallatinstatic int 1197169871Sgallatinmxge_reset(mxge_softc_t *sc, int interrupts_setup) 1198155852Sgallatin{ 1199175365Sgallatin struct mxge_slice_state *ss; 1200175365Sgallatin mxge_rx_done_t *rx_done; 1201175365Sgallatin volatile uint32_t *irq_claim; 1202159571Sgallatin mxge_cmd_t cmd; 1203175365Sgallatin int slice, status; 1204155852Sgallatin 1205155852Sgallatin /* try to send a reset command to the card to see if it 1206155852Sgallatin is alive */ 1207155852Sgallatin memset(&cmd, 0, sizeof (cmd)); 1208159612Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 1209155852Sgallatin if (status != 0) { 1210155852Sgallatin device_printf(sc->dev, "failed reset\n"); 1211155852Sgallatin return ENXIO; 1212155852Sgallatin } 1213155852Sgallatin 1214160876Sgallatin mxge_dummy_rdma(sc, 1); 1215160876Sgallatin 1216175365Sgallatin 1217175365Sgallatin /* set the intrq size */ 1218175365Sgallatin cmd.data0 = sc->rx_ring_size; 1219175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 1220175365Sgallatin 1221175365Sgallatin /* 1222175365Sgallatin * Even though we already know how many slices are supported 1223175365Sgallatin * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES 1224175365Sgallatin * has magic side effects, and must be called after a reset. 1225175365Sgallatin * It must be called prior to calling any RSS related cmds, 1226175365Sgallatin * including assigning an interrupt queue for anything but 1227175365Sgallatin * slice 0. It must also be called *after* 1228175365Sgallatin * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 1229175365Sgallatin * the firmware to compute offsets. 1230175365Sgallatin */ 1231175365Sgallatin 1232175365Sgallatin if (sc->num_slices > 1) { 1233175365Sgallatin /* ask the maximum number of slices it supports */ 1234175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 1235175365Sgallatin &cmd); 1236175365Sgallatin if (status != 0) { 1237175365Sgallatin device_printf(sc->dev, 1238175365Sgallatin "failed to get number of slices\n"); 1239175365Sgallatin return status; 1240175365Sgallatin } 1241175365Sgallatin /* 1242175365Sgallatin * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 1243175365Sgallatin * to setting up the interrupt queue DMA 1244175365Sgallatin */ 1245175365Sgallatin cmd.data0 = sc->num_slices; 1246175365Sgallatin cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 1247191562Sgallatin#ifdef IFNET_BUF_RING 1248191562Sgallatin cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 1249191562Sgallatin#endif 1250175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES, 1251175365Sgallatin &cmd); 1252175365Sgallatin if (status != 0) { 1253175365Sgallatin device_printf(sc->dev, 1254175365Sgallatin "failed to set number of slices\n"); 1255175365Sgallatin return status; 1256175365Sgallatin } 1257175365Sgallatin } 1258175365Sgallatin 1259175365Sgallatin 1260169871Sgallatin if (interrupts_setup) { 1261169871Sgallatin /* Now exchange information about interrupts */ 1262175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 1263175365Sgallatin rx_done = &sc->ss[slice].rx_done; 1264175365Sgallatin memset(rx_done->entry, 0, sc->rx_ring_size); 1265175365Sgallatin cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr); 1266175365Sgallatin cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr); 1267175365Sgallatin cmd.data2 = slice; 1268175365Sgallatin status |= mxge_send_cmd(sc, 1269175365Sgallatin MXGEFW_CMD_SET_INTRQ_DMA, 1270175365Sgallatin &cmd); 1271175365Sgallatin } 1272169871Sgallatin } 1273155852Sgallatin 1274159612Sgallatin status |= mxge_send_cmd(sc, 1275159612Sgallatin MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd); 1276159612Sgallatin 1277155852Sgallatin 1278159612Sgallatin sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0); 1279159612Sgallatin 1280159612Sgallatin status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd); 1281175365Sgallatin irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0); 1282159612Sgallatin 1283159612Sgallatin 1284159612Sgallatin status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1285159612Sgallatin &cmd); 1286159612Sgallatin sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0); 1287155852Sgallatin if (status != 0) { 1288155852Sgallatin device_printf(sc->dev, "failed set interrupt parameters\n"); 1289155852Sgallatin return status; 1290155852Sgallatin } 1291159612Sgallatin 1292155852Sgallatin 1293159612Sgallatin *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay); 1294159612Sgallatin 1295159612Sgallatin 1296159612Sgallatin /* run a DMA benchmark */ 1297169376Sgallatin (void) mxge_dma_test(sc, MXGEFW_DMA_TEST); 1298159612Sgallatin 1299175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 1300175365Sgallatin ss = &sc->ss[slice]; 1301175365Sgallatin 1302175365Sgallatin ss->irq_claim = irq_claim + (2 * slice); 1303175365Sgallatin /* reset mcp/driver shared state back to 0 */ 1304175365Sgallatin ss->rx_done.idx = 0; 1305175365Sgallatin ss->rx_done.cnt = 0; 1306175365Sgallatin ss->tx.req = 0; 1307175365Sgallatin ss->tx.done = 0; 1308175365Sgallatin ss->tx.pkt_done = 0; 1309191562Sgallatin ss->tx.queue_active = 0; 1310191562Sgallatin ss->tx.activate = 0; 1311191562Sgallatin ss->tx.deactivate = 0; 1312175365Sgallatin ss->tx.wake = 0; 1313175365Sgallatin ss->tx.defrag = 0; 1314175365Sgallatin ss->tx.stall = 0; 1315175365Sgallatin ss->rx_big.cnt = 0; 1316175365Sgallatin ss->rx_small.cnt = 0; 1317247133Sgallatin ss->lc.lro_bad_csum = 0; 1318247133Sgallatin ss->lc.lro_queued = 0; 1319247133Sgallatin ss->lc.lro_flushed = 0; 1320175365Sgallatin if (ss->fw_stats != NULL) { 1321197395Sgallatin bzero(ss->fw_stats, sizeof *ss->fw_stats); 1322175365Sgallatin } 1323175365Sgallatin } 1324155852Sgallatin sc->rdma_tags_available = 15; 1325159571Sgallatin status = mxge_update_mac_address(sc); 1326194761Sgallatin mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC); 1327159571Sgallatin mxge_change_pause(sc, sc->pause); 1328162328Sgallatin mxge_set_multicast_list(sc); 1329197391Sgallatin if (sc->throttle) { 1330197391Sgallatin cmd.data0 = sc->throttle; 1331197391Sgallatin if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, 1332197391Sgallatin &cmd)) { 1333197391Sgallatin device_printf(sc->dev, 1334197391Sgallatin "can't enable throttle\n"); 1335197391Sgallatin } 1336197391Sgallatin } 1337155852Sgallatin return status; 1338155852Sgallatin} 1339155852Sgallatin 1340155852Sgallatinstatic int 1341197391Sgallatinmxge_change_throttle(SYSCTL_HANDLER_ARGS) 1342197391Sgallatin{ 1343197391Sgallatin mxge_cmd_t cmd; 1344197391Sgallatin mxge_softc_t *sc; 1345197391Sgallatin int err; 1346197391Sgallatin unsigned int throttle; 1347197391Sgallatin 1348197391Sgallatin sc = arg1; 1349197391Sgallatin throttle = sc->throttle; 1350197391Sgallatin err = sysctl_handle_int(oidp, &throttle, arg2, req); 1351197391Sgallatin if (err != 0) { 1352197391Sgallatin return err; 1353197391Sgallatin } 1354197391Sgallatin 1355197391Sgallatin if (throttle == sc->throttle) 1356197391Sgallatin return 0; 1357197391Sgallatin 1358197391Sgallatin if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE) 1359197391Sgallatin return EINVAL; 1360197391Sgallatin 1361197391Sgallatin mtx_lock(&sc->driver_mtx); 1362197391Sgallatin cmd.data0 = throttle; 1363197391Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd); 1364197391Sgallatin if (err == 0) 1365197391Sgallatin sc->throttle = throttle; 1366197391Sgallatin mtx_unlock(&sc->driver_mtx); 1367197391Sgallatin return err; 1368197391Sgallatin} 1369197391Sgallatin 1370197391Sgallatinstatic int 1371159571Sgallatinmxge_change_intr_coal(SYSCTL_HANDLER_ARGS) 1372155852Sgallatin{ 1373159571Sgallatin mxge_softc_t *sc; 1374155852Sgallatin unsigned int intr_coal_delay; 1375155852Sgallatin int err; 1376155852Sgallatin 1377155852Sgallatin sc = arg1; 1378155852Sgallatin intr_coal_delay = sc->intr_coal_delay; 1379155852Sgallatin err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req); 1380155852Sgallatin if (err != 0) { 1381155852Sgallatin return err; 1382155852Sgallatin } 1383155852Sgallatin if (intr_coal_delay == sc->intr_coal_delay) 1384155852Sgallatin return 0; 1385155852Sgallatin 1386155852Sgallatin if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000) 1387155852Sgallatin return EINVAL; 1388155852Sgallatin 1389166370Sgallatin mtx_lock(&sc->driver_mtx); 1390159612Sgallatin *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay); 1391159612Sgallatin sc->intr_coal_delay = intr_coal_delay; 1392159612Sgallatin 1393166370Sgallatin mtx_unlock(&sc->driver_mtx); 1394155852Sgallatin return err; 1395155852Sgallatin} 1396155852Sgallatin 1397155852Sgallatinstatic int 1398159571Sgallatinmxge_change_flow_control(SYSCTL_HANDLER_ARGS) 1399155852Sgallatin{ 1400159571Sgallatin mxge_softc_t *sc; 1401155852Sgallatin unsigned int enabled; 1402155852Sgallatin int err; 1403155852Sgallatin 1404155852Sgallatin sc = arg1; 1405155852Sgallatin enabled = sc->pause; 1406155852Sgallatin err = sysctl_handle_int(oidp, &enabled, arg2, req); 1407155852Sgallatin if (err != 0) { 1408155852Sgallatin return err; 1409155852Sgallatin } 1410155852Sgallatin if (enabled == sc->pause) 1411155852Sgallatin return 0; 1412155852Sgallatin 1413166370Sgallatin mtx_lock(&sc->driver_mtx); 1414159571Sgallatin err = mxge_change_pause(sc, enabled); 1415166370Sgallatin mtx_unlock(&sc->driver_mtx); 1416155852Sgallatin return err; 1417155852Sgallatin} 1418155852Sgallatin 1419155852Sgallatinstatic int 1420159571Sgallatinmxge_handle_be32(SYSCTL_HANDLER_ARGS) 1421155852Sgallatin{ 1422155852Sgallatin int err; 1423155852Sgallatin 1424155852Sgallatin if (arg1 == NULL) 1425155852Sgallatin return EFAULT; 1426155852Sgallatin arg2 = be32toh(*(int *)arg1); 1427155852Sgallatin arg1 = NULL; 1428155852Sgallatin err = sysctl_handle_int(oidp, arg1, arg2, req); 1429155852Sgallatin 1430155852Sgallatin return err; 1431155852Sgallatin} 1432155852Sgallatin 1433155852Sgallatinstatic void 1434175365Sgallatinmxge_rem_sysctls(mxge_softc_t *sc) 1435175365Sgallatin{ 1436175365Sgallatin struct mxge_slice_state *ss; 1437175365Sgallatin int slice; 1438175365Sgallatin 1439175365Sgallatin if (sc->slice_sysctl_tree == NULL) 1440175365Sgallatin return; 1441175365Sgallatin 1442175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 1443175365Sgallatin ss = &sc->ss[slice]; 1444175365Sgallatin if (ss == NULL || ss->sysctl_tree == NULL) 1445175365Sgallatin continue; 1446175365Sgallatin sysctl_ctx_free(&ss->sysctl_ctx); 1447175365Sgallatin ss->sysctl_tree = NULL; 1448175365Sgallatin } 1449175365Sgallatin sysctl_ctx_free(&sc->slice_sysctl_ctx); 1450175365Sgallatin sc->slice_sysctl_tree = NULL; 1451175365Sgallatin} 1452175365Sgallatin 1453175365Sgallatinstatic void 1454159571Sgallatinmxge_add_sysctls(mxge_softc_t *sc) 1455155852Sgallatin{ 1456155852Sgallatin struct sysctl_ctx_list *ctx; 1457155852Sgallatin struct sysctl_oid_list *children; 1458159612Sgallatin mcp_irq_data_t *fw; 1459175365Sgallatin struct mxge_slice_state *ss; 1460175365Sgallatin int slice; 1461175365Sgallatin char slice_num[8]; 1462155852Sgallatin 1463155852Sgallatin ctx = device_get_sysctl_ctx(sc->dev); 1464155852Sgallatin children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1465175365Sgallatin fw = sc->ss[0].fw_stats; 1466155852Sgallatin 1467159612Sgallatin /* random information */ 1468159612Sgallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1469159612Sgallatin "firmware_version", 1470273736Shselasky CTLFLAG_RD, sc->fw_version, 1471159612Sgallatin 0, "firmware version"); 1472159612Sgallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1473159612Sgallatin "serial_number", 1474273736Shselasky CTLFLAG_RD, sc->serial_number_string, 1475159612Sgallatin 0, "serial number"); 1476159612Sgallatin SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 1477159612Sgallatin "product_code", 1478273736Shselasky CTLFLAG_RD, sc->product_code_string, 1479159612Sgallatin 0, "product_code"); 1480159612Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1481164513Sgallatin "pcie_link_width", 1482164513Sgallatin CTLFLAG_RD, &sc->link_width, 1483164513Sgallatin 0, "tx_boundary"); 1484164513Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1485159612Sgallatin "tx_boundary", 1486175365Sgallatin CTLFLAG_RD, &sc->tx_boundary, 1487159612Sgallatin 0, "tx_boundary"); 1488159612Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1489160876Sgallatin "write_combine", 1490160876Sgallatin CTLFLAG_RD, &sc->wc, 1491160876Sgallatin 0, "write combining PIO?"); 1492160876Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1493159612Sgallatin "read_dma_MBs", 1494159612Sgallatin CTLFLAG_RD, &sc->read_dma, 1495159612Sgallatin 0, "DMA Read speed in MB/s"); 1496159612Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1497159612Sgallatin "write_dma_MBs", 1498159612Sgallatin CTLFLAG_RD, &sc->write_dma, 1499159612Sgallatin 0, "DMA Write speed in MB/s"); 1500159612Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1501159612Sgallatin "read_write_dma_MBs", 1502159612Sgallatin CTLFLAG_RD, &sc->read_write_dma, 1503159612Sgallatin 0, "DMA concurrent Read/Write speed in MB/s"); 1504197395Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1505197395Sgallatin "watchdog_resets", 1506197395Sgallatin CTLFLAG_RD, &sc->watchdog_resets, 1507197395Sgallatin 0, "Number of times NIC was reset"); 1508159612Sgallatin 1509159612Sgallatin 1510159612Sgallatin /* performance related tunables */ 1511155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1512155852Sgallatin "intr_coal_delay", 1513155852Sgallatin CTLTYPE_INT|CTLFLAG_RW, sc, 1514159571Sgallatin 0, mxge_change_intr_coal, 1515155852Sgallatin "I", "interrupt coalescing delay in usecs"); 1516155852Sgallatin 1517155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1518197391Sgallatin "throttle", 1519197391Sgallatin CTLTYPE_INT|CTLFLAG_RW, sc, 1520197391Sgallatin 0, mxge_change_throttle, 1521197391Sgallatin "I", "transmit throttling"); 1522197391Sgallatin 1523197391Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1524155852Sgallatin "flow_control_enabled", 1525155852Sgallatin CTLTYPE_INT|CTLFLAG_RW, sc, 1526159571Sgallatin 0, mxge_change_flow_control, 1527155852Sgallatin "I", "interrupt coalescing delay in usecs"); 1528155852Sgallatin 1529155852Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1530159612Sgallatin "deassert_wait", 1531159612Sgallatin CTLFLAG_RW, &mxge_deassert_wait, 1532159612Sgallatin 0, "Wait for IRQ line to go low in ihandler"); 1533155852Sgallatin 1534155852Sgallatin /* stats block from firmware is in network byte order. 1535155852Sgallatin Need to swap it */ 1536155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1537155852Sgallatin "link_up", 1538155852Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->link_up, 1539159571Sgallatin 0, mxge_handle_be32, 1540155852Sgallatin "I", "link up"); 1541155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1542155852Sgallatin "rdma_tags_available", 1543155852Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available, 1544159571Sgallatin 0, mxge_handle_be32, 1545155852Sgallatin "I", "rdma_tags_available"); 1546155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1547169871Sgallatin "dropped_bad_crc32", 1548169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1549169871Sgallatin &fw->dropped_bad_crc32, 1550159571Sgallatin 0, mxge_handle_be32, 1551169871Sgallatin "I", "dropped_bad_crc32"); 1552155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1553169871Sgallatin "dropped_bad_phy", 1554169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1555169871Sgallatin &fw->dropped_bad_phy, 1556169871Sgallatin 0, mxge_handle_be32, 1557169871Sgallatin "I", "dropped_bad_phy"); 1558169871Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1559155852Sgallatin "dropped_link_error_or_filtered", 1560155852Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1561155852Sgallatin &fw->dropped_link_error_or_filtered, 1562159571Sgallatin 0, mxge_handle_be32, 1563155852Sgallatin "I", "dropped_link_error_or_filtered"); 1564155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1565169871Sgallatin "dropped_link_overflow", 1566169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow, 1567169871Sgallatin 0, mxge_handle_be32, 1568169871Sgallatin "I", "dropped_link_overflow"); 1569169871Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1570162328Sgallatin "dropped_multicast_filtered", 1571162328Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1572162328Sgallatin &fw->dropped_multicast_filtered, 1573162328Sgallatin 0, mxge_handle_be32, 1574162328Sgallatin "I", "dropped_multicast_filtered"); 1575162328Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1576169871Sgallatin "dropped_no_big_buffer", 1577169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer, 1578159571Sgallatin 0, mxge_handle_be32, 1579169871Sgallatin "I", "dropped_no_big_buffer"); 1580155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1581169871Sgallatin "dropped_no_small_buffer", 1582169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1583169871Sgallatin &fw->dropped_no_small_buffer, 1584169871Sgallatin 0, mxge_handle_be32, 1585169871Sgallatin "I", "dropped_no_small_buffer"); 1586169871Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1587155852Sgallatin "dropped_overrun", 1588155852Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun, 1589159571Sgallatin 0, mxge_handle_be32, 1590155852Sgallatin "I", "dropped_overrun"); 1591155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1592169871Sgallatin "dropped_pause", 1593155852Sgallatin CTLTYPE_INT|CTLFLAG_RD, 1594169871Sgallatin &fw->dropped_pause, 1595159571Sgallatin 0, mxge_handle_be32, 1596169871Sgallatin "I", "dropped_pause"); 1597155852Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1598169871Sgallatin "dropped_runt", 1599169871Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt, 1600159571Sgallatin 0, mxge_handle_be32, 1601169871Sgallatin "I", "dropped_runt"); 1602155852Sgallatin 1603169995Sgallatin SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 1604169995Sgallatin "dropped_unicast_filtered", 1605169995Sgallatin CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered, 1606169995Sgallatin 0, mxge_handle_be32, 1607169995Sgallatin "I", "dropped_unicast_filtered"); 1608169995Sgallatin 1609159612Sgallatin /* verbose printing? */ 1610155852Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1611159612Sgallatin "verbose", 1612159612Sgallatin CTLFLAG_RW, &mxge_verbose, 1613159612Sgallatin 0, "verbose printing"); 1614155852Sgallatin 1615175365Sgallatin /* add counters exported for debugging from all slices */ 1616175365Sgallatin sysctl_ctx_init(&sc->slice_sysctl_ctx); 1617175365Sgallatin sc->slice_sysctl_tree = 1618175365Sgallatin SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO, 1619175365Sgallatin "slice", CTLFLAG_RD, 0, ""); 1620169840Sgallatin 1621175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 1622175365Sgallatin ss = &sc->ss[slice]; 1623175365Sgallatin sysctl_ctx_init(&ss->sysctl_ctx); 1624175365Sgallatin ctx = &ss->sysctl_ctx; 1625175365Sgallatin children = SYSCTL_CHILDREN(sc->slice_sysctl_tree); 1626175365Sgallatin sprintf(slice_num, "%d", slice); 1627175365Sgallatin ss->sysctl_tree = 1628175365Sgallatin SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num, 1629175365Sgallatin CTLFLAG_RD, 0, ""); 1630175365Sgallatin children = SYSCTL_CHILDREN(ss->sysctl_tree); 1631175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1632175365Sgallatin "rx_small_cnt", 1633175365Sgallatin CTLFLAG_RD, &ss->rx_small.cnt, 1634175365Sgallatin 0, "rx_small_cnt"); 1635175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1636175365Sgallatin "rx_big_cnt", 1637175365Sgallatin CTLFLAG_RD, &ss->rx_big.cnt, 1638175365Sgallatin 0, "rx_small_cnt"); 1639175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1640247133Sgallatin "lro_flushed", CTLFLAG_RD, &ss->lc.lro_flushed, 1641175365Sgallatin 0, "number of lro merge queues flushed"); 1642175365Sgallatin 1643175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1644247133Sgallatin "lro_bad_csum", CTLFLAG_RD, &ss->lc.lro_bad_csum, 1645247133Sgallatin 0, "number of bad csums preventing LRO"); 1646247133Sgallatin 1647247133Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1648247133Sgallatin "lro_queued", CTLFLAG_RD, &ss->lc.lro_queued, 1649175365Sgallatin 0, "number of frames appended to lro merge" 1650175365Sgallatin "queues"); 1651175365Sgallatin 1652191562Sgallatin#ifndef IFNET_BUF_RING 1653175365Sgallatin /* only transmit from slice 0 for now */ 1654175365Sgallatin if (slice > 0) 1655175365Sgallatin continue; 1656191562Sgallatin#endif 1657191562Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1658191562Sgallatin "tx_req", 1659191562Sgallatin CTLFLAG_RD, &ss->tx.req, 1660191562Sgallatin 0, "tx_req"); 1661175365Sgallatin 1662175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1663175365Sgallatin "tx_done", 1664175365Sgallatin CTLFLAG_RD, &ss->tx.done, 1665175365Sgallatin 0, "tx_done"); 1666175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1667175365Sgallatin "tx_pkt_done", 1668175365Sgallatin CTLFLAG_RD, &ss->tx.pkt_done, 1669175365Sgallatin 0, "tx_done"); 1670175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1671175365Sgallatin "tx_stall", 1672175365Sgallatin CTLFLAG_RD, &ss->tx.stall, 1673175365Sgallatin 0, "tx_stall"); 1674175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1675175365Sgallatin "tx_wake", 1676175365Sgallatin CTLFLAG_RD, &ss->tx.wake, 1677175365Sgallatin 0, "tx_wake"); 1678175365Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1679175365Sgallatin "tx_defrag", 1680175365Sgallatin CTLFLAG_RD, &ss->tx.defrag, 1681175365Sgallatin 0, "tx_defrag"); 1682191562Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1683191562Sgallatin "tx_queue_active", 1684191562Sgallatin CTLFLAG_RD, &ss->tx.queue_active, 1685191562Sgallatin 0, "tx_queue_active"); 1686191562Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1687191562Sgallatin "tx_activate", 1688191562Sgallatin CTLFLAG_RD, &ss->tx.activate, 1689191562Sgallatin 0, "tx_activate"); 1690191562Sgallatin SYSCTL_ADD_INT(ctx, children, OID_AUTO, 1691191562Sgallatin "tx_deactivate", 1692191562Sgallatin CTLFLAG_RD, &ss->tx.deactivate, 1693191562Sgallatin 0, "tx_deactivate"); 1694175365Sgallatin } 1695155852Sgallatin} 1696155852Sgallatin 1697155852Sgallatin/* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1698155852Sgallatin backwards one at a time and handle ring wraps */ 1699155852Sgallatin 1700155852Sgallatinstatic inline void 1701175365Sgallatinmxge_submit_req_backwards(mxge_tx_ring_t *tx, 1702155852Sgallatin mcp_kreq_ether_send_t *src, int cnt) 1703155852Sgallatin{ 1704155852Sgallatin int idx, starting_slot; 1705155852Sgallatin starting_slot = tx->req; 1706155852Sgallatin while (cnt > 1) { 1707155852Sgallatin cnt--; 1708155852Sgallatin idx = (starting_slot + cnt) & tx->mask; 1709159571Sgallatin mxge_pio_copy(&tx->lanai[idx], 1710159571Sgallatin &src[cnt], sizeof(*src)); 1711185255Sgallatin wmb(); 1712155852Sgallatin } 1713155852Sgallatin} 1714155852Sgallatin 1715155852Sgallatin/* 1716155852Sgallatin * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy 1717155852Sgallatin * at most 32 bytes at a time, so as to avoid involving the software 1718155852Sgallatin * pio handler in the nic. We re-write the first segment's flags 1719155852Sgallatin * to mark them valid only after writing the entire chain 1720155852Sgallatin */ 1721155852Sgallatin 1722155852Sgallatinstatic inline void 1723175365Sgallatinmxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, 1724155852Sgallatin int cnt) 1725155852Sgallatin{ 1726155852Sgallatin int idx, i; 1727155852Sgallatin uint32_t *src_ints; 1728155852Sgallatin volatile uint32_t *dst_ints; 1729155852Sgallatin mcp_kreq_ether_send_t *srcp; 1730155852Sgallatin volatile mcp_kreq_ether_send_t *dstp, *dst; 1731159612Sgallatin uint8_t last_flags; 1732155852Sgallatin 1733155852Sgallatin idx = tx->req & tx->mask; 1734155852Sgallatin 1735159612Sgallatin last_flags = src->flags; 1736159612Sgallatin src->flags = 0; 1737185255Sgallatin wmb(); 1738155852Sgallatin dst = dstp = &tx->lanai[idx]; 1739155852Sgallatin srcp = src; 1740155852Sgallatin 1741155852Sgallatin if ((idx + cnt) < tx->mask) { 1742155852Sgallatin for (i = 0; i < (cnt - 1); i += 2) { 1743159571Sgallatin mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 1744185255Sgallatin wmb(); /* force write every 32 bytes */ 1745155852Sgallatin srcp += 2; 1746155852Sgallatin dstp += 2; 1747155852Sgallatin } 1748155852Sgallatin } else { 1749155852Sgallatin /* submit all but the first request, and ensure 1750155852Sgallatin that it is submitted below */ 1751159571Sgallatin mxge_submit_req_backwards(tx, src, cnt); 1752155852Sgallatin i = 0; 1753155852Sgallatin } 1754155852Sgallatin if (i < cnt) { 1755155852Sgallatin /* submit the first request */ 1756159571Sgallatin mxge_pio_copy(dstp, srcp, sizeof(*src)); 1757185255Sgallatin wmb(); /* barrier before setting valid flag */ 1758155852Sgallatin } 1759155852Sgallatin 1760155852Sgallatin /* re-write the last 32-bits with the valid flags */ 1761159612Sgallatin src->flags = last_flags; 1762155852Sgallatin src_ints = (uint32_t *)src; 1763155852Sgallatin src_ints+=3; 1764155852Sgallatin dst_ints = (volatile uint32_t *)dst; 1765155852Sgallatin dst_ints+=3; 1766155852Sgallatin *dst_ints = *src_ints; 1767155852Sgallatin tx->req += cnt; 1768185255Sgallatin wmb(); 1769155852Sgallatin} 1770155852Sgallatin 1771247011Sgallatinstatic int 1772247011Sgallatinmxge_parse_tx(struct mxge_slice_state *ss, struct mbuf *m, 1773247011Sgallatin struct mxge_pkt_info *pi) 1774247011Sgallatin{ 1775247011Sgallatin struct ether_vlan_header *eh; 1776247011Sgallatin uint16_t etype; 1777247011Sgallatin int tso = m->m_pkthdr.csum_flags & (CSUM_TSO); 1778247011Sgallatin#if IFCAP_TSO6 && defined(INET6) 1779247011Sgallatin int nxt; 1780247011Sgallatin#endif 1781247011Sgallatin 1782247011Sgallatin eh = mtod(m, struct ether_vlan_header *); 1783247011Sgallatin if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1784247011Sgallatin etype = ntohs(eh->evl_proto); 1785247011Sgallatin pi->ip_off = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1786247011Sgallatin } else { 1787247011Sgallatin etype = ntohs(eh->evl_encap_proto); 1788247011Sgallatin pi->ip_off = ETHER_HDR_LEN; 1789247011Sgallatin } 1790247011Sgallatin 1791247011Sgallatin switch (etype) { 1792247011Sgallatin case ETHERTYPE_IP: 1793247011Sgallatin /* 1794247011Sgallatin * ensure ip header is in first mbuf, copy it to a 1795247011Sgallatin * scratch buffer if not 1796247011Sgallatin */ 1797247011Sgallatin pi->ip = (struct ip *)(m->m_data + pi->ip_off); 1798247011Sgallatin pi->ip6 = NULL; 1799247011Sgallatin if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip))) { 1800247011Sgallatin m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip), 1801247011Sgallatin ss->scratch); 1802247011Sgallatin pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 1803247011Sgallatin } 1804247011Sgallatin pi->ip_hlen = pi->ip->ip_hl << 2; 1805247011Sgallatin if (!tso) 1806247011Sgallatin return 0; 1807247011Sgallatin 1808247011Sgallatin if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 1809247011Sgallatin sizeof(struct tcphdr))) { 1810247011Sgallatin m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 1811247011Sgallatin sizeof(struct tcphdr), ss->scratch); 1812247011Sgallatin pi->ip = (struct ip *)(ss->scratch + pi->ip_off); 1813247011Sgallatin } 1814247011Sgallatin pi->tcp = (struct tcphdr *)((char *)pi->ip + pi->ip_hlen); 1815247011Sgallatin break; 1816247011Sgallatin#if IFCAP_TSO6 && defined(INET6) 1817247011Sgallatin case ETHERTYPE_IPV6: 1818247011Sgallatin pi->ip6 = (struct ip6_hdr *)(m->m_data + pi->ip_off); 1819247011Sgallatin if (__predict_false(m->m_len < pi->ip_off + sizeof(*pi->ip6))) { 1820247011Sgallatin m_copydata(m, 0, pi->ip_off + sizeof(*pi->ip6), 1821247011Sgallatin ss->scratch); 1822247011Sgallatin pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 1823247011Sgallatin } 1824247011Sgallatin nxt = 0; 1825247011Sgallatin pi->ip_hlen = ip6_lasthdr(m, pi->ip_off, IPPROTO_IPV6, &nxt); 1826247011Sgallatin pi->ip_hlen -= pi->ip_off; 1827247011Sgallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 1828247011Sgallatin return EINVAL; 1829247011Sgallatin 1830247011Sgallatin if (!tso) 1831247011Sgallatin return 0; 1832247011Sgallatin 1833247011Sgallatin if (pi->ip_off + pi->ip_hlen > ss->sc->max_tso6_hlen) 1834247011Sgallatin return EINVAL; 1835247011Sgallatin 1836247011Sgallatin if (__predict_false(m->m_len < pi->ip_off + pi->ip_hlen + 1837247011Sgallatin sizeof(struct tcphdr))) { 1838247011Sgallatin m_copydata(m, 0, pi->ip_off + pi->ip_hlen + 1839247011Sgallatin sizeof(struct tcphdr), ss->scratch); 1840247011Sgallatin pi->ip6 = (struct ip6_hdr *)(ss->scratch + pi->ip_off); 1841247011Sgallatin } 1842247011Sgallatin pi->tcp = (struct tcphdr *)((char *)pi->ip6 + pi->ip_hlen); 1843247011Sgallatin break; 1844247011Sgallatin#endif 1845247011Sgallatin default: 1846247011Sgallatin return EINVAL; 1847247011Sgallatin } 1848247011Sgallatin return 0; 1849247011Sgallatin} 1850247011Sgallatin 1851176261Sgallatin#if IFCAP_TSO4 1852176261Sgallatin 1853155852Sgallatinstatic void 1854175365Sgallatinmxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m, 1855247011Sgallatin int busdma_seg_cnt, struct mxge_pkt_info *pi) 1856162322Sgallatin{ 1857175365Sgallatin mxge_tx_ring_t *tx; 1858162322Sgallatin mcp_kreq_ether_send_t *req; 1859162322Sgallatin bus_dma_segment_t *seg; 1860162322Sgallatin uint32_t low, high_swapped; 1861162322Sgallatin int len, seglen, cum_len, cum_len_next; 1862162322Sgallatin int next_is_first, chop, cnt, rdma_count, small; 1863247011Sgallatin uint16_t pseudo_hdr_offset, cksum_offset, mss, sum; 1864162322Sgallatin uint8_t flags, flags_next; 1865162322Sgallatin static int once; 1866175365Sgallatin 1867162322Sgallatin mss = m->m_pkthdr.tso_segsz; 1868162322Sgallatin 1869162322Sgallatin /* negative cum_len signifies to the 1870162322Sgallatin * send loop that we are still in the 1871162322Sgallatin * header portion of the TSO packet. 1872162322Sgallatin */ 1873162322Sgallatin 1874247011Sgallatin cksum_offset = pi->ip_off + pi->ip_hlen; 1875247011Sgallatin cum_len = -(cksum_offset + (pi->tcp->th_off << 2)); 1876162322Sgallatin 1877162322Sgallatin /* TSO implies checksum offload on this hardware */ 1878247011Sgallatin if (__predict_false((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) == 0)) { 1879215686Sgallatin /* 1880215686Sgallatin * If packet has full TCP csum, replace it with pseudo hdr 1881215686Sgallatin * sum that the NIC expects, otherwise the NIC will emit 1882215686Sgallatin * packets with bad TCP checksums. 1883215686Sgallatin */ 1884215686Sgallatin m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 1885247011Sgallatin if (pi->ip6) { 1886247011Sgallatin#if (CSUM_TCP_IPV6 != 0) && defined(INET6) 1887247011Sgallatin m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6; 1888247011Sgallatin sum = in6_cksum_pseudo(pi->ip6, 1889247011Sgallatin m->m_pkthdr.len - cksum_offset, 1890247011Sgallatin IPPROTO_TCP, 0); 1891247011Sgallatin#endif 1892247011Sgallatin } else { 1893247152Sgallatin#ifdef INET 1894247011Sgallatin m->m_pkthdr.csum_flags |= CSUM_TCP; 1895247011Sgallatin sum = in_pseudo(pi->ip->ip_src.s_addr, 1896247011Sgallatin pi->ip->ip_dst.s_addr, 1897247011Sgallatin htons(IPPROTO_TCP + (m->m_pkthdr.len - 1898247011Sgallatin cksum_offset))); 1899247152Sgallatin#endif 1900247011Sgallatin } 1901247011Sgallatin m_copyback(m, offsetof(struct tcphdr, th_sum) + 1902247011Sgallatin cksum_offset, sizeof(sum), (caddr_t)&sum); 1903215686Sgallatin } 1904162322Sgallatin flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST; 1905162322Sgallatin 1906162322Sgallatin 1907162322Sgallatin /* for TSO, pseudo_hdr_offset holds mss. 1908162322Sgallatin * The firmware figures out where to put 1909162322Sgallatin * the checksum by parsing the header. */ 1910162322Sgallatin pseudo_hdr_offset = htobe16(mss); 1911162322Sgallatin 1912247011Sgallatin if (pi->ip6) { 1913247011Sgallatin /* 1914247011Sgallatin * for IPv6 TSO, the "checksum offset" is re-purposed 1915247011Sgallatin * to store the TCP header len 1916247011Sgallatin */ 1917247011Sgallatin cksum_offset = (pi->tcp->th_off << 2); 1918247011Sgallatin } 1919247011Sgallatin 1920175365Sgallatin tx = &ss->tx; 1921162322Sgallatin req = tx->req_list; 1922162322Sgallatin seg = tx->seg_list; 1923162322Sgallatin cnt = 0; 1924162322Sgallatin rdma_count = 0; 1925162322Sgallatin /* "rdma_count" is the number of RDMAs belonging to the 1926162322Sgallatin * current packet BEFORE the current send request. For 1927162322Sgallatin * non-TSO packets, this is equal to "count". 1928162322Sgallatin * For TSO packets, rdma_count needs to be reset 1929162322Sgallatin * to 0 after a segment cut. 1930162322Sgallatin * 1931162322Sgallatin * The rdma_count field of the send request is 1932162322Sgallatin * the number of RDMAs of the packet starting at 1933162322Sgallatin * that request. For TSO send requests with one ore more cuts 1934162322Sgallatin * in the middle, this is the number of RDMAs starting 1935162322Sgallatin * after the last cut in the request. All previous 1936162322Sgallatin * segments before the last cut implicitly have 1 RDMA. 1937162322Sgallatin * 1938162322Sgallatin * Since the number of RDMAs is not known beforehand, 1939162322Sgallatin * it must be filled-in retroactively - after each 1940162322Sgallatin * segmentation cut or at the end of the entire packet. 1941162322Sgallatin */ 1942162322Sgallatin 1943162322Sgallatin while (busdma_seg_cnt) { 1944162322Sgallatin /* Break the busdma segment up into pieces*/ 1945162322Sgallatin low = MXGE_LOWPART_TO_U32(seg->ds_addr); 1946162322Sgallatin high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 1947168298Sgallatin len = seg->ds_len; 1948162322Sgallatin 1949162322Sgallatin while (len) { 1950162322Sgallatin flags_next = flags & ~MXGEFW_FLAGS_FIRST; 1951168298Sgallatin seglen = len; 1952162322Sgallatin cum_len_next = cum_len + seglen; 1953162322Sgallatin (req-rdma_count)->rdma_count = rdma_count + 1; 1954162322Sgallatin if (__predict_true(cum_len >= 0)) { 1955162322Sgallatin /* payload */ 1956162322Sgallatin chop = (cum_len_next > mss); 1957162322Sgallatin cum_len_next = cum_len_next % mss; 1958162322Sgallatin next_is_first = (cum_len_next == 0); 1959162322Sgallatin flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 1960162322Sgallatin flags_next |= next_is_first * 1961162322Sgallatin MXGEFW_FLAGS_FIRST; 1962162322Sgallatin rdma_count |= -(chop | next_is_first); 1963162322Sgallatin rdma_count += chop & !next_is_first; 1964162322Sgallatin } else if (cum_len_next >= 0) { 1965162322Sgallatin /* header ends */ 1966162322Sgallatin rdma_count = -1; 1967162322Sgallatin cum_len_next = 0; 1968162322Sgallatin seglen = -cum_len; 1969162322Sgallatin small = (mss <= MXGEFW_SEND_SMALL_SIZE); 1970162322Sgallatin flags_next = MXGEFW_FLAGS_TSO_PLD | 1971162322Sgallatin MXGEFW_FLAGS_FIRST | 1972162322Sgallatin (small * MXGEFW_FLAGS_SMALL); 1973162322Sgallatin } 1974162322Sgallatin 1975162322Sgallatin req->addr_high = high_swapped; 1976162322Sgallatin req->addr_low = htobe32(low); 1977162322Sgallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 1978162322Sgallatin req->pad = 0; 1979162322Sgallatin req->rdma_count = 1; 1980162322Sgallatin req->length = htobe16(seglen); 1981162322Sgallatin req->cksum_offset = cksum_offset; 1982162322Sgallatin req->flags = flags | ((cum_len & 1) * 1983162322Sgallatin MXGEFW_FLAGS_ALIGN_ODD); 1984162322Sgallatin low += seglen; 1985162322Sgallatin len -= seglen; 1986162322Sgallatin cum_len = cum_len_next; 1987162322Sgallatin flags = flags_next; 1988162322Sgallatin req++; 1989162322Sgallatin cnt++; 1990162322Sgallatin rdma_count++; 1991247011Sgallatin if (cksum_offset != 0 && !pi->ip6) { 1992247011Sgallatin if (__predict_false(cksum_offset > seglen)) 1993247011Sgallatin cksum_offset -= seglen; 1994247011Sgallatin else 1995247011Sgallatin cksum_offset = 0; 1996247011Sgallatin } 1997169871Sgallatin if (__predict_false(cnt > tx->max_desc)) 1998162322Sgallatin goto drop; 1999162322Sgallatin } 2000162322Sgallatin busdma_seg_cnt--; 2001162322Sgallatin seg++; 2002162322Sgallatin } 2003162322Sgallatin (req-rdma_count)->rdma_count = rdma_count; 2004162322Sgallatin 2005162322Sgallatin do { 2006162322Sgallatin req--; 2007162322Sgallatin req->flags |= MXGEFW_FLAGS_TSO_LAST; 2008162322Sgallatin } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST))); 2009162322Sgallatin 2010162322Sgallatin tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 2011169871Sgallatin mxge_submit_req(tx, tx->req_list, cnt); 2012191562Sgallatin#ifdef IFNET_BUF_RING 2013191562Sgallatin if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2014191562Sgallatin /* tell the NIC to start polling this slice */ 2015191562Sgallatin *tx->send_go = 1; 2016191562Sgallatin tx->queue_active = 1; 2017191562Sgallatin tx->activate++; 2018191562Sgallatin wmb(); 2019191562Sgallatin } 2020191562Sgallatin#endif 2021162322Sgallatin return; 2022162322Sgallatin 2023162322Sgallatindrop: 2024168298Sgallatin bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map); 2025162322Sgallatin m_freem(m); 2026191562Sgallatin ss->oerrors++; 2027162322Sgallatin if (!once) { 2028169871Sgallatin printf("tx->max_desc exceeded via TSO!\n"); 2029169871Sgallatin printf("mss = %d, %ld, %d!\n", mss, 2030169871Sgallatin (long)seg - (long)tx->seg_list, tx->max_desc); 2031162322Sgallatin once = 1; 2032162322Sgallatin } 2033162322Sgallatin return; 2034162322Sgallatin 2035162322Sgallatin} 2036162322Sgallatin 2037176261Sgallatin#endif /* IFCAP_TSO4 */ 2038176261Sgallatin 2039176261Sgallatin#ifdef MXGE_NEW_VLAN_API 2040169905Sgallatin/* 2041169905Sgallatin * We reproduce the software vlan tag insertion from 2042169905Sgallatin * net/if_vlan.c:vlan_start() here so that we can advertise "hardware" 2043169905Sgallatin * vlan tag insertion. We need to advertise this in order to have the 2044169905Sgallatin * vlan interface respect our csum offload flags. 2045169905Sgallatin */ 2046169905Sgallatinstatic struct mbuf * 2047169905Sgallatinmxge_vlan_tag_insert(struct mbuf *m) 2048169905Sgallatin{ 2049169905Sgallatin struct ether_vlan_header *evl; 2050169905Sgallatin 2051243857Sglebius M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_NOWAIT); 2052169905Sgallatin if (__predict_false(m == NULL)) 2053169905Sgallatin return NULL; 2054169905Sgallatin if (m->m_len < sizeof(*evl)) { 2055169905Sgallatin m = m_pullup(m, sizeof(*evl)); 2056169905Sgallatin if (__predict_false(m == NULL)) 2057169905Sgallatin return NULL; 2058169905Sgallatin } 2059169905Sgallatin /* 2060169905Sgallatin * Transform the Ethernet header into an Ethernet header 2061169905Sgallatin * with 802.1Q encapsulation. 2062169905Sgallatin */ 2063169905Sgallatin evl = mtod(m, struct ether_vlan_header *); 2064169905Sgallatin bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN, 2065169905Sgallatin (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN); 2066169905Sgallatin evl->evl_encap_proto = htons(ETHERTYPE_VLAN); 2067169905Sgallatin evl->evl_tag = htons(m->m_pkthdr.ether_vtag); 2068169905Sgallatin m->m_flags &= ~M_VLANTAG; 2069169905Sgallatin return m; 2070169905Sgallatin} 2071176261Sgallatin#endif /* MXGE_NEW_VLAN_API */ 2072169905Sgallatin 2073162322Sgallatinstatic void 2074175365Sgallatinmxge_encap(struct mxge_slice_state *ss, struct mbuf *m) 2075155852Sgallatin{ 2076247011Sgallatin struct mxge_pkt_info pi = {0,0,0,0}; 2077175365Sgallatin mxge_softc_t *sc; 2078155852Sgallatin mcp_kreq_ether_send_t *req; 2079155852Sgallatin bus_dma_segment_t *seg; 2080155852Sgallatin struct mbuf *m_tmp; 2081155852Sgallatin struct ifnet *ifp; 2082175365Sgallatin mxge_tx_ring_t *tx; 2083247011Sgallatin int cnt, cum_len, err, i, idx, odd_flag; 2084162322Sgallatin uint16_t pseudo_hdr_offset; 2085162322Sgallatin uint8_t flags, cksum_offset; 2086155852Sgallatin 2087155852Sgallatin 2088175365Sgallatin sc = ss->sc; 2089155852Sgallatin ifp = sc->ifp; 2090175365Sgallatin tx = &ss->tx; 2091155852Sgallatin 2092176261Sgallatin#ifdef MXGE_NEW_VLAN_API 2093169905Sgallatin if (m->m_flags & M_VLANTAG) { 2094169905Sgallatin m = mxge_vlan_tag_insert(m); 2095169905Sgallatin if (__predict_false(m == NULL)) 2096247011Sgallatin goto drop_without_m; 2097247011Sgallatin } 2098247011Sgallatin#endif 2099247011Sgallatin if (m->m_pkthdr.csum_flags & 2100247011Sgallatin (CSUM_TSO | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 2101247011Sgallatin if (mxge_parse_tx(ss, m, &pi)) 2102169905Sgallatin goto drop; 2103169905Sgallatin } 2104247011Sgallatin 2105155852Sgallatin /* (try to) map the frame for DMA */ 2106155852Sgallatin idx = tx->req & tx->mask; 2107155852Sgallatin err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map, 2108162322Sgallatin m, tx->seg_list, &cnt, 2109155852Sgallatin BUS_DMA_NOWAIT); 2110169871Sgallatin if (__predict_false(err == EFBIG)) { 2111155852Sgallatin /* Too many segments in the chain. Try 2112155852Sgallatin to defrag */ 2113155852Sgallatin m_tmp = m_defrag(m, M_NOWAIT); 2114155852Sgallatin if (m_tmp == NULL) { 2115155852Sgallatin goto drop; 2116155852Sgallatin } 2117175365Sgallatin ss->tx.defrag++; 2118155852Sgallatin m = m_tmp; 2119155852Sgallatin err = bus_dmamap_load_mbuf_sg(tx->dmat, 2120155852Sgallatin tx->info[idx].map, 2121162322Sgallatin m, tx->seg_list, &cnt, 2122155852Sgallatin BUS_DMA_NOWAIT); 2123155852Sgallatin } 2124169871Sgallatin if (__predict_false(err != 0)) { 2125162322Sgallatin device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d" 2126162322Sgallatin " packet len = %d\n", err, m->m_pkthdr.len); 2127155852Sgallatin goto drop; 2128155852Sgallatin } 2129155852Sgallatin bus_dmamap_sync(tx->dmat, tx->info[idx].map, 2130155852Sgallatin BUS_DMASYNC_PREWRITE); 2131159612Sgallatin tx->info[idx].m = m; 2132159612Sgallatin 2133176261Sgallatin#if IFCAP_TSO4 2134162322Sgallatin /* TSO is different enough, we handle it in another routine */ 2135162322Sgallatin if (m->m_pkthdr.csum_flags & (CSUM_TSO)) { 2136247011Sgallatin mxge_encap_tso(ss, m, cnt, &pi); 2137162322Sgallatin return; 2138162322Sgallatin } 2139176261Sgallatin#endif 2140162322Sgallatin 2141155852Sgallatin req = tx->req_list; 2142155852Sgallatin cksum_offset = 0; 2143159612Sgallatin pseudo_hdr_offset = 0; 2144159612Sgallatin flags = MXGEFW_FLAGS_NO_TSO; 2145155852Sgallatin 2146155852Sgallatin /* checksum offloading? */ 2147247011Sgallatin if (m->m_pkthdr.csum_flags & 2148247011Sgallatin (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6)) { 2149162322Sgallatin /* ensure ip header is in first mbuf, copy 2150162322Sgallatin it to a scratch buffer if not */ 2151247011Sgallatin cksum_offset = pi.ip_off + pi.ip_hlen; 2152155852Sgallatin pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data; 2153159612Sgallatin pseudo_hdr_offset = htobe16(pseudo_hdr_offset); 2154155852Sgallatin req->cksum_offset = cksum_offset; 2155159612Sgallatin flags |= MXGEFW_FLAGS_CKSUM; 2156162322Sgallatin odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2157162322Sgallatin } else { 2158162322Sgallatin odd_flag = 0; 2159155852Sgallatin } 2160159612Sgallatin if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE) 2161159612Sgallatin flags |= MXGEFW_FLAGS_SMALL; 2162155852Sgallatin 2163155852Sgallatin /* convert segments into a request list */ 2164155852Sgallatin cum_len = 0; 2165162322Sgallatin seg = tx->seg_list; 2166159612Sgallatin req->flags = MXGEFW_FLAGS_FIRST; 2167155852Sgallatin for (i = 0; i < cnt; i++) { 2168155852Sgallatin req->addr_low = 2169159571Sgallatin htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2170155852Sgallatin req->addr_high = 2171159571Sgallatin htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2172155852Sgallatin req->length = htobe16(seg->ds_len); 2173155852Sgallatin req->cksum_offset = cksum_offset; 2174155852Sgallatin if (cksum_offset > seg->ds_len) 2175155852Sgallatin cksum_offset -= seg->ds_len; 2176155852Sgallatin else 2177155852Sgallatin cksum_offset = 0; 2178159612Sgallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 2179159612Sgallatin req->pad = 0; /* complete solid 16-byte block */ 2180159612Sgallatin req->rdma_count = 1; 2181162322Sgallatin req->flags |= flags | ((cum_len & 1) * odd_flag); 2182155852Sgallatin cum_len += seg->ds_len; 2183155852Sgallatin seg++; 2184155852Sgallatin req++; 2185155852Sgallatin req->flags = 0; 2186155852Sgallatin } 2187155852Sgallatin req--; 2188155852Sgallatin /* pad runts to 60 bytes */ 2189155852Sgallatin if (cum_len < 60) { 2190155852Sgallatin req++; 2191155852Sgallatin req->addr_low = 2192159571Sgallatin htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr)); 2193155852Sgallatin req->addr_high = 2194159571Sgallatin htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr)); 2195155852Sgallatin req->length = htobe16(60 - cum_len); 2196159612Sgallatin req->cksum_offset = 0; 2197159612Sgallatin req->pseudo_hdr_offset = pseudo_hdr_offset; 2198159612Sgallatin req->pad = 0; /* complete solid 16-byte block */ 2199159612Sgallatin req->rdma_count = 1; 2200162322Sgallatin req->flags |= flags | ((cum_len & 1) * odd_flag); 2201155852Sgallatin cnt++; 2202155852Sgallatin } 2203159612Sgallatin 2204159612Sgallatin tx->req_list[0].rdma_count = cnt; 2205159612Sgallatin#if 0 2206159612Sgallatin /* print what the firmware will see */ 2207159612Sgallatin for (i = 0; i < cnt; i++) { 2208159612Sgallatin printf("%d: addr: 0x%x 0x%x len:%d pso%d," 2209159612Sgallatin "cso:%d, flags:0x%x, rdma:%d\n", 2210159612Sgallatin i, (int)ntohl(tx->req_list[i].addr_high), 2211159612Sgallatin (int)ntohl(tx->req_list[i].addr_low), 2212159612Sgallatin (int)ntohs(tx->req_list[i].length), 2213159612Sgallatin (int)ntohs(tx->req_list[i].pseudo_hdr_offset), 2214159612Sgallatin tx->req_list[i].cksum_offset, tx->req_list[i].flags, 2215159612Sgallatin tx->req_list[i].rdma_count); 2216159612Sgallatin } 2217159612Sgallatin printf("--------------\n"); 2218159612Sgallatin#endif 2219159612Sgallatin tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; 2220169871Sgallatin mxge_submit_req(tx, tx->req_list, cnt); 2221191562Sgallatin#ifdef IFNET_BUF_RING 2222191562Sgallatin if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { 2223191562Sgallatin /* tell the NIC to start polling this slice */ 2224191562Sgallatin *tx->send_go = 1; 2225191562Sgallatin tx->queue_active = 1; 2226191562Sgallatin tx->activate++; 2227191562Sgallatin wmb(); 2228191562Sgallatin } 2229191562Sgallatin#endif 2230155852Sgallatin return; 2231155852Sgallatin 2232155852Sgallatindrop: 2233155852Sgallatin m_freem(m); 2234247011Sgallatindrop_without_m: 2235191562Sgallatin ss->oerrors++; 2236155852Sgallatin return; 2237155852Sgallatin} 2238155852Sgallatin 2239191562Sgallatin#ifdef IFNET_BUF_RING 2240191562Sgallatinstatic void 2241191562Sgallatinmxge_qflush(struct ifnet *ifp) 2242191562Sgallatin{ 2243191562Sgallatin mxge_softc_t *sc = ifp->if_softc; 2244191562Sgallatin mxge_tx_ring_t *tx; 2245191562Sgallatin struct mbuf *m; 2246191562Sgallatin int slice; 2247155852Sgallatin 2248191562Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 2249191562Sgallatin tx = &sc->ss[slice].tx; 2250191562Sgallatin mtx_lock(&tx->mtx); 2251191562Sgallatin while ((m = buf_ring_dequeue_sc(tx->br)) != NULL) 2252191562Sgallatin m_freem(m); 2253191562Sgallatin mtx_unlock(&tx->mtx); 2254191562Sgallatin } 2255191562Sgallatin if_qflush(ifp); 2256191562Sgallatin} 2257159623Sgallatin 2258191562Sgallatinstatic inline void 2259191562Sgallatinmxge_start_locked(struct mxge_slice_state *ss) 2260191562Sgallatin{ 2261191562Sgallatin mxge_softc_t *sc; 2262191562Sgallatin struct mbuf *m; 2263191562Sgallatin struct ifnet *ifp; 2264191562Sgallatin mxge_tx_ring_t *tx; 2265159623Sgallatin 2266191562Sgallatin sc = ss->sc; 2267191562Sgallatin ifp = sc->ifp; 2268191562Sgallatin tx = &ss->tx; 2269191562Sgallatin 2270191562Sgallatin while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 2271191562Sgallatin m = drbr_dequeue(ifp, tx->br); 2272191562Sgallatin if (m == NULL) { 2273191562Sgallatin return; 2274191562Sgallatin } 2275191562Sgallatin /* let BPF see it */ 2276191562Sgallatin BPF_MTAP(ifp, m); 2277191562Sgallatin 2278191562Sgallatin /* give it to the nic */ 2279191562Sgallatin mxge_encap(ss, m); 2280191562Sgallatin } 2281191562Sgallatin /* ran out of transmit slots */ 2282191562Sgallatin if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0) 2283191562Sgallatin && (!drbr_empty(ifp, tx->br))) { 2284191562Sgallatin ss->if_drv_flags |= IFF_DRV_OACTIVE; 2285191562Sgallatin tx->stall++; 2286191562Sgallatin } 2287191562Sgallatin} 2288191562Sgallatin 2289191562Sgallatinstatic int 2290191562Sgallatinmxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m) 2291191562Sgallatin{ 2292191562Sgallatin mxge_softc_t *sc; 2293191562Sgallatin struct ifnet *ifp; 2294191562Sgallatin mxge_tx_ring_t *tx; 2295191562Sgallatin int err; 2296191562Sgallatin 2297191562Sgallatin sc = ss->sc; 2298191562Sgallatin ifp = sc->ifp; 2299191562Sgallatin tx = &ss->tx; 2300191562Sgallatin 2301191562Sgallatin if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 2302191562Sgallatin IFF_DRV_RUNNING) { 2303191562Sgallatin err = drbr_enqueue(ifp, tx->br, m); 2304191562Sgallatin return (err); 2305191562Sgallatin } 2306191562Sgallatin 2307203834Smlaier if (!drbr_needs_enqueue(ifp, tx->br) && 2308191562Sgallatin ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) { 2309191562Sgallatin /* let BPF see it */ 2310191562Sgallatin BPF_MTAP(ifp, m); 2311191562Sgallatin /* give it to the nic */ 2312191562Sgallatin mxge_encap(ss, m); 2313191562Sgallatin } else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) { 2314191562Sgallatin return (err); 2315191562Sgallatin } 2316191562Sgallatin if (!drbr_empty(ifp, tx->br)) 2317191562Sgallatin mxge_start_locked(ss); 2318191562Sgallatin return (0); 2319191562Sgallatin} 2320191562Sgallatin 2321191562Sgallatinstatic int 2322191562Sgallatinmxge_transmit(struct ifnet *ifp, struct mbuf *m) 2323191562Sgallatin{ 2324191562Sgallatin mxge_softc_t *sc = ifp->if_softc; 2325191562Sgallatin struct mxge_slice_state *ss; 2326191562Sgallatin mxge_tx_ring_t *tx; 2327191562Sgallatin int err = 0; 2328191562Sgallatin int slice; 2329191562Sgallatin 2330191562Sgallatin slice = m->m_pkthdr.flowid; 2331191562Sgallatin slice &= (sc->num_slices - 1); /* num_slices always power of 2 */ 2332191562Sgallatin 2333191562Sgallatin ss = &sc->ss[slice]; 2334191562Sgallatin tx = &ss->tx; 2335191562Sgallatin 2336191562Sgallatin if (mtx_trylock(&tx->mtx)) { 2337191562Sgallatin err = mxge_transmit_locked(ss, m); 2338191562Sgallatin mtx_unlock(&tx->mtx); 2339191562Sgallatin } else { 2340191562Sgallatin err = drbr_enqueue(ifp, tx->br, m); 2341191562Sgallatin } 2342191562Sgallatin 2343191562Sgallatin return (err); 2344191562Sgallatin} 2345191562Sgallatin 2346191562Sgallatin#else 2347191562Sgallatin 2348159623Sgallatinstatic inline void 2349175365Sgallatinmxge_start_locked(struct mxge_slice_state *ss) 2350155852Sgallatin{ 2351175365Sgallatin mxge_softc_t *sc; 2352155852Sgallatin struct mbuf *m; 2353155852Sgallatin struct ifnet *ifp; 2354175365Sgallatin mxge_tx_ring_t *tx; 2355155852Sgallatin 2356175365Sgallatin sc = ss->sc; 2357155852Sgallatin ifp = sc->ifp; 2358175365Sgallatin tx = &ss->tx; 2359169871Sgallatin while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { 2360155852Sgallatin IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 2361159623Sgallatin if (m == NULL) { 2362159623Sgallatin return; 2363159623Sgallatin } 2364155852Sgallatin /* let BPF see it */ 2365155852Sgallatin BPF_MTAP(ifp, m); 2366155852Sgallatin 2367155852Sgallatin /* give it to the nic */ 2368175365Sgallatin mxge_encap(ss, m); 2369155852Sgallatin } 2370159623Sgallatin /* ran out of transmit slots */ 2371166345Sgallatin if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { 2372166345Sgallatin sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE; 2373169871Sgallatin tx->stall++; 2374166345Sgallatin } 2375155852Sgallatin} 2376191562Sgallatin#endif 2377155852Sgallatinstatic void 2378159571Sgallatinmxge_start(struct ifnet *ifp) 2379155852Sgallatin{ 2380159571Sgallatin mxge_softc_t *sc = ifp->if_softc; 2381175365Sgallatin struct mxge_slice_state *ss; 2382155852Sgallatin 2383175365Sgallatin /* only use the first slice for now */ 2384175365Sgallatin ss = &sc->ss[0]; 2385175365Sgallatin mtx_lock(&ss->tx.mtx); 2386175365Sgallatin mxge_start_locked(ss); 2387175365Sgallatin mtx_unlock(&ss->tx.mtx); 2388155852Sgallatin} 2389155852Sgallatin 2390159612Sgallatin/* 2391159612Sgallatin * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy 2392159612Sgallatin * at most 32 bytes at a time, so as to avoid involving the software 2393159612Sgallatin * pio handler in the nic. We re-write the first segment's low 2394159612Sgallatin * DMA address to mark it valid only after we write the entire chunk 2395159612Sgallatin * in a burst 2396159612Sgallatin */ 2397159612Sgallatinstatic inline void 2398159612Sgallatinmxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst, 2399159612Sgallatin mcp_kreq_ether_recv_t *src) 2400159612Sgallatin{ 2401159612Sgallatin uint32_t low; 2402159612Sgallatin 2403159612Sgallatin low = src->addr_low; 2404159612Sgallatin src->addr_low = 0xffffffff; 2405164751Sgallatin mxge_pio_copy(dst, src, 4 * sizeof (*src)); 2406185255Sgallatin wmb(); 2407164751Sgallatin mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); 2408185255Sgallatin wmb(); 2409167942Sgallatin src->addr_low = low; 2410159612Sgallatin dst->addr_low = low; 2411185255Sgallatin wmb(); 2412159612Sgallatin} 2413159612Sgallatin 2414155852Sgallatinstatic int 2415175365Sgallatinmxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2416155852Sgallatin{ 2417155852Sgallatin bus_dma_segment_t seg; 2418155852Sgallatin struct mbuf *m; 2419175365Sgallatin mxge_rx_ring_t *rx = &ss->rx_small; 2420155852Sgallatin int cnt, err; 2421155852Sgallatin 2422243857Sglebius m = m_gethdr(M_NOWAIT, MT_DATA); 2423155852Sgallatin if (m == NULL) { 2424155852Sgallatin rx->alloc_fail++; 2425155852Sgallatin err = ENOBUFS; 2426155852Sgallatin goto done; 2427155852Sgallatin } 2428155852Sgallatin m->m_len = MHLEN; 2429155852Sgallatin err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2430155852Sgallatin &seg, &cnt, BUS_DMA_NOWAIT); 2431155852Sgallatin if (err != 0) { 2432155852Sgallatin m_free(m); 2433155852Sgallatin goto done; 2434155852Sgallatin } 2435155852Sgallatin rx->info[idx].m = m; 2436155852Sgallatin rx->shadow[idx].addr_low = 2437159571Sgallatin htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr)); 2438155852Sgallatin rx->shadow[idx].addr_high = 2439159571Sgallatin htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr)); 2440155852Sgallatin 2441155852Sgallatindone: 2442169871Sgallatin if ((idx & 7) == 7) 2443169871Sgallatin mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]); 2444155852Sgallatin return err; 2445155852Sgallatin} 2446155852Sgallatin 2447155852Sgallatinstatic int 2448175365Sgallatinmxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) 2449155852Sgallatin{ 2450169840Sgallatin bus_dma_segment_t seg[3]; 2451155852Sgallatin struct mbuf *m; 2452175365Sgallatin mxge_rx_ring_t *rx = &ss->rx_big; 2453169840Sgallatin int cnt, err, i; 2454155852Sgallatin 2455243857Sglebius m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx->cl_size); 2456155852Sgallatin if (m == NULL) { 2457155852Sgallatin rx->alloc_fail++; 2458155852Sgallatin err = ENOBUFS; 2459155852Sgallatin goto done; 2460155852Sgallatin } 2461193250Sgallatin m->m_len = rx->mlen; 2462155852Sgallatin err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, 2463169840Sgallatin seg, &cnt, BUS_DMA_NOWAIT); 2464155852Sgallatin if (err != 0) { 2465155852Sgallatin m_free(m); 2466155852Sgallatin goto done; 2467155852Sgallatin } 2468155852Sgallatin rx->info[idx].m = m; 2469175579Sgallatin rx->shadow[idx].addr_low = 2470175579Sgallatin htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); 2471175579Sgallatin rx->shadow[idx].addr_high = 2472175579Sgallatin htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); 2473155852Sgallatin 2474175579Sgallatin#if MXGE_VIRT_JUMBOS 2475175579Sgallatin for (i = 1; i < cnt; i++) { 2476169840Sgallatin rx->shadow[idx + i].addr_low = 2477169840Sgallatin htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr)); 2478169840Sgallatin rx->shadow[idx + i].addr_high = 2479169840Sgallatin htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr)); 2480169840Sgallatin } 2481175579Sgallatin#endif 2482169840Sgallatin 2483155852Sgallatindone: 2484169840Sgallatin for (i = 0; i < rx->nbufs; i++) { 2485169840Sgallatin if ((idx & 7) == 7) { 2486169871Sgallatin mxge_submit_8rx(&rx->lanai[idx - 7], 2487169871Sgallatin &rx->shadow[idx - 7]); 2488159612Sgallatin } 2489169840Sgallatin idx++; 2490169840Sgallatin } 2491155852Sgallatin return err; 2492155852Sgallatin} 2493155852Sgallatin 2494247133Sgallatin#ifdef INET6 2495247133Sgallatin 2496247133Sgallatinstatic uint16_t 2497247133Sgallatinmxge_csum_generic(uint16_t *raw, int len) 2498247133Sgallatin{ 2499247133Sgallatin uint32_t csum; 2500247133Sgallatin 2501247133Sgallatin 2502247133Sgallatin csum = 0; 2503247133Sgallatin while (len > 0) { 2504247133Sgallatin csum += *raw; 2505247133Sgallatin raw++; 2506247133Sgallatin len -= 2; 2507247133Sgallatin } 2508247133Sgallatin csum = (csum >> 16) + (csum & 0xffff); 2509247133Sgallatin csum = (csum >> 16) + (csum & 0xffff); 2510247133Sgallatin return (uint16_t)csum; 2511247133Sgallatin} 2512247133Sgallatin 2513247133Sgallatinstatic inline uint16_t 2514247133Sgallatinmxge_rx_csum6(void *p, struct mbuf *m, uint32_t csum) 2515247133Sgallatin{ 2516247133Sgallatin uint32_t partial; 2517247133Sgallatin int nxt, cksum_offset; 2518247133Sgallatin struct ip6_hdr *ip6 = p; 2519247133Sgallatin uint16_t c; 2520247133Sgallatin 2521247133Sgallatin nxt = ip6->ip6_nxt; 2522247133Sgallatin cksum_offset = sizeof (*ip6) + ETHER_HDR_LEN; 2523247133Sgallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) { 2524247133Sgallatin cksum_offset = ip6_lasthdr(m, ETHER_HDR_LEN, 2525247133Sgallatin IPPROTO_IPV6, &nxt); 2526247133Sgallatin if (nxt != IPPROTO_TCP && nxt != IPPROTO_UDP) 2527247133Sgallatin return (1); 2528247133Sgallatin } 2529247133Sgallatin 2530247133Sgallatin /* 2531247133Sgallatin * IPv6 headers do not contain a checksum, and hence 2532247133Sgallatin * do not checksum to zero, so they don't "fall out" 2533247133Sgallatin * of the partial checksum calculation like IPv4 2534247133Sgallatin * headers do. We need to fix the partial checksum by 2535247133Sgallatin * subtracting the checksum of the IPv6 header. 2536247133Sgallatin */ 2537247133Sgallatin 2538247133Sgallatin partial = mxge_csum_generic((uint16_t *)ip6, cksum_offset - 2539247133Sgallatin ETHER_HDR_LEN); 2540247133Sgallatin csum += ~partial; 2541247133Sgallatin csum += (csum < ~partial); 2542247133Sgallatin csum = (csum >> 16) + (csum & 0xFFFF); 2543247133Sgallatin csum = (csum >> 16) + (csum & 0xFFFF); 2544247133Sgallatin c = in6_cksum_pseudo(ip6, m->m_pkthdr.len - cksum_offset, nxt, 2545247133Sgallatin csum); 2546247133Sgallatin c ^= 0xffff; 2547247133Sgallatin return (c); 2548247133Sgallatin} 2549247133Sgallatin#endif /* INET6 */ 2550169840Sgallatin/* 2551169840Sgallatin * Myri10GE hardware checksums are not valid if the sender 2552169840Sgallatin * padded the frame with non-zero padding. This is because 2553169840Sgallatin * the firmware just does a simple 16-bit 1s complement 2554169840Sgallatin * checksum across the entire frame, excluding the first 14 2555169840Sgallatin * bytes. It is best to simply to check the checksum and 2556169840Sgallatin * tell the stack about it only if the checksum is good 2557169840Sgallatin */ 2558169840Sgallatin 2559169840Sgallatinstatic inline uint16_t 2560159612Sgallatinmxge_rx_csum(struct mbuf *m, int csum) 2561159612Sgallatin{ 2562159612Sgallatin struct ether_header *eh; 2563247133Sgallatin#ifdef INET 2564159612Sgallatin struct ip *ip; 2565247133Sgallatin#endif 2566247152Sgallatin#if defined(INET) || defined(INET6) 2567247152Sgallatin int cap = m->m_pkthdr.rcvif->if_capenable; 2568247152Sgallatin#endif 2569247133Sgallatin uint16_t c, etype; 2570159612Sgallatin 2571247133Sgallatin 2572159612Sgallatin eh = mtod(m, struct ether_header *); 2573247133Sgallatin etype = ntohs(eh->ether_type); 2574247133Sgallatin switch (etype) { 2575194743Sgallatin#ifdef INET 2576247133Sgallatin case ETHERTYPE_IP: 2577247133Sgallatin if ((cap & IFCAP_RXCSUM) == 0) 2578247133Sgallatin return (1); 2579247133Sgallatin ip = (struct ip *)(eh + 1); 2580247133Sgallatin if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP) 2581247133Sgallatin return (1); 2582247133Sgallatin c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2583247133Sgallatin htonl(ntohs(csum) + ntohs(ip->ip_len) - 2584247133Sgallatin (ip->ip_hl << 2) + ip->ip_p)); 2585247133Sgallatin c ^= 0xffff; 2586247133Sgallatin break; 2587194743Sgallatin#endif 2588247133Sgallatin#ifdef INET6 2589247133Sgallatin case ETHERTYPE_IPV6: 2590247133Sgallatin if ((cap & IFCAP_RXCSUM_IPV6) == 0) 2591247133Sgallatin return (1); 2592247133Sgallatin c = mxge_rx_csum6((eh + 1), m, csum); 2593247133Sgallatin break; 2594247133Sgallatin#endif 2595247133Sgallatin default: 2596247133Sgallatin c = 1; 2597247133Sgallatin } 2598169840Sgallatin return (c); 2599159612Sgallatin} 2600159612Sgallatin 2601169905Sgallatinstatic void 2602169905Sgallatinmxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum) 2603169905Sgallatin{ 2604169905Sgallatin struct ether_vlan_header *evl; 2605169905Sgallatin struct ether_header *eh; 2606169905Sgallatin uint32_t partial; 2607169840Sgallatin 2608169905Sgallatin evl = mtod(m, struct ether_vlan_header *); 2609169905Sgallatin eh = mtod(m, struct ether_header *); 2610169905Sgallatin 2611169905Sgallatin /* 2612169905Sgallatin * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes 2613169905Sgallatin * after what the firmware thought was the end of the ethernet 2614169905Sgallatin * header. 2615169905Sgallatin */ 2616169905Sgallatin 2617169905Sgallatin /* put checksum into host byte order */ 2618169905Sgallatin *csum = ntohs(*csum); 2619169905Sgallatin partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN)); 2620169905Sgallatin (*csum) += ~partial; 2621169905Sgallatin (*csum) += ((*csum) < ~partial); 2622169905Sgallatin (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2623169905Sgallatin (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); 2624169905Sgallatin 2625169905Sgallatin /* restore checksum to network byte order; 2626169905Sgallatin later consumers expect this */ 2627169905Sgallatin *csum = htons(*csum); 2628169905Sgallatin 2629169905Sgallatin /* save the tag */ 2630176261Sgallatin#ifdef MXGE_NEW_VLAN_API 2631169905Sgallatin m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag); 2632176261Sgallatin#else 2633176261Sgallatin { 2634176261Sgallatin struct m_tag *mtag; 2635176261Sgallatin mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int), 2636176261Sgallatin M_NOWAIT); 2637176261Sgallatin if (mtag == NULL) 2638176261Sgallatin return; 2639176261Sgallatin VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag); 2640176261Sgallatin m_tag_prepend(m, mtag); 2641176261Sgallatin } 2642169905Sgallatin 2643176261Sgallatin#endif 2644176261Sgallatin m->m_flags |= M_VLANTAG; 2645176261Sgallatin 2646169905Sgallatin /* 2647169905Sgallatin * Remove the 802.1q header by copying the Ethernet 2648169905Sgallatin * addresses over it and adjusting the beginning of 2649169905Sgallatin * the data in the mbuf. The encapsulated Ethernet 2650169905Sgallatin * type field is already in place. 2651169905Sgallatin */ 2652169905Sgallatin bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN, 2653169905Sgallatin ETHER_HDR_LEN - ETHER_TYPE_LEN); 2654169905Sgallatin m_adj(m, ETHER_VLAN_ENCAP_LEN); 2655169905Sgallatin} 2656169905Sgallatin 2657169905Sgallatin 2658169840Sgallatinstatic inline void 2659247133Sgallatinmxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, 2660247133Sgallatin uint32_t csum, int lro) 2661155852Sgallatin{ 2662175365Sgallatin mxge_softc_t *sc; 2663155852Sgallatin struct ifnet *ifp; 2664169840Sgallatin struct mbuf *m; 2665169905Sgallatin struct ether_header *eh; 2666175365Sgallatin mxge_rx_ring_t *rx; 2667155852Sgallatin bus_dmamap_t old_map; 2668155852Sgallatin int idx; 2669155852Sgallatin 2670175365Sgallatin sc = ss->sc; 2671169840Sgallatin ifp = sc->ifp; 2672175365Sgallatin rx = &ss->rx_big; 2673169840Sgallatin idx = rx->cnt & rx->mask; 2674169840Sgallatin rx->cnt += rx->nbufs; 2675169840Sgallatin /* save a pointer to the received mbuf */ 2676169840Sgallatin m = rx->info[idx].m; 2677169840Sgallatin /* try to replace the received mbuf */ 2678175365Sgallatin if (mxge_get_buf_big(ss, rx->extra_map, idx)) { 2679169840Sgallatin /* drop the frame -- the old mbuf is re-cycled */ 2680169840Sgallatin ifp->if_ierrors++; 2681169840Sgallatin return; 2682169840Sgallatin } 2683155852Sgallatin 2684169840Sgallatin /* unmap the received buffer */ 2685169840Sgallatin old_map = rx->info[idx].map; 2686169840Sgallatin bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2687169840Sgallatin bus_dmamap_unload(rx->dmat, old_map); 2688155852Sgallatin 2689169840Sgallatin /* swap the bus_dmamap_t's */ 2690169840Sgallatin rx->info[idx].map = rx->extra_map; 2691169840Sgallatin rx->extra_map = old_map; 2692155852Sgallatin 2693169840Sgallatin /* mcp implicitly skips 1st 2 bytes so that packet is properly 2694169840Sgallatin * aligned */ 2695169840Sgallatin m->m_data += MXGEFW_PAD; 2696155852Sgallatin 2697169840Sgallatin m->m_pkthdr.rcvif = ifp; 2698169840Sgallatin m->m_len = m->m_pkthdr.len = len; 2699175365Sgallatin ss->ipackets++; 2700169905Sgallatin eh = mtod(m, struct ether_header *); 2701169905Sgallatin if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2702169905Sgallatin mxge_vlan_tag_remove(m, &csum); 2703169905Sgallatin } 2704302052Ssephe /* flowid only valid if RSS hashing is enabled */ 2705302052Ssephe if (sc->num_slices > 1) { 2706302052Ssephe m->m_pkthdr.flowid = (ss - sc->ss); 2707302052Ssephe M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2708302052Ssephe } 2709155852Sgallatin /* if the checksum is valid, mark it in the mbuf header */ 2710247133Sgallatin if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 2711247133Sgallatin (0 == mxge_rx_csum(m, csum))) { 2712247133Sgallatin /* Tell the stack that the checksum is good */ 2713247133Sgallatin m->m_pkthdr.csum_data = 0xffff; 2714247133Sgallatin m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 2715247133Sgallatin CSUM_DATA_VALID; 2716247133Sgallatin 2717247133Sgallatin#if defined(INET) || defined (INET6) 2718247133Sgallatin if (lro && (0 == tcp_lro_rx(&ss->lc, m, 0))) 2719169840Sgallatin return; 2720247133Sgallatin#endif 2721169840Sgallatin } 2722155852Sgallatin /* pass the frame up the stack */ 2723169840Sgallatin (*ifp->if_input)(ifp, m); 2724155852Sgallatin} 2725155852Sgallatin 2726155852Sgallatinstatic inline void 2727247133Sgallatinmxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, 2728247133Sgallatin uint32_t csum, int lro) 2729155852Sgallatin{ 2730175365Sgallatin mxge_softc_t *sc; 2731155852Sgallatin struct ifnet *ifp; 2732169905Sgallatin struct ether_header *eh; 2733155852Sgallatin struct mbuf *m; 2734175365Sgallatin mxge_rx_ring_t *rx; 2735155852Sgallatin bus_dmamap_t old_map; 2736155852Sgallatin int idx; 2737155852Sgallatin 2738175365Sgallatin sc = ss->sc; 2739155852Sgallatin ifp = sc->ifp; 2740175365Sgallatin rx = &ss->rx_small; 2741155852Sgallatin idx = rx->cnt & rx->mask; 2742155852Sgallatin rx->cnt++; 2743155852Sgallatin /* save a pointer to the received mbuf */ 2744155852Sgallatin m = rx->info[idx].m; 2745155852Sgallatin /* try to replace the received mbuf */ 2746175365Sgallatin if (mxge_get_buf_small(ss, rx->extra_map, idx)) { 2747155852Sgallatin /* drop the frame -- the old mbuf is re-cycled */ 2748155852Sgallatin ifp->if_ierrors++; 2749155852Sgallatin return; 2750155852Sgallatin } 2751155852Sgallatin 2752155852Sgallatin /* unmap the received buffer */ 2753155852Sgallatin old_map = rx->info[idx].map; 2754155852Sgallatin bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); 2755155852Sgallatin bus_dmamap_unload(rx->dmat, old_map); 2756155852Sgallatin 2757155852Sgallatin /* swap the bus_dmamap_t's */ 2758155852Sgallatin rx->info[idx].map = rx->extra_map; 2759155852Sgallatin rx->extra_map = old_map; 2760155852Sgallatin 2761155852Sgallatin /* mcp implicitly skips 1st 2 bytes so that packet is properly 2762155852Sgallatin * aligned */ 2763159612Sgallatin m->m_data += MXGEFW_PAD; 2764155852Sgallatin 2765169070Sgallatin m->m_pkthdr.rcvif = ifp; 2766169070Sgallatin m->m_len = m->m_pkthdr.len = len; 2767175365Sgallatin ss->ipackets++; 2768169905Sgallatin eh = mtod(m, struct ether_header *); 2769169905Sgallatin if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2770169905Sgallatin mxge_vlan_tag_remove(m, &csum); 2771169905Sgallatin } 2772302052Ssephe /* flowid only valid if RSS hashing is enabled */ 2773302052Ssephe if (sc->num_slices > 1) { 2774302052Ssephe m->m_pkthdr.flowid = (ss - sc->ss); 2775302052Ssephe M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2776302052Ssephe } 2777155852Sgallatin /* if the checksum is valid, mark it in the mbuf header */ 2778247133Sgallatin if ((ifp->if_capenable & (IFCAP_RXCSUM_IPV6 | IFCAP_RXCSUM)) && 2779247133Sgallatin (0 == mxge_rx_csum(m, csum))) { 2780247133Sgallatin /* Tell the stack that the checksum is good */ 2781247133Sgallatin m->m_pkthdr.csum_data = 0xffff; 2782247133Sgallatin m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | 2783247133Sgallatin CSUM_DATA_VALID; 2784247133Sgallatin 2785247133Sgallatin#if defined(INET) || defined (INET6) 2786247133Sgallatin if (lro && (0 == tcp_lro_rx(&ss->lc, m, csum))) 2787169840Sgallatin return; 2788247133Sgallatin#endif 2789169840Sgallatin } 2790155852Sgallatin /* pass the frame up the stack */ 2791155852Sgallatin (*ifp->if_input)(ifp, m); 2792155852Sgallatin} 2793155852Sgallatin 2794155852Sgallatinstatic inline void 2795175365Sgallatinmxge_clean_rx_done(struct mxge_slice_state *ss) 2796159612Sgallatin{ 2797175365Sgallatin mxge_rx_done_t *rx_done = &ss->rx_done; 2798159612Sgallatin int limit = 0; 2799159612Sgallatin uint16_t length; 2800159612Sgallatin uint16_t checksum; 2801247133Sgallatin int lro; 2802159612Sgallatin 2803247133Sgallatin lro = ss->sc->ifp->if_capenable & IFCAP_LRO; 2804159612Sgallatin while (rx_done->entry[rx_done->idx].length != 0) { 2805159612Sgallatin length = ntohs(rx_done->entry[rx_done->idx].length); 2806159612Sgallatin rx_done->entry[rx_done->idx].length = 0; 2807169840Sgallatin checksum = rx_done->entry[rx_done->idx].checksum; 2808163467Sgallatin if (length <= (MHLEN - MXGEFW_PAD)) 2809247133Sgallatin mxge_rx_done_small(ss, length, checksum, lro); 2810159612Sgallatin else 2811247133Sgallatin mxge_rx_done_big(ss, length, checksum, lro); 2812159612Sgallatin rx_done->cnt++; 2813169871Sgallatin rx_done->idx = rx_done->cnt & rx_done->mask; 2814159612Sgallatin 2815159612Sgallatin /* limit potential for livelock */ 2816170733Sgallatin if (__predict_false(++limit > rx_done->mask / 2)) 2817159612Sgallatin break; 2818159612Sgallatin } 2819247133Sgallatin#if defined(INET) || defined (INET6) 2820247133Sgallatin while (!SLIST_EMPTY(&ss->lc.lro_active)) { 2821247133Sgallatin struct lro_entry *lro = SLIST_FIRST(&ss->lc.lro_active); 2822247133Sgallatin SLIST_REMOVE_HEAD(&ss->lc.lro_active, next); 2823247133Sgallatin tcp_lro_flush(&ss->lc, lro); 2824169840Sgallatin } 2825194743Sgallatin#endif 2826159612Sgallatin} 2827159612Sgallatin 2828159612Sgallatin 2829159612Sgallatinstatic inline void 2830175365Sgallatinmxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx) 2831155852Sgallatin{ 2832155852Sgallatin struct ifnet *ifp; 2833175365Sgallatin mxge_tx_ring_t *tx; 2834155852Sgallatin struct mbuf *m; 2835155852Sgallatin bus_dmamap_t map; 2836170733Sgallatin int idx; 2837191562Sgallatin int *flags; 2838155852Sgallatin 2839175365Sgallatin tx = &ss->tx; 2840175365Sgallatin ifp = ss->sc->ifp; 2841159612Sgallatin while (tx->pkt_done != mcp_idx) { 2842155852Sgallatin idx = tx->done & tx->mask; 2843155852Sgallatin tx->done++; 2844155852Sgallatin m = tx->info[idx].m; 2845155852Sgallatin /* mbuf and DMA map only attached to the first 2846155852Sgallatin segment per-mbuf */ 2847155852Sgallatin if (m != NULL) { 2848194751Sgallatin ss->obytes += m->m_pkthdr.len; 2849194751Sgallatin if (m->m_flags & M_MCAST) 2850194751Sgallatin ss->omcasts++; 2851191562Sgallatin ss->opackets++; 2852155852Sgallatin tx->info[idx].m = NULL; 2853155852Sgallatin map = tx->info[idx].map; 2854155852Sgallatin bus_dmamap_unload(tx->dmat, map); 2855155852Sgallatin m_freem(m); 2856155852Sgallatin } 2857159612Sgallatin if (tx->info[idx].flag) { 2858159612Sgallatin tx->info[idx].flag = 0; 2859159612Sgallatin tx->pkt_done++; 2860159612Sgallatin } 2861155852Sgallatin } 2862155852Sgallatin 2863155852Sgallatin /* If we have space, clear IFF_OACTIVE to tell the stack that 2864155852Sgallatin its OK to send packets */ 2865191562Sgallatin#ifdef IFNET_BUF_RING 2866191562Sgallatin flags = &ss->if_drv_flags; 2867191562Sgallatin#else 2868191562Sgallatin flags = &ifp->if_drv_flags; 2869191562Sgallatin#endif 2870191562Sgallatin mtx_lock(&ss->tx.mtx); 2871191562Sgallatin if ((*flags) & IFF_DRV_OACTIVE && 2872155852Sgallatin tx->req - tx->done < (tx->mask + 1)/4) { 2873191562Sgallatin *(flags) &= ~IFF_DRV_OACTIVE; 2874175365Sgallatin ss->tx.wake++; 2875175365Sgallatin mxge_start_locked(ss); 2876155852Sgallatin } 2877191562Sgallatin#ifdef IFNET_BUF_RING 2878191562Sgallatin if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) { 2879191562Sgallatin /* let the NIC stop polling this queue, since there 2880191562Sgallatin * are no more transmits pending */ 2881191562Sgallatin if (tx->req == tx->done) { 2882191562Sgallatin *tx->send_stop = 1; 2883191562Sgallatin tx->queue_active = 0; 2884191562Sgallatin tx->deactivate++; 2885191562Sgallatin wmb(); 2886191562Sgallatin } 2887191562Sgallatin } 2888191562Sgallatin#endif 2889191562Sgallatin mtx_unlock(&ss->tx.mtx); 2890191562Sgallatin 2891155852Sgallatin} 2892155852Sgallatin 2893188736Sgallatinstatic struct mxge_media_type mxge_xfp_media_types[] = 2894171917Sgallatin{ 2895171917Sgallatin {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"}, 2896171917Sgallatin {IFM_10G_SR, (1 << 7), "10GBASE-SR"}, 2897171917Sgallatin {IFM_10G_LR, (1 << 6), "10GBASE-LR"}, 2898171917Sgallatin {0, (1 << 5), "10GBASE-ER"}, 2899188736Sgallatin {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"}, 2900171917Sgallatin {0, (1 << 3), "10GBASE-SW"}, 2901171917Sgallatin {0, (1 << 2), "10GBASE-LW"}, 2902171917Sgallatin {0, (1 << 1), "10GBASE-EW"}, 2903171917Sgallatin {0, (1 << 0), "Reserved"} 2904171917Sgallatin}; 2905188736Sgallatinstatic struct mxge_media_type mxge_sfp_media_types[] = 2906188736Sgallatin{ 2907202119Sgallatin {IFM_10G_TWINAX, 0, "10GBASE-Twinax"}, 2908188737Sgallatin {0, (1 << 7), "Reserved"}, 2909188736Sgallatin {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"}, 2910188736Sgallatin {IFM_10G_LR, (1 << 5), "10GBASE-LR"}, 2911208312Sgallatin {IFM_10G_SR, (1 << 4), "10GBASE-SR"}, 2912208312Sgallatin {IFM_10G_TWINAX,(1 << 0), "10GBASE-Twinax"} 2913188736Sgallatin}; 2914171917Sgallatin 2915155852Sgallatinstatic void 2916206662Sgallatinmxge_media_set(mxge_softc_t *sc, int media_type) 2917171917Sgallatin{ 2918206662Sgallatin 2919206662Sgallatin 2920206662Sgallatin ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type, 2921206662Sgallatin 0, NULL); 2922206662Sgallatin ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type); 2923206662Sgallatin sc->current_media = media_type; 2924206662Sgallatin sc->media.ifm_media = sc->media.ifm_cur->ifm_media; 2925171917Sgallatin} 2926171917Sgallatin 2927171917Sgallatinstatic void 2928206662Sgallatinmxge_media_init(mxge_softc_t *sc) 2929171917Sgallatin{ 2930171917Sgallatin char *ptr; 2931206662Sgallatin int i; 2932171917Sgallatin 2933206662Sgallatin ifmedia_removeall(&sc->media); 2934206662Sgallatin mxge_media_set(sc, IFM_AUTO); 2935171917Sgallatin 2936171917Sgallatin /* 2937171917Sgallatin * parse the product code to deterimine the interface type 2938171917Sgallatin * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 2939171917Sgallatin * after the 3rd dash in the driver's cached copy of the 2940171917Sgallatin * EEPROM's product code string. 2941171917Sgallatin */ 2942171917Sgallatin ptr = sc->product_code_string; 2943171917Sgallatin if (ptr == NULL) { 2944171917Sgallatin device_printf(sc->dev, "Missing product code\n"); 2945206662Sgallatin return; 2946171917Sgallatin } 2947171917Sgallatin 2948171917Sgallatin for (i = 0; i < 3; i++, ptr++) { 2949229272Sed ptr = strchr(ptr, '-'); 2950171917Sgallatin if (ptr == NULL) { 2951171917Sgallatin device_printf(sc->dev, 2952171917Sgallatin "only %d dashes in PC?!?\n", i); 2953171917Sgallatin return; 2954171917Sgallatin } 2955171917Sgallatin } 2956223957Sgallatin if (*ptr == 'C' || *(ptr +1) == 'C') { 2957188736Sgallatin /* -C is CX4 */ 2958206662Sgallatin sc->connector = MXGE_CX4; 2959206662Sgallatin mxge_media_set(sc, IFM_10G_CX4); 2960206662Sgallatin } else if (*ptr == 'Q') { 2961188736Sgallatin /* -Q is Quad Ribbon Fiber */ 2962206662Sgallatin sc->connector = MXGE_QRF; 2963171917Sgallatin device_printf(sc->dev, "Quad Ribbon Fiber Media\n"); 2964171917Sgallatin /* FreeBSD has no media type for Quad ribbon fiber */ 2965206662Sgallatin } else if (*ptr == 'R') { 2966206662Sgallatin /* -R is XFP */ 2967206662Sgallatin sc->connector = MXGE_XFP; 2968206662Sgallatin } else if (*ptr == 'S' || *(ptr +1) == 'S') { 2969206662Sgallatin /* -S or -2S is SFP+ */ 2970206662Sgallatin sc->connector = MXGE_SFP; 2971206662Sgallatin } else { 2972206662Sgallatin device_printf(sc->dev, "Unknown media type: %c\n", *ptr); 2973171917Sgallatin } 2974206662Sgallatin} 2975171917Sgallatin 2976206662Sgallatin/* 2977206662Sgallatin * Determine the media type for a NIC. Some XFPs will identify 2978206662Sgallatin * themselves only when their link is up, so this is initiated via a 2979206662Sgallatin * link up interrupt. However, this can potentially take up to 2980206662Sgallatin * several milliseconds, so it is run via the watchdog routine, rather 2981206662Sgallatin * than in the interrupt handler itself. 2982206662Sgallatin */ 2983206662Sgallatinstatic void 2984206662Sgallatinmxge_media_probe(mxge_softc_t *sc) 2985206662Sgallatin{ 2986206662Sgallatin mxge_cmd_t cmd; 2987206662Sgallatin char *cage_type; 2988206662Sgallatin 2989206662Sgallatin struct mxge_media_type *mxge_media_types = NULL; 2990206662Sgallatin int i, err, ms, mxge_media_type_entries; 2991206662Sgallatin uint32_t byte; 2992206662Sgallatin 2993206662Sgallatin sc->need_media_probe = 0; 2994206662Sgallatin 2995206662Sgallatin if (sc->connector == MXGE_XFP) { 2996188736Sgallatin /* -R is XFP */ 2997188736Sgallatin mxge_media_types = mxge_xfp_media_types; 2998188736Sgallatin mxge_media_type_entries = 2999188736Sgallatin sizeof (mxge_xfp_media_types) / 3000188736Sgallatin sizeof (mxge_xfp_media_types[0]); 3001188736Sgallatin byte = MXGE_XFP_COMPLIANCE_BYTE; 3002188736Sgallatin cage_type = "XFP"; 3003206662Sgallatin } else if (sc->connector == MXGE_SFP) { 3004188736Sgallatin /* -S or -2S is SFP+ */ 3005188736Sgallatin mxge_media_types = mxge_sfp_media_types; 3006188736Sgallatin mxge_media_type_entries = 3007188736Sgallatin sizeof (mxge_sfp_media_types) / 3008188736Sgallatin sizeof (mxge_sfp_media_types[0]); 3009188736Sgallatin cage_type = "SFP+"; 3010188736Sgallatin byte = 3; 3011206662Sgallatin } else { 3012206662Sgallatin /* nothing to do; media type cannot change */ 3013171917Sgallatin return; 3014171917Sgallatin } 3015171917Sgallatin 3016171917Sgallatin /* 3017171917Sgallatin * At this point we know the NIC has an XFP cage, so now we 3018171917Sgallatin * try to determine what is in the cage by using the 3019171917Sgallatin * firmware's XFP I2C commands to read the XFP 10GbE compilance 3020171917Sgallatin * register. We read just one byte, which may take over 3021171917Sgallatin * a millisecond 3022171917Sgallatin */ 3023171917Sgallatin 3024171917Sgallatin cmd.data0 = 0; /* just fetch 1 byte, not all 256 */ 3025188736Sgallatin cmd.data1 = byte; 3026188736Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd); 3027188736Sgallatin if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) { 3028171917Sgallatin device_printf(sc->dev, "failed to read XFP\n"); 3029171917Sgallatin } 3030188736Sgallatin if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) { 3031188736Sgallatin device_printf(sc->dev, "Type R/S with no XFP!?!?\n"); 3032171917Sgallatin } 3033171917Sgallatin if (err != MXGEFW_CMD_OK) { 3034171917Sgallatin return; 3035171917Sgallatin } 3036171917Sgallatin 3037171917Sgallatin /* now we wait for the data to be cached */ 3038188736Sgallatin cmd.data0 = byte; 3039188736Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3040171917Sgallatin for (ms = 0; (err == EBUSY) && (ms < 50); ms++) { 3041171917Sgallatin DELAY(1000); 3042188736Sgallatin cmd.data0 = byte; 3043188736Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); 3044171917Sgallatin } 3045171917Sgallatin if (err != MXGEFW_CMD_OK) { 3046188736Sgallatin device_printf(sc->dev, "failed to read %s (%d, %dms)\n", 3047188736Sgallatin cage_type, err, ms); 3048171917Sgallatin return; 3049171917Sgallatin } 3050171917Sgallatin 3051171917Sgallatin if (cmd.data0 == mxge_media_types[0].bitmask) { 3052171917Sgallatin if (mxge_verbose) 3053188736Sgallatin device_printf(sc->dev, "%s:%s\n", cage_type, 3054171917Sgallatin mxge_media_types[0].name); 3055206662Sgallatin if (sc->current_media != mxge_media_types[0].flag) { 3056206662Sgallatin mxge_media_init(sc); 3057206662Sgallatin mxge_media_set(sc, mxge_media_types[0].flag); 3058206662Sgallatin } 3059171917Sgallatin return; 3060171917Sgallatin } 3061188736Sgallatin for (i = 1; i < mxge_media_type_entries; i++) { 3062171917Sgallatin if (cmd.data0 & mxge_media_types[i].bitmask) { 3063171917Sgallatin if (mxge_verbose) 3064188736Sgallatin device_printf(sc->dev, "%s:%s\n", 3065188736Sgallatin cage_type, 3066171917Sgallatin mxge_media_types[i].name); 3067171917Sgallatin 3068206662Sgallatin if (sc->current_media != mxge_media_types[i].flag) { 3069206662Sgallatin mxge_media_init(sc); 3070206662Sgallatin mxge_media_set(sc, mxge_media_types[i].flag); 3071206662Sgallatin } 3072171917Sgallatin return; 3073171917Sgallatin } 3074171917Sgallatin } 3075206662Sgallatin if (mxge_verbose) 3076206662Sgallatin device_printf(sc->dev, "%s media 0x%x unknown\n", 3077206662Sgallatin cage_type, cmd.data0); 3078171917Sgallatin 3079171917Sgallatin return; 3080171917Sgallatin} 3081171917Sgallatin 3082171917Sgallatinstatic void 3083159571Sgallatinmxge_intr(void *arg) 3084155852Sgallatin{ 3085175365Sgallatin struct mxge_slice_state *ss = arg; 3086175365Sgallatin mxge_softc_t *sc = ss->sc; 3087175365Sgallatin mcp_irq_data_t *stats = ss->fw_stats; 3088175365Sgallatin mxge_tx_ring_t *tx = &ss->tx; 3089175365Sgallatin mxge_rx_done_t *rx_done = &ss->rx_done; 3090159612Sgallatin uint32_t send_done_count; 3091159612Sgallatin uint8_t valid; 3092155852Sgallatin 3093155852Sgallatin 3094191562Sgallatin#ifndef IFNET_BUF_RING 3095175365Sgallatin /* an interrupt on a non-zero slice is implicitly valid 3096175365Sgallatin since MSI-X irqs are not shared */ 3097175365Sgallatin if (ss != sc->ss) { 3098175365Sgallatin mxge_clean_rx_done(ss); 3099175365Sgallatin *ss->irq_claim = be32toh(3); 3100175365Sgallatin return; 3101175365Sgallatin } 3102191562Sgallatin#endif 3103175365Sgallatin 3104159612Sgallatin /* make sure the DMA has finished */ 3105159612Sgallatin if (!stats->valid) { 3106159612Sgallatin return; 3107155852Sgallatin } 3108159612Sgallatin valid = stats->valid; 3109155852Sgallatin 3110176281Sgallatin if (sc->legacy_irq) { 3111164472Sgallatin /* lower legacy IRQ */ 3112164472Sgallatin *sc->irq_deassert = 0; 3113164472Sgallatin if (!mxge_deassert_wait) 3114164472Sgallatin /* don't wait for conf. that irq is low */ 3115164472Sgallatin stats->valid = 0; 3116164472Sgallatin } else { 3117159612Sgallatin stats->valid = 0; 3118164472Sgallatin } 3119164472Sgallatin 3120164472Sgallatin /* loop while waiting for legacy irq deassertion */ 3121159612Sgallatin do { 3122159612Sgallatin /* check for transmit completes and receives */ 3123159612Sgallatin send_done_count = be32toh(stats->send_done_count); 3124159612Sgallatin while ((send_done_count != tx->pkt_done) || 3125159612Sgallatin (rx_done->entry[rx_done->idx].length != 0)) { 3126191562Sgallatin if (send_done_count != tx->pkt_done) 3127191562Sgallatin mxge_tx_done(ss, (int)send_done_count); 3128175365Sgallatin mxge_clean_rx_done(ss); 3129159612Sgallatin send_done_count = be32toh(stats->send_done_count); 3130155852Sgallatin } 3131176281Sgallatin if (sc->legacy_irq && mxge_deassert_wait) 3132185255Sgallatin wmb(); 3133159612Sgallatin } while (*((volatile uint8_t *) &stats->valid)); 3134155852Sgallatin 3135191562Sgallatin /* fw link & error stats meaningful only on the first slice */ 3136191562Sgallatin if (__predict_false((ss == sc->ss) && stats->stats_updated)) { 3137159612Sgallatin if (sc->link_state != stats->link_up) { 3138159612Sgallatin sc->link_state = stats->link_up; 3139159612Sgallatin if (sc->link_state) { 3140159612Sgallatin if_link_state_change(sc->ifp, LINK_STATE_UP); 3141241687Sglebius if_initbaudrate(sc->ifp, IF_Gbps(10)); 3142159612Sgallatin if (mxge_verbose) 3143159612Sgallatin device_printf(sc->dev, "link up\n"); 3144159612Sgallatin } else { 3145159612Sgallatin if_link_state_change(sc->ifp, LINK_STATE_DOWN); 3146206662Sgallatin sc->ifp->if_baudrate = 0; 3147159612Sgallatin if (mxge_verbose) 3148159612Sgallatin device_printf(sc->dev, "link down\n"); 3149155852Sgallatin } 3150171917Sgallatin sc->need_media_probe = 1; 3151155852Sgallatin } 3152159612Sgallatin if (sc->rdma_tags_available != 3153175365Sgallatin be32toh(stats->rdma_tags_available)) { 3154159612Sgallatin sc->rdma_tags_available = 3155175365Sgallatin be32toh(stats->rdma_tags_available); 3156159612Sgallatin device_printf(sc->dev, "RDMA timed out! %d tags " 3157159612Sgallatin "left\n", sc->rdma_tags_available); 3158155852Sgallatin } 3159171917Sgallatin 3160171917Sgallatin if (stats->link_down) { 3161171917Sgallatin sc->down_cnt += stats->link_down; 3162171917Sgallatin sc->link_state = 0; 3163171917Sgallatin if_link_state_change(sc->ifp, LINK_STATE_DOWN); 3164171917Sgallatin } 3165155852Sgallatin } 3166155852Sgallatin 3167159612Sgallatin /* check to see if we have rx token to pass back */ 3168159612Sgallatin if (valid & 0x1) 3169175365Sgallatin *ss->irq_claim = be32toh(3); 3170175365Sgallatin *(ss->irq_claim + 1) = be32toh(3); 3171155852Sgallatin} 3172155852Sgallatin 3173155852Sgallatinstatic void 3174159571Sgallatinmxge_init(void *arg) 3175155852Sgallatin{ 3176220385Sgallatin mxge_softc_t *sc = arg; 3177220385Sgallatin struct ifnet *ifp = sc->ifp; 3178220385Sgallatin 3179220385Sgallatin 3180220385Sgallatin mtx_lock(&sc->driver_mtx); 3181220385Sgallatin if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3182220385Sgallatin (void) mxge_open(sc); 3183220385Sgallatin mtx_unlock(&sc->driver_mtx); 3184155852Sgallatin} 3185155852Sgallatin 3186155852Sgallatin 3187155852Sgallatin 3188155852Sgallatinstatic void 3189175365Sgallatinmxge_free_slice_mbufs(struct mxge_slice_state *ss) 3190155852Sgallatin{ 3191155852Sgallatin int i; 3192155852Sgallatin 3193247133Sgallatin#if defined(INET) || defined(INET6) 3194247133Sgallatin tcp_lro_free(&ss->lc); 3195247133Sgallatin#endif 3196175365Sgallatin for (i = 0; i <= ss->rx_big.mask; i++) { 3197175365Sgallatin if (ss->rx_big.info[i].m == NULL) 3198155852Sgallatin continue; 3199175365Sgallatin bus_dmamap_unload(ss->rx_big.dmat, 3200175365Sgallatin ss->rx_big.info[i].map); 3201175365Sgallatin m_freem(ss->rx_big.info[i].m); 3202175365Sgallatin ss->rx_big.info[i].m = NULL; 3203155852Sgallatin } 3204155852Sgallatin 3205175365Sgallatin for (i = 0; i <= ss->rx_small.mask; i++) { 3206175365Sgallatin if (ss->rx_small.info[i].m == NULL) 3207155852Sgallatin continue; 3208175365Sgallatin bus_dmamap_unload(ss->rx_small.dmat, 3209175365Sgallatin ss->rx_small.info[i].map); 3210175365Sgallatin m_freem(ss->rx_small.info[i].m); 3211175365Sgallatin ss->rx_small.info[i].m = NULL; 3212155852Sgallatin } 3213155852Sgallatin 3214175365Sgallatin /* transmit ring used only on the first slice */ 3215175365Sgallatin if (ss->tx.info == NULL) 3216175365Sgallatin return; 3217175365Sgallatin 3218175365Sgallatin for (i = 0; i <= ss->tx.mask; i++) { 3219175365Sgallatin ss->tx.info[i].flag = 0; 3220175365Sgallatin if (ss->tx.info[i].m == NULL) 3221155852Sgallatin continue; 3222175365Sgallatin bus_dmamap_unload(ss->tx.dmat, 3223175365Sgallatin ss->tx.info[i].map); 3224175365Sgallatin m_freem(ss->tx.info[i].m); 3225175365Sgallatin ss->tx.info[i].m = NULL; 3226155852Sgallatin } 3227155852Sgallatin} 3228155852Sgallatin 3229155852Sgallatinstatic void 3230175365Sgallatinmxge_free_mbufs(mxge_softc_t *sc) 3231155852Sgallatin{ 3232175365Sgallatin int slice; 3233175365Sgallatin 3234175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) 3235175365Sgallatin mxge_free_slice_mbufs(&sc->ss[slice]); 3236175365Sgallatin} 3237175365Sgallatin 3238175365Sgallatinstatic void 3239175365Sgallatinmxge_free_slice_rings(struct mxge_slice_state *ss) 3240175365Sgallatin{ 3241155852Sgallatin int i; 3242155852Sgallatin 3243175365Sgallatin 3244175365Sgallatin if (ss->rx_done.entry != NULL) 3245175365Sgallatin mxge_dma_free(&ss->rx_done.dma); 3246175365Sgallatin ss->rx_done.entry = NULL; 3247175365Sgallatin 3248175365Sgallatin if (ss->tx.req_bytes != NULL) 3249175365Sgallatin free(ss->tx.req_bytes, M_DEVBUF); 3250175365Sgallatin ss->tx.req_bytes = NULL; 3251175365Sgallatin 3252175365Sgallatin if (ss->tx.seg_list != NULL) 3253175365Sgallatin free(ss->tx.seg_list, M_DEVBUF); 3254175365Sgallatin ss->tx.seg_list = NULL; 3255175365Sgallatin 3256175365Sgallatin if (ss->rx_small.shadow != NULL) 3257175365Sgallatin free(ss->rx_small.shadow, M_DEVBUF); 3258175365Sgallatin ss->rx_small.shadow = NULL; 3259175365Sgallatin 3260175365Sgallatin if (ss->rx_big.shadow != NULL) 3261175365Sgallatin free(ss->rx_big.shadow, M_DEVBUF); 3262175365Sgallatin ss->rx_big.shadow = NULL; 3263175365Sgallatin 3264175365Sgallatin if (ss->tx.info != NULL) { 3265175365Sgallatin if (ss->tx.dmat != NULL) { 3266175365Sgallatin for (i = 0; i <= ss->tx.mask; i++) { 3267175365Sgallatin bus_dmamap_destroy(ss->tx.dmat, 3268175365Sgallatin ss->tx.info[i].map); 3269166371Sgallatin } 3270175365Sgallatin bus_dma_tag_destroy(ss->tx.dmat); 3271155852Sgallatin } 3272175365Sgallatin free(ss->tx.info, M_DEVBUF); 3273155852Sgallatin } 3274175365Sgallatin ss->tx.info = NULL; 3275175365Sgallatin 3276175365Sgallatin if (ss->rx_small.info != NULL) { 3277175365Sgallatin if (ss->rx_small.dmat != NULL) { 3278175365Sgallatin for (i = 0; i <= ss->rx_small.mask; i++) { 3279175365Sgallatin bus_dmamap_destroy(ss->rx_small.dmat, 3280175365Sgallatin ss->rx_small.info[i].map); 3281166371Sgallatin } 3282175365Sgallatin bus_dmamap_destroy(ss->rx_small.dmat, 3283175365Sgallatin ss->rx_small.extra_map); 3284175365Sgallatin bus_dma_tag_destroy(ss->rx_small.dmat); 3285155852Sgallatin } 3286175365Sgallatin free(ss->rx_small.info, M_DEVBUF); 3287155852Sgallatin } 3288175365Sgallatin ss->rx_small.info = NULL; 3289175365Sgallatin 3290175365Sgallatin if (ss->rx_big.info != NULL) { 3291175365Sgallatin if (ss->rx_big.dmat != NULL) { 3292175365Sgallatin for (i = 0; i <= ss->rx_big.mask; i++) { 3293175365Sgallatin bus_dmamap_destroy(ss->rx_big.dmat, 3294175365Sgallatin ss->rx_big.info[i].map); 3295166371Sgallatin } 3296175365Sgallatin bus_dmamap_destroy(ss->rx_big.dmat, 3297175365Sgallatin ss->rx_big.extra_map); 3298175365Sgallatin bus_dma_tag_destroy(ss->rx_big.dmat); 3299155852Sgallatin } 3300175365Sgallatin free(ss->rx_big.info, M_DEVBUF); 3301155852Sgallatin } 3302175365Sgallatin ss->rx_big.info = NULL; 3303155852Sgallatin} 3304155852Sgallatin 3305175365Sgallatinstatic void 3306175365Sgallatinmxge_free_rings(mxge_softc_t *sc) 3307155852Sgallatin{ 3308175365Sgallatin int slice; 3309155852Sgallatin 3310175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) 3311175365Sgallatin mxge_free_slice_rings(&sc->ss[slice]); 3312175365Sgallatin} 3313155852Sgallatin 3314175365Sgallatinstatic int 3315175365Sgallatinmxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries, 3316175365Sgallatin int tx_ring_entries) 3317175365Sgallatin{ 3318175365Sgallatin mxge_softc_t *sc = ss->sc; 3319175365Sgallatin size_t bytes; 3320175365Sgallatin int err, i; 3321155852Sgallatin 3322175365Sgallatin /* allocate per-slice receive resources */ 3323169871Sgallatin 3324175365Sgallatin ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 3325175365Sgallatin ss->rx_done.mask = (2 * rx_ring_entries) - 1; 3326155852Sgallatin 3327155852Sgallatin /* allocate the rx shadow rings */ 3328175365Sgallatin bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow); 3329175365Sgallatin ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3330155852Sgallatin 3331175365Sgallatin bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow); 3332175365Sgallatin ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3333155852Sgallatin 3334175365Sgallatin /* allocate the rx host info rings */ 3335175365Sgallatin bytes = rx_ring_entries * sizeof (*ss->rx_small.info); 3336175365Sgallatin ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3337155852Sgallatin 3338175365Sgallatin bytes = rx_ring_entries * sizeof (*ss->rx_big.info); 3339175365Sgallatin ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3340155852Sgallatin 3341175365Sgallatin /* allocate the rx busdma resources */ 3342155852Sgallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3343155852Sgallatin 1, /* alignment */ 3344155852Sgallatin 4096, /* boundary */ 3345155852Sgallatin BUS_SPACE_MAXADDR, /* low */ 3346155852Sgallatin BUS_SPACE_MAXADDR, /* high */ 3347155852Sgallatin NULL, NULL, /* filter */ 3348155852Sgallatin MHLEN, /* maxsize */ 3349155852Sgallatin 1, /* num segs */ 3350155852Sgallatin MHLEN, /* maxsegsize */ 3351155852Sgallatin BUS_DMA_ALLOCNOW, /* flags */ 3352155852Sgallatin NULL, NULL, /* lock */ 3353175365Sgallatin &ss->rx_small.dmat); /* tag */ 3354155852Sgallatin if (err != 0) { 3355155852Sgallatin device_printf(sc->dev, "Err %d allocating rx_small dmat\n", 3356155852Sgallatin err); 3357201758Smbr return err; 3358155852Sgallatin } 3359155852Sgallatin 3360155852Sgallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3361155852Sgallatin 1, /* alignment */ 3362175579Sgallatin#if MXGE_VIRT_JUMBOS 3363155852Sgallatin 4096, /* boundary */ 3364175579Sgallatin#else 3365175579Sgallatin 0, /* boundary */ 3366175579Sgallatin#endif 3367155852Sgallatin BUS_SPACE_MAXADDR, /* low */ 3368155852Sgallatin BUS_SPACE_MAXADDR, /* high */ 3369155852Sgallatin NULL, NULL, /* filter */ 3370169840Sgallatin 3*4096, /* maxsize */ 3371175579Sgallatin#if MXGE_VIRT_JUMBOS 3372169840Sgallatin 3, /* num segs */ 3373175579Sgallatin 4096, /* maxsegsize*/ 3374175579Sgallatin#else 3375175579Sgallatin 1, /* num segs */ 3376175579Sgallatin MJUM9BYTES, /* maxsegsize*/ 3377175579Sgallatin#endif 3378155852Sgallatin BUS_DMA_ALLOCNOW, /* flags */ 3379155852Sgallatin NULL, NULL, /* lock */ 3380175365Sgallatin &ss->rx_big.dmat); /* tag */ 3381155852Sgallatin if (err != 0) { 3382155852Sgallatin device_printf(sc->dev, "Err %d allocating rx_big dmat\n", 3383155852Sgallatin err); 3384201758Smbr return err; 3385155852Sgallatin } 3386175365Sgallatin for (i = 0; i <= ss->rx_small.mask; i++) { 3387175365Sgallatin err = bus_dmamap_create(ss->rx_small.dmat, 0, 3388175365Sgallatin &ss->rx_small.info[i].map); 3389155852Sgallatin if (err != 0) { 3390155852Sgallatin device_printf(sc->dev, "Err %d rx_small dmamap\n", 3391159571Sgallatin err); 3392201758Smbr return err; 3393155852Sgallatin } 3394155852Sgallatin } 3395175365Sgallatin err = bus_dmamap_create(ss->rx_small.dmat, 0, 3396175365Sgallatin &ss->rx_small.extra_map); 3397155852Sgallatin if (err != 0) { 3398155852Sgallatin device_printf(sc->dev, "Err %d extra rx_small dmamap\n", 3399155852Sgallatin err); 3400201758Smbr return err; 3401155852Sgallatin } 3402155852Sgallatin 3403175365Sgallatin for (i = 0; i <= ss->rx_big.mask; i++) { 3404175365Sgallatin err = bus_dmamap_create(ss->rx_big.dmat, 0, 3405175365Sgallatin &ss->rx_big.info[i].map); 3406155852Sgallatin if (err != 0) { 3407155852Sgallatin device_printf(sc->dev, "Err %d rx_big dmamap\n", 3408175365Sgallatin err); 3409201758Smbr return err; 3410155852Sgallatin } 3411155852Sgallatin } 3412175365Sgallatin err = bus_dmamap_create(ss->rx_big.dmat, 0, 3413175365Sgallatin &ss->rx_big.extra_map); 3414155852Sgallatin if (err != 0) { 3415155852Sgallatin device_printf(sc->dev, "Err %d extra rx_big dmamap\n", 3416155852Sgallatin err); 3417201758Smbr return err; 3418155852Sgallatin } 3419175365Sgallatin 3420249586Sgabor /* now allocate TX resources */ 3421175365Sgallatin 3422191562Sgallatin#ifndef IFNET_BUF_RING 3423175365Sgallatin /* only use a single TX ring for now */ 3424175365Sgallatin if (ss != ss->sc->ss) 3425175365Sgallatin return 0; 3426191562Sgallatin#endif 3427175365Sgallatin 3428175365Sgallatin ss->tx.mask = tx_ring_entries - 1; 3429175365Sgallatin ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4); 3430175365Sgallatin 3431175365Sgallatin 3432175365Sgallatin /* allocate the tx request copy block */ 3433175365Sgallatin bytes = 8 + 3434175365Sgallatin sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4); 3435175365Sgallatin ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK); 3436175365Sgallatin /* ensure req_list entries are aligned to 8 bytes */ 3437175365Sgallatin ss->tx.req_list = (mcp_kreq_ether_send_t *) 3438175365Sgallatin ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL); 3439175365Sgallatin 3440175365Sgallatin /* allocate the tx busdma segment list */ 3441175365Sgallatin bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc; 3442175365Sgallatin ss->tx.seg_list = (bus_dma_segment_t *) 3443175365Sgallatin malloc(bytes, M_DEVBUF, M_WAITOK); 3444175365Sgallatin 3445175365Sgallatin /* allocate the tx host info ring */ 3446175365Sgallatin bytes = tx_ring_entries * sizeof (*ss->tx.info); 3447175365Sgallatin ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); 3448175365Sgallatin 3449175365Sgallatin /* allocate the tx busdma resources */ 3450175365Sgallatin err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 3451175365Sgallatin 1, /* alignment */ 3452175365Sgallatin sc->tx_boundary, /* boundary */ 3453175365Sgallatin BUS_SPACE_MAXADDR, /* low */ 3454175365Sgallatin BUS_SPACE_MAXADDR, /* high */ 3455175365Sgallatin NULL, NULL, /* filter */ 3456175365Sgallatin 65536 + 256, /* maxsize */ 3457175365Sgallatin ss->tx.max_desc - 2, /* num segs */ 3458175365Sgallatin sc->tx_boundary, /* maxsegsz */ 3459175365Sgallatin BUS_DMA_ALLOCNOW, /* flags */ 3460175365Sgallatin NULL, NULL, /* lock */ 3461175365Sgallatin &ss->tx.dmat); /* tag */ 3462175365Sgallatin 3463175365Sgallatin if (err != 0) { 3464175365Sgallatin device_printf(sc->dev, "Err %d allocating tx dmat\n", 3465175365Sgallatin err); 3466201758Smbr return err; 3467175365Sgallatin } 3468175365Sgallatin 3469175365Sgallatin /* now use these tags to setup dmamaps for each slot 3470175365Sgallatin in the ring */ 3471175365Sgallatin for (i = 0; i <= ss->tx.mask; i++) { 3472175365Sgallatin err = bus_dmamap_create(ss->tx.dmat, 0, 3473175365Sgallatin &ss->tx.info[i].map); 3474175365Sgallatin if (err != 0) { 3475175365Sgallatin device_printf(sc->dev, "Err %d tx dmamap\n", 3476175365Sgallatin err); 3477201758Smbr return err; 3478175365Sgallatin } 3479175365Sgallatin } 3480155852Sgallatin return 0; 3481155852Sgallatin 3482175365Sgallatin} 3483175365Sgallatin 3484175365Sgallatinstatic int 3485175365Sgallatinmxge_alloc_rings(mxge_softc_t *sc) 3486175365Sgallatin{ 3487175365Sgallatin mxge_cmd_t cmd; 3488175365Sgallatin int tx_ring_size; 3489175365Sgallatin int tx_ring_entries, rx_ring_entries; 3490175365Sgallatin int err, slice; 3491175365Sgallatin 3492175365Sgallatin /* get ring sizes */ 3493175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd); 3494175365Sgallatin tx_ring_size = cmd.data0; 3495175365Sgallatin if (err != 0) { 3496175365Sgallatin device_printf(sc->dev, "Cannot determine tx ring sizes\n"); 3497175365Sgallatin goto abort; 3498175365Sgallatin } 3499175365Sgallatin 3500175365Sgallatin tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t); 3501175365Sgallatin rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t); 3502175365Sgallatin IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1); 3503175365Sgallatin sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen; 3504175365Sgallatin IFQ_SET_READY(&sc->ifp->if_snd); 3505175365Sgallatin 3506175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 3507175365Sgallatin err = mxge_alloc_slice_rings(&sc->ss[slice], 3508175365Sgallatin rx_ring_entries, 3509175365Sgallatin tx_ring_entries); 3510175365Sgallatin if (err != 0) 3511175365Sgallatin goto abort; 3512175365Sgallatin } 3513175365Sgallatin return 0; 3514175365Sgallatin 3515175365Sgallatinabort: 3516159571Sgallatin mxge_free_rings(sc); 3517175365Sgallatin return err; 3518155852Sgallatin 3519155852Sgallatin} 3520155852Sgallatin 3521175365Sgallatin 3522169840Sgallatinstatic void 3523169840Sgallatinmxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs) 3524169840Sgallatin{ 3525169905Sgallatin int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 3526169840Sgallatin 3527169840Sgallatin if (bufsize < MCLBYTES) { 3528169840Sgallatin /* easy, everything fits in a single buffer */ 3529169840Sgallatin *big_buf_size = MCLBYTES; 3530169840Sgallatin *cl_size = MCLBYTES; 3531169840Sgallatin *nbufs = 1; 3532169840Sgallatin return; 3533169840Sgallatin } 3534169840Sgallatin 3535169840Sgallatin if (bufsize < MJUMPAGESIZE) { 3536169840Sgallatin /* still easy, everything still fits in a single buffer */ 3537169840Sgallatin *big_buf_size = MJUMPAGESIZE; 3538169840Sgallatin *cl_size = MJUMPAGESIZE; 3539169840Sgallatin *nbufs = 1; 3540169840Sgallatin return; 3541169840Sgallatin } 3542175579Sgallatin#if MXGE_VIRT_JUMBOS 3543169840Sgallatin /* now we need to use virtually contiguous buffers */ 3544169840Sgallatin *cl_size = MJUM9BYTES; 3545169840Sgallatin *big_buf_size = 4096; 3546169840Sgallatin *nbufs = mtu / 4096 + 1; 3547169840Sgallatin /* needs to be a power of two, so round up */ 3548169840Sgallatin if (*nbufs == 3) 3549169840Sgallatin *nbufs = 4; 3550175579Sgallatin#else 3551175579Sgallatin *cl_size = MJUM9BYTES; 3552175579Sgallatin *big_buf_size = MJUM9BYTES; 3553175579Sgallatin *nbufs = 1; 3554175579Sgallatin#endif 3555169840Sgallatin} 3556169840Sgallatin 3557175365Sgallatinstatic int 3558175365Sgallatinmxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size) 3559155852Sgallatin{ 3560175365Sgallatin mxge_softc_t *sc; 3561159571Sgallatin mxge_cmd_t cmd; 3562155852Sgallatin bus_dmamap_t map; 3563175365Sgallatin int err, i, slice; 3564155852Sgallatin 3565155852Sgallatin 3566175365Sgallatin sc = ss->sc; 3567175365Sgallatin slice = ss - sc->ss; 3568175365Sgallatin 3569247133Sgallatin#if defined(INET) || defined(INET6) 3570247133Sgallatin (void)tcp_lro_init(&ss->lc); 3571247133Sgallatin#endif 3572247133Sgallatin ss->lc.ifp = sc->ifp; 3573247133Sgallatin 3574175365Sgallatin /* get the lanai pointers to the send and receive rings */ 3575169840Sgallatin 3576175365Sgallatin err = 0; 3577191562Sgallatin#ifndef IFNET_BUF_RING 3578175365Sgallatin /* We currently only send from the first slice */ 3579175365Sgallatin if (slice == 0) { 3580191562Sgallatin#endif 3581175365Sgallatin cmd.data0 = slice; 3582175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd); 3583175365Sgallatin ss->tx.lanai = 3584175365Sgallatin (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0); 3585191562Sgallatin ss->tx.send_go = (volatile uint32_t *) 3586191562Sgallatin (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 3587191562Sgallatin ss->tx.send_stop = (volatile uint32_t *) 3588191562Sgallatin (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 3589191562Sgallatin#ifndef IFNET_BUF_RING 3590155852Sgallatin } 3591191562Sgallatin#endif 3592175365Sgallatin cmd.data0 = slice; 3593159571Sgallatin err |= mxge_send_cmd(sc, 3594175365Sgallatin MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd); 3595175365Sgallatin ss->rx_small.lanai = 3596155852Sgallatin (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 3597175365Sgallatin cmd.data0 = slice; 3598159612Sgallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd); 3599175365Sgallatin ss->rx_big.lanai = 3600155852Sgallatin (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); 3601155852Sgallatin 3602155852Sgallatin if (err != 0) { 3603155852Sgallatin device_printf(sc->dev, 3604155852Sgallatin "failed to get ring sizes or locations\n"); 3605166370Sgallatin return EIO; 3606155852Sgallatin } 3607155852Sgallatin 3608155852Sgallatin /* stock receive rings */ 3609175365Sgallatin for (i = 0; i <= ss->rx_small.mask; i++) { 3610175365Sgallatin map = ss->rx_small.info[i].map; 3611175365Sgallatin err = mxge_get_buf_small(ss, map, i); 3612155852Sgallatin if (err) { 3613155852Sgallatin device_printf(sc->dev, "alloced %d/%d smalls\n", 3614175365Sgallatin i, ss->rx_small.mask + 1); 3615175365Sgallatin return ENOMEM; 3616155852Sgallatin } 3617155852Sgallatin } 3618175365Sgallatin for (i = 0; i <= ss->rx_big.mask; i++) { 3619175365Sgallatin ss->rx_big.shadow[i].addr_low = 0xffffffff; 3620175365Sgallatin ss->rx_big.shadow[i].addr_high = 0xffffffff; 3621169840Sgallatin } 3622175365Sgallatin ss->rx_big.nbufs = nbufs; 3623175365Sgallatin ss->rx_big.cl_size = cl_size; 3624193250Sgallatin ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN + 3625193250Sgallatin ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; 3626175365Sgallatin for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) { 3627175365Sgallatin map = ss->rx_big.info[i].map; 3628175365Sgallatin err = mxge_get_buf_big(ss, map, i); 3629155852Sgallatin if (err) { 3630155852Sgallatin device_printf(sc->dev, "alloced %d/%d bigs\n", 3631175365Sgallatin i, ss->rx_big.mask + 1); 3632175365Sgallatin return ENOMEM; 3633155852Sgallatin } 3634155852Sgallatin } 3635175365Sgallatin return 0; 3636175365Sgallatin} 3637155852Sgallatin 3638175365Sgallatinstatic int 3639175365Sgallatinmxge_open(mxge_softc_t *sc) 3640175365Sgallatin{ 3641175365Sgallatin mxge_cmd_t cmd; 3642175365Sgallatin int err, big_bytes, nbufs, slice, cl_size, i; 3643175365Sgallatin bus_addr_t bus; 3644175365Sgallatin volatile uint8_t *itable; 3645191562Sgallatin struct mxge_slice_state *ss; 3646175365Sgallatin 3647175365Sgallatin /* Copy the MAC address in case it was overridden */ 3648175365Sgallatin bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN); 3649175365Sgallatin 3650175365Sgallatin err = mxge_reset(sc, 1); 3651175365Sgallatin if (err != 0) { 3652175365Sgallatin device_printf(sc->dev, "failed to reset\n"); 3653175365Sgallatin return EIO; 3654175365Sgallatin } 3655175365Sgallatin 3656175365Sgallatin if (sc->num_slices > 1) { 3657175365Sgallatin /* setup the indirection table */ 3658175365Sgallatin cmd.data0 = sc->num_slices; 3659175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 3660175365Sgallatin &cmd); 3661175365Sgallatin 3662175365Sgallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 3663175365Sgallatin &cmd); 3664175365Sgallatin if (err != 0) { 3665175365Sgallatin device_printf(sc->dev, 3666175365Sgallatin "failed to setup rss tables\n"); 3667175365Sgallatin return err; 3668175365Sgallatin } 3669175365Sgallatin 3670175365Sgallatin /* just enable an identity mapping */ 3671175365Sgallatin itable = sc->sram + cmd.data0; 3672175365Sgallatin for (i = 0; i < sc->num_slices; i++) 3673175365Sgallatin itable[i] = (uint8_t)i; 3674175365Sgallatin 3675175365Sgallatin cmd.data0 = 1; 3676175365Sgallatin cmd.data1 = mxge_rss_hash_type; 3677175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd); 3678175365Sgallatin if (err != 0) { 3679175365Sgallatin device_printf(sc->dev, "failed to enable slices\n"); 3680175365Sgallatin return err; 3681175365Sgallatin } 3682175365Sgallatin } 3683175365Sgallatin 3684175365Sgallatin 3685175365Sgallatin mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs); 3686175365Sgallatin 3687175365Sgallatin cmd.data0 = nbufs; 3688175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, 3689175365Sgallatin &cmd); 3690175365Sgallatin /* error is only meaningful if we're trying to set 3691175365Sgallatin MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */ 3692175365Sgallatin if (err && nbufs > 1) { 3693175365Sgallatin device_printf(sc->dev, 3694175365Sgallatin "Failed to set alway-use-n to %d\n", 3695175365Sgallatin nbufs); 3696175365Sgallatin return EIO; 3697175365Sgallatin } 3698155852Sgallatin /* Give the firmware the mtu and the big and small buffer 3699155852Sgallatin sizes. The firmware wants the big buf size to be a power 3700155852Sgallatin of two. Luckily, FreeBSD's clusters are powers of two */ 3701169905Sgallatin cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3702159612Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd); 3703163467Sgallatin cmd.data0 = MHLEN - MXGEFW_PAD; 3704159612Sgallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, 3705159571Sgallatin &cmd); 3706169840Sgallatin cmd.data0 = big_bytes; 3707159612Sgallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd); 3708162328Sgallatin 3709162328Sgallatin if (err != 0) { 3710162328Sgallatin device_printf(sc->dev, "failed to setup params\n"); 3711162328Sgallatin goto abort; 3712162328Sgallatin } 3713162328Sgallatin 3714155852Sgallatin /* Now give him the pointer to the stats block */ 3715191562Sgallatin for (slice = 0; 3716191562Sgallatin#ifdef IFNET_BUF_RING 3717191562Sgallatin slice < sc->num_slices; 3718191562Sgallatin#else 3719191562Sgallatin slice < 1; 3720191562Sgallatin#endif 3721191562Sgallatin slice++) { 3722191562Sgallatin ss = &sc->ss[slice]; 3723191562Sgallatin cmd.data0 = 3724191562Sgallatin MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr); 3725191562Sgallatin cmd.data1 = 3726191562Sgallatin MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr); 3727191562Sgallatin cmd.data2 = sizeof(struct mcp_irq_data); 3728191562Sgallatin cmd.data2 |= (slice << 16); 3729191562Sgallatin err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd); 3730191562Sgallatin } 3731155852Sgallatin 3732155852Sgallatin if (err != 0) { 3733175365Sgallatin bus = sc->ss->fw_stats_dma.bus_addr; 3734162328Sgallatin bus += offsetof(struct mcp_irq_data, send_done_count); 3735162328Sgallatin cmd.data0 = MXGE_LOWPART_TO_U32(bus); 3736162328Sgallatin cmd.data1 = MXGE_HIGHPART_TO_U32(bus); 3737162328Sgallatin err = mxge_send_cmd(sc, 3738162328Sgallatin MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 3739162328Sgallatin &cmd); 3740162328Sgallatin /* Firmware cannot support multicast without STATS_DMA_V2 */ 3741162328Sgallatin sc->fw_multicast_support = 0; 3742162328Sgallatin } else { 3743162328Sgallatin sc->fw_multicast_support = 1; 3744162328Sgallatin } 3745162328Sgallatin 3746162328Sgallatin if (err != 0) { 3747155852Sgallatin device_printf(sc->dev, "failed to setup params\n"); 3748155852Sgallatin goto abort; 3749155852Sgallatin } 3750155852Sgallatin 3751175365Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 3752175365Sgallatin err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size); 3753175365Sgallatin if (err != 0) { 3754175365Sgallatin device_printf(sc->dev, "couldn't open slice %d\n", 3755175365Sgallatin slice); 3756175365Sgallatin goto abort; 3757175365Sgallatin } 3758175365Sgallatin } 3759175365Sgallatin 3760155852Sgallatin /* Finally, start the firmware running */ 3761159612Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd); 3762155852Sgallatin if (err) { 3763155852Sgallatin device_printf(sc->dev, "Couldn't bring up link\n"); 3764155852Sgallatin goto abort; 3765155852Sgallatin } 3766191562Sgallatin#ifdef IFNET_BUF_RING 3767191562Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 3768191562Sgallatin ss = &sc->ss[slice]; 3769191562Sgallatin ss->if_drv_flags |= IFF_DRV_RUNNING; 3770191562Sgallatin ss->if_drv_flags &= ~IFF_DRV_OACTIVE; 3771191562Sgallatin } 3772191562Sgallatin#endif 3773155852Sgallatin sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 3774155852Sgallatin sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3775155852Sgallatin 3776155852Sgallatin return 0; 3777155852Sgallatin 3778155852Sgallatin 3779155852Sgallatinabort: 3780159571Sgallatin mxge_free_mbufs(sc); 3781166370Sgallatin 3782155852Sgallatin return err; 3783155852Sgallatin} 3784155852Sgallatin 3785155852Sgallatinstatic int 3786197395Sgallatinmxge_close(mxge_softc_t *sc, int down) 3787155852Sgallatin{ 3788159571Sgallatin mxge_cmd_t cmd; 3789155852Sgallatin int err, old_down_cnt; 3790191562Sgallatin#ifdef IFNET_BUF_RING 3791191562Sgallatin struct mxge_slice_state *ss; 3792191562Sgallatin int slice; 3793191562Sgallatin#endif 3794155852Sgallatin 3795191562Sgallatin#ifdef IFNET_BUF_RING 3796191562Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 3797191562Sgallatin ss = &sc->ss[slice]; 3798191562Sgallatin ss->if_drv_flags &= ~IFF_DRV_RUNNING; 3799191562Sgallatin } 3800191562Sgallatin#endif 3801155852Sgallatin sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3802197395Sgallatin if (!down) { 3803197395Sgallatin old_down_cnt = sc->down_cnt; 3804197395Sgallatin wmb(); 3805197395Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); 3806197395Sgallatin if (err) { 3807197395Sgallatin device_printf(sc->dev, 3808197395Sgallatin "Couldn't bring down link\n"); 3809197395Sgallatin } 3810197395Sgallatin if (old_down_cnt == sc->down_cnt) { 3811197395Sgallatin /* wait for down irq */ 3812197395Sgallatin DELAY(10 * sc->intr_coal_delay); 3813197395Sgallatin } 3814197395Sgallatin wmb(); 3815197395Sgallatin if (old_down_cnt == sc->down_cnt) { 3816197395Sgallatin device_printf(sc->dev, "never got down irq\n"); 3817197395Sgallatin } 3818155852Sgallatin } 3819159571Sgallatin mxge_free_mbufs(sc); 3820166370Sgallatin 3821155852Sgallatin return 0; 3822155852Sgallatin} 3823155852Sgallatin 3824166373Sgallatinstatic void 3825166373Sgallatinmxge_setup_cfg_space(mxge_softc_t *sc) 3826166373Sgallatin{ 3827166373Sgallatin device_t dev = sc->dev; 3828166373Sgallatin int reg; 3829254263Sscottl uint16_t lnk, pectl; 3830155852Sgallatin 3831166373Sgallatin /* find the PCIe link width and set max read request to 4KB*/ 3832219902Sjhb if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 3833166373Sgallatin lnk = pci_read_config(dev, reg + 0x12, 2); 3834166373Sgallatin sc->link_width = (lnk >> 4) & 0x3f; 3835197645Sgallatin 3836197645Sgallatin if (sc->pectl == 0) { 3837197645Sgallatin pectl = pci_read_config(dev, reg + 0x8, 2); 3838197645Sgallatin pectl = (pectl & ~0x7000) | (5 << 12); 3839197645Sgallatin pci_write_config(dev, reg + 0x8, pectl, 2); 3840197645Sgallatin sc->pectl = pectl; 3841197645Sgallatin } else { 3842197645Sgallatin /* restore saved pectl after watchdog reset */ 3843197645Sgallatin pci_write_config(dev, reg + 0x8, sc->pectl, 2); 3844197645Sgallatin } 3845166373Sgallatin } 3846166373Sgallatin 3847166373Sgallatin /* Enable DMA and Memory space access */ 3848166373Sgallatin pci_enable_busmaster(dev); 3849166373Sgallatin} 3850166373Sgallatin 3851166373Sgallatinstatic uint32_t 3852166373Sgallatinmxge_read_reboot(mxge_softc_t *sc) 3853166373Sgallatin{ 3854166373Sgallatin device_t dev = sc->dev; 3855166373Sgallatin uint32_t vs; 3856166373Sgallatin 3857166373Sgallatin /* find the vendor specific offset */ 3858219902Sjhb if (pci_find_cap(dev, PCIY_VENDOR, &vs) != 0) { 3859166373Sgallatin device_printf(sc->dev, 3860166373Sgallatin "could not find vendor specific offset\n"); 3861166373Sgallatin return (uint32_t)-1; 3862166373Sgallatin } 3863166373Sgallatin /* enable read32 mode */ 3864166373Sgallatin pci_write_config(dev, vs + 0x10, 0x3, 1); 3865166373Sgallatin /* tell NIC which register to read */ 3866166373Sgallatin pci_write_config(dev, vs + 0x18, 0xfffffff0, 4); 3867166373Sgallatin return (pci_read_config(dev, vs + 0x14, 4)); 3868166373Sgallatin} 3869166373Sgallatin 3870198250Sgallatinstatic void 3871198250Sgallatinmxge_watchdog_reset(mxge_softc_t *sc) 3872166373Sgallatin{ 3873180567Sgallatin struct pci_devinfo *dinfo; 3874197395Sgallatin struct mxge_slice_state *ss; 3875197395Sgallatin int err, running, s, num_tx_slices = 1; 3876166373Sgallatin uint32_t reboot; 3877166373Sgallatin uint16_t cmd; 3878166373Sgallatin 3879166373Sgallatin err = ENXIO; 3880166373Sgallatin 3881166373Sgallatin device_printf(sc->dev, "Watchdog reset!\n"); 3882166373Sgallatin 3883166373Sgallatin /* 3884166373Sgallatin * check to see if the NIC rebooted. If it did, then all of 3885166373Sgallatin * PCI config space has been reset, and things like the 3886166373Sgallatin * busmaster bit will be zero. If this is the case, then we 3887166373Sgallatin * must restore PCI config space before the NIC can be used 3888166373Sgallatin * again 3889166373Sgallatin */ 3890166373Sgallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3891166373Sgallatin if (cmd == 0xffff) { 3892166373Sgallatin /* 3893166373Sgallatin * maybe the watchdog caught the NIC rebooting; wait 3894166373Sgallatin * up to 100ms for it to finish. If it does not come 3895166373Sgallatin * back, then give up 3896166373Sgallatin */ 3897166373Sgallatin DELAY(1000*100); 3898166373Sgallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 3899166373Sgallatin if (cmd == 0xffff) { 3900166373Sgallatin device_printf(sc->dev, "NIC disappeared!\n"); 3901166373Sgallatin } 3902166373Sgallatin } 3903166373Sgallatin if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 3904166373Sgallatin /* print the reboot status */ 3905166373Sgallatin reboot = mxge_read_reboot(sc); 3906166373Sgallatin device_printf(sc->dev, "NIC rebooted, status = 0x%x\n", 3907166373Sgallatin reboot); 3908197395Sgallatin running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 3909197395Sgallatin if (running) { 3910197395Sgallatin 3911197395Sgallatin /* 3912197395Sgallatin * quiesce NIC so that TX routines will not try to 3913197395Sgallatin * xmit after restoration of BAR 3914197395Sgallatin */ 3915197395Sgallatin 3916197395Sgallatin /* Mark the link as down */ 3917197395Sgallatin if (sc->link_state) { 3918197395Sgallatin sc->link_state = 0; 3919197395Sgallatin if_link_state_change(sc->ifp, 3920197395Sgallatin LINK_STATE_DOWN); 3921197395Sgallatin } 3922197395Sgallatin#ifdef IFNET_BUF_RING 3923197395Sgallatin num_tx_slices = sc->num_slices; 3924197395Sgallatin#endif 3925197395Sgallatin /* grab all TX locks to ensure no tx */ 3926197395Sgallatin for (s = 0; s < num_tx_slices; s++) { 3927197395Sgallatin ss = &sc->ss[s]; 3928197395Sgallatin mtx_lock(&ss->tx.mtx); 3929197395Sgallatin } 3930197395Sgallatin mxge_close(sc, 1); 3931197395Sgallatin } 3932166373Sgallatin /* restore PCI configuration space */ 3933180567Sgallatin dinfo = device_get_ivars(sc->dev); 3934180567Sgallatin pci_cfg_restore(sc->dev, dinfo); 3935166373Sgallatin 3936166373Sgallatin /* and redo any changes we made to our config space */ 3937166373Sgallatin mxge_setup_cfg_space(sc); 3938175757Sgallatin 3939197395Sgallatin /* reload f/w */ 3940197395Sgallatin err = mxge_load_firmware(sc, 0); 3941197395Sgallatin if (err) { 3942197395Sgallatin device_printf(sc->dev, 3943197395Sgallatin "Unable to re-load f/w\n"); 3944175757Sgallatin } 3945197395Sgallatin if (running) { 3946197395Sgallatin if (!err) 3947197395Sgallatin err = mxge_open(sc); 3948197395Sgallatin /* release all TX locks */ 3949197395Sgallatin for (s = 0; s < num_tx_slices; s++) { 3950197395Sgallatin ss = &sc->ss[s]; 3951197645Sgallatin#ifdef IFNET_BUF_RING 3952197645Sgallatin mxge_start_locked(ss); 3953197645Sgallatin#endif 3954197395Sgallatin mtx_unlock(&ss->tx.mtx); 3955197395Sgallatin } 3956197395Sgallatin } 3957197395Sgallatin sc->watchdog_resets++; 3958166373Sgallatin } else { 3959191562Sgallatin device_printf(sc->dev, 3960198250Sgallatin "NIC did not reboot, not resetting\n"); 3961198250Sgallatin err = 0; 3962166373Sgallatin } 3963198250Sgallatin if (err) { 3964197395Sgallatin device_printf(sc->dev, "watchdog reset failed\n"); 3965198250Sgallatin } else { 3966198303Sgallatin if (sc->dying == 2) 3967198303Sgallatin sc->dying = 0; 3968198303Sgallatin callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 3969198250Sgallatin } 3970198250Sgallatin} 3971197395Sgallatin 3972198250Sgallatinstatic void 3973198250Sgallatinmxge_watchdog_task(void *arg, int pending) 3974198250Sgallatin{ 3975198250Sgallatin mxge_softc_t *sc = arg; 3976198250Sgallatin 3977198250Sgallatin 3978198250Sgallatin mtx_lock(&sc->driver_mtx); 3979198250Sgallatin mxge_watchdog_reset(sc); 3980198250Sgallatin mtx_unlock(&sc->driver_mtx); 3981166373Sgallatin} 3982166373Sgallatin 3983198250Sgallatinstatic void 3984198250Sgallatinmxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice) 3985198250Sgallatin{ 3986198250Sgallatin tx = &sc->ss[slice].tx; 3987198250Sgallatin device_printf(sc->dev, "slice %d struck? ring state:\n", slice); 3988198250Sgallatin device_printf(sc->dev, 3989198250Sgallatin "tx.req=%d tx.done=%d, tx.queue_active=%d\n", 3990198250Sgallatin tx->req, tx->done, tx->queue_active); 3991198250Sgallatin device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n", 3992198250Sgallatin tx->activate, tx->deactivate); 3993198250Sgallatin device_printf(sc->dev, "pkt_done=%d fw=%d\n", 3994198250Sgallatin tx->pkt_done, 3995198250Sgallatin be32toh(sc->ss->fw_stats->send_done_count)); 3996198250Sgallatin} 3997198250Sgallatin 3998180567Sgallatinstatic int 3999166373Sgallatinmxge_watchdog(mxge_softc_t *sc) 4000166373Sgallatin{ 4001191562Sgallatin mxge_tx_ring_t *tx; 4002175365Sgallatin uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause); 4003191562Sgallatin int i, err = 0; 4004166373Sgallatin 4005166373Sgallatin /* see if we have outstanding transmits, which 4006166373Sgallatin have been pending for more than mxge_ticks */ 4007191562Sgallatin for (i = 0; 4008191562Sgallatin#ifdef IFNET_BUF_RING 4009191562Sgallatin (i < sc->num_slices) && (err == 0); 4010191562Sgallatin#else 4011191562Sgallatin (i < 1) && (err == 0); 4012191562Sgallatin#endif 4013191562Sgallatin i++) { 4014191562Sgallatin tx = &sc->ss[i].tx; 4015191562Sgallatin if (tx->req != tx->done && 4016191562Sgallatin tx->watchdog_req != tx->watchdog_done && 4017191562Sgallatin tx->done == tx->watchdog_done) { 4018191562Sgallatin /* check for pause blocking before resetting */ 4019198250Sgallatin if (tx->watchdog_rx_pause == rx_pause) { 4020198250Sgallatin mxge_warn_stuck(sc, tx, i); 4021198250Sgallatin taskqueue_enqueue(sc->tq, &sc->watchdog_task); 4022198250Sgallatin return (ENXIO); 4023198250Sgallatin } 4024191562Sgallatin else 4025191562Sgallatin device_printf(sc->dev, "Flow control blocking " 4026191562Sgallatin "xmits, check link partner\n"); 4027191562Sgallatin } 4028191562Sgallatin 4029191562Sgallatin tx->watchdog_req = tx->req; 4030191562Sgallatin tx->watchdog_done = tx->done; 4031191562Sgallatin tx->watchdog_rx_pause = rx_pause; 4032171917Sgallatin } 4033166373Sgallatin 4034171917Sgallatin if (sc->need_media_probe) 4035171917Sgallatin mxge_media_probe(sc); 4036180567Sgallatin return (err); 4037166373Sgallatin} 4038166373Sgallatin 4039198303Sgallatinstatic u_long 4040175365Sgallatinmxge_update_stats(mxge_softc_t *sc) 4041175365Sgallatin{ 4042175365Sgallatin struct mxge_slice_state *ss; 4043198303Sgallatin u_long pkts = 0; 4044175365Sgallatin u_long ipackets = 0; 4045191562Sgallatin u_long opackets = 0; 4046194751Sgallatin#ifdef IFNET_BUF_RING 4047194751Sgallatin u_long obytes = 0; 4048194751Sgallatin u_long omcasts = 0; 4049194751Sgallatin u_long odrops = 0; 4050194751Sgallatin#endif 4051191562Sgallatin u_long oerrors = 0; 4052175365Sgallatin int slice; 4053175365Sgallatin 4054191562Sgallatin for (slice = 0; slice < sc->num_slices; slice++) { 4055175365Sgallatin ss = &sc->ss[slice]; 4056175365Sgallatin ipackets += ss->ipackets; 4057191562Sgallatin opackets += ss->opackets; 4058194751Sgallatin#ifdef IFNET_BUF_RING 4059194751Sgallatin obytes += ss->obytes; 4060194751Sgallatin omcasts += ss->omcasts; 4061194751Sgallatin odrops += ss->tx.br->br_drops; 4062194751Sgallatin#endif 4063191562Sgallatin oerrors += ss->oerrors; 4064175365Sgallatin } 4065198303Sgallatin pkts = (ipackets - sc->ifp->if_ipackets); 4066198303Sgallatin pkts += (opackets - sc->ifp->if_opackets); 4067175365Sgallatin sc->ifp->if_ipackets = ipackets; 4068191562Sgallatin sc->ifp->if_opackets = opackets; 4069194751Sgallatin#ifdef IFNET_BUF_RING 4070194751Sgallatin sc->ifp->if_obytes = obytes; 4071194751Sgallatin sc->ifp->if_omcasts = omcasts; 4072194751Sgallatin sc->ifp->if_snd.ifq_drops = odrops; 4073194751Sgallatin#endif 4074191562Sgallatin sc->ifp->if_oerrors = oerrors; 4075198303Sgallatin return pkts; 4076191562Sgallatin} 4077175365Sgallatin 4078175365Sgallatinstatic void 4079166373Sgallatinmxge_tick(void *arg) 4080166373Sgallatin{ 4081166373Sgallatin mxge_softc_t *sc = arg; 4082198303Sgallatin u_long pkts = 0; 4083180567Sgallatin int err = 0; 4084198303Sgallatin int running, ticks; 4085198303Sgallatin uint16_t cmd; 4086166373Sgallatin 4087198303Sgallatin ticks = mxge_ticks; 4088198303Sgallatin running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; 4089198303Sgallatin if (running) { 4090198303Sgallatin /* aggregate stats from different slices */ 4091198303Sgallatin pkts = mxge_update_stats(sc); 4092198303Sgallatin if (!sc->watchdog_countdown) { 4093198303Sgallatin err = mxge_watchdog(sc); 4094198303Sgallatin sc->watchdog_countdown = 4; 4095198303Sgallatin } 4096198303Sgallatin sc->watchdog_countdown--; 4097175365Sgallatin } 4098198303Sgallatin if (pkts == 0) { 4099198303Sgallatin /* ensure NIC did not suffer h/w fault while idle */ 4100198303Sgallatin cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); 4101198303Sgallatin if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { 4102198303Sgallatin sc->dying = 2; 4103198303Sgallatin taskqueue_enqueue(sc->tq, &sc->watchdog_task); 4104198303Sgallatin err = ENXIO; 4105198303Sgallatin } 4106198303Sgallatin /* look less often if NIC is idle */ 4107198303Sgallatin ticks *= 4; 4108198303Sgallatin } 4109198303Sgallatin 4110180567Sgallatin if (err == 0) 4111198303Sgallatin callout_reset(&sc->co_hdl, ticks, mxge_tick, sc); 4112180567Sgallatin 4113166373Sgallatin} 4114166373Sgallatin 4115155852Sgallatinstatic int 4116159571Sgallatinmxge_media_change(struct ifnet *ifp) 4117155852Sgallatin{ 4118155852Sgallatin return EINVAL; 4119155852Sgallatin} 4120155852Sgallatin 4121155852Sgallatinstatic int 4122159571Sgallatinmxge_change_mtu(mxge_softc_t *sc, int mtu) 4123155852Sgallatin{ 4124155852Sgallatin struct ifnet *ifp = sc->ifp; 4125155852Sgallatin int real_mtu, old_mtu; 4126155852Sgallatin int err = 0; 4127155852Sgallatin 4128155852Sgallatin 4129169905Sgallatin real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 4130169840Sgallatin if ((real_mtu > sc->max_mtu) || real_mtu < 60) 4131155852Sgallatin return EINVAL; 4132166370Sgallatin mtx_lock(&sc->driver_mtx); 4133155852Sgallatin old_mtu = ifp->if_mtu; 4134155852Sgallatin ifp->if_mtu = mtu; 4135155852Sgallatin if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4136197395Sgallatin mxge_close(sc, 0); 4137159571Sgallatin err = mxge_open(sc); 4138155852Sgallatin if (err != 0) { 4139155852Sgallatin ifp->if_mtu = old_mtu; 4140197395Sgallatin mxge_close(sc, 0); 4141159571Sgallatin (void) mxge_open(sc); 4142155852Sgallatin } 4143155852Sgallatin } 4144166370Sgallatin mtx_unlock(&sc->driver_mtx); 4145155852Sgallatin return err; 4146155852Sgallatin} 4147155852Sgallatin 4148155852Sgallatinstatic void 4149159571Sgallatinmxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 4150155852Sgallatin{ 4151159571Sgallatin mxge_softc_t *sc = ifp->if_softc; 4152155852Sgallatin 4153155852Sgallatin 4154155852Sgallatin if (sc == NULL) 4155155852Sgallatin return; 4156155852Sgallatin ifmr->ifm_status = IFM_AVALID; 4157206662Sgallatin ifmr->ifm_active = IFM_ETHER | IFM_FDX; 4158171917Sgallatin ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0; 4159206662Sgallatin ifmr->ifm_active |= sc->current_media; 4160155852Sgallatin} 4161155852Sgallatin 4162155852Sgallatinstatic int 4163159571Sgallatinmxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 4164155852Sgallatin{ 4165159571Sgallatin mxge_softc_t *sc = ifp->if_softc; 4166155852Sgallatin struct ifreq *ifr = (struct ifreq *)data; 4167155852Sgallatin int err, mask; 4168155852Sgallatin 4169155852Sgallatin err = 0; 4170155852Sgallatin switch (command) { 4171155852Sgallatin case SIOCSIFMTU: 4172159571Sgallatin err = mxge_change_mtu(sc, ifr->ifr_mtu); 4173155852Sgallatin break; 4174155852Sgallatin 4175155852Sgallatin case SIOCSIFFLAGS: 4176166370Sgallatin mtx_lock(&sc->driver_mtx); 4177194909Sgallatin if (sc->dying) { 4178194909Sgallatin mtx_unlock(&sc->driver_mtx); 4179194909Sgallatin return EINVAL; 4180194909Sgallatin } 4181155852Sgallatin if (ifp->if_flags & IFF_UP) { 4182166373Sgallatin if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4183159571Sgallatin err = mxge_open(sc); 4184166373Sgallatin } else { 4185162328Sgallatin /* take care of promis can allmulti 4186162328Sgallatin flag chages */ 4187162328Sgallatin mxge_change_promisc(sc, 4188162328Sgallatin ifp->if_flags & IFF_PROMISC); 4189162328Sgallatin mxge_set_multicast_list(sc); 4190162328Sgallatin } 4191155852Sgallatin } else { 4192166373Sgallatin if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 4193197395Sgallatin mxge_close(sc, 0); 4194166373Sgallatin } 4195155852Sgallatin } 4196166370Sgallatin mtx_unlock(&sc->driver_mtx); 4197155852Sgallatin break; 4198155852Sgallatin 4199155852Sgallatin case SIOCADDMULTI: 4200155852Sgallatin case SIOCDELMULTI: 4201166370Sgallatin mtx_lock(&sc->driver_mtx); 4202162328Sgallatin mxge_set_multicast_list(sc); 4203166370Sgallatin mtx_unlock(&sc->driver_mtx); 4204155852Sgallatin break; 4205155852Sgallatin 4206155852Sgallatin case SIOCSIFCAP: 4207166370Sgallatin mtx_lock(&sc->driver_mtx); 4208155852Sgallatin mask = ifr->ifr_reqcap ^ ifp->if_capenable; 4209155852Sgallatin if (mask & IFCAP_TXCSUM) { 4210155852Sgallatin if (IFCAP_TXCSUM & ifp->if_capenable) { 4211162322Sgallatin ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 4212247011Sgallatin ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP); 4213155852Sgallatin } else { 4214155852Sgallatin ifp->if_capenable |= IFCAP_TXCSUM; 4215155852Sgallatin ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 4216155852Sgallatin } 4217155852Sgallatin } else if (mask & IFCAP_RXCSUM) { 4218155852Sgallatin if (IFCAP_RXCSUM & ifp->if_capenable) { 4219155852Sgallatin ifp->if_capenable &= ~IFCAP_RXCSUM; 4220155852Sgallatin } else { 4221155852Sgallatin ifp->if_capenable |= IFCAP_RXCSUM; 4222155852Sgallatin } 4223155852Sgallatin } 4224162322Sgallatin if (mask & IFCAP_TSO4) { 4225162322Sgallatin if (IFCAP_TSO4 & ifp->if_capenable) { 4226162322Sgallatin ifp->if_capenable &= ~IFCAP_TSO4; 4227162322Sgallatin } else if (IFCAP_TXCSUM & ifp->if_capenable) { 4228162322Sgallatin ifp->if_capenable |= IFCAP_TSO4; 4229162322Sgallatin ifp->if_hwassist |= CSUM_TSO; 4230162322Sgallatin } else { 4231162322Sgallatin printf("mxge requires tx checksum offload" 4232162322Sgallatin " be enabled to use TSO\n"); 4233162322Sgallatin err = EINVAL; 4234162322Sgallatin } 4235162322Sgallatin } 4236247011Sgallatin#if IFCAP_TSO6 4237247011Sgallatin if (mask & IFCAP_TXCSUM_IPV6) { 4238247011Sgallatin if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 4239247011Sgallatin ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 4240247011Sgallatin | IFCAP_TSO6); 4241247011Sgallatin ifp->if_hwassist &= ~(CSUM_TCP_IPV6 4242247011Sgallatin | CSUM_UDP); 4243247011Sgallatin } else { 4244247011Sgallatin ifp->if_capenable |= IFCAP_TXCSUM_IPV6; 4245247011Sgallatin ifp->if_hwassist |= (CSUM_TCP_IPV6 4246247011Sgallatin | CSUM_UDP_IPV6); 4247247011Sgallatin } 4248247133Sgallatin } else if (mask & IFCAP_RXCSUM_IPV6) { 4249247133Sgallatin if (IFCAP_RXCSUM_IPV6 & ifp->if_capenable) { 4250247133Sgallatin ifp->if_capenable &= ~IFCAP_RXCSUM_IPV6; 4251247011Sgallatin } else { 4252247133Sgallatin ifp->if_capenable |= IFCAP_RXCSUM_IPV6; 4253247011Sgallatin } 4254247011Sgallatin } 4255247011Sgallatin if (mask & IFCAP_TSO6) { 4256247011Sgallatin if (IFCAP_TSO6 & ifp->if_capenable) { 4257247011Sgallatin ifp->if_capenable &= ~IFCAP_TSO6; 4258247011Sgallatin } else if (IFCAP_TXCSUM_IPV6 & ifp->if_capenable) { 4259247011Sgallatin ifp->if_capenable |= IFCAP_TSO6; 4260247011Sgallatin ifp->if_hwassist |= CSUM_TSO; 4261247011Sgallatin } else { 4262247011Sgallatin printf("mxge requires tx checksum offload" 4263247011Sgallatin " be enabled to use TSO\n"); 4264247011Sgallatin err = EINVAL; 4265247011Sgallatin } 4266247011Sgallatin } 4267247011Sgallatin#endif /*IFCAP_TSO6 */ 4268247011Sgallatin 4269247133Sgallatin if (mask & IFCAP_LRO) 4270247133Sgallatin ifp->if_capenable ^= IFCAP_LRO; 4271169905Sgallatin if (mask & IFCAP_VLAN_HWTAGGING) 4272169905Sgallatin ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 4273204212Sgallatin if (mask & IFCAP_VLAN_HWTSO) 4274204212Sgallatin ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 4275204212Sgallatin 4276204212Sgallatin if (!(ifp->if_capabilities & IFCAP_VLAN_HWTSO) || 4277204212Sgallatin !(ifp->if_capenable & IFCAP_VLAN_HWTAGGING)) 4278204212Sgallatin ifp->if_capenable &= ~IFCAP_VLAN_HWTSO; 4279204212Sgallatin 4280166370Sgallatin mtx_unlock(&sc->driver_mtx); 4281169905Sgallatin VLAN_CAPABILITIES(ifp); 4282169905Sgallatin 4283155852Sgallatin break; 4284155852Sgallatin 4285155852Sgallatin case SIOCGIFMEDIA: 4286206662Sgallatin mtx_lock(&sc->driver_mtx); 4287206662Sgallatin mxge_media_probe(sc); 4288206662Sgallatin mtx_unlock(&sc->driver_mtx); 4289155852Sgallatin err = ifmedia_ioctl(ifp, (struct ifreq *)data, 4290155852Sgallatin &sc->media, command); 4291155852Sgallatin break; 4292155852Sgallatin 4293155852Sgallatin default: 4294329834Srpokala err = ether_ioctl(ifp, command, data); 4295329834Srpokala break; 4296155852Sgallatin } 4297155852Sgallatin return err; 4298155852Sgallatin} 4299155852Sgallatin 4300155852Sgallatinstatic void 4301159571Sgallatinmxge_fetch_tunables(mxge_softc_t *sc) 4302155852Sgallatin{ 4303175365Sgallatin 4304175365Sgallatin TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices); 4305159571Sgallatin TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled", 4306159571Sgallatin &mxge_flow_control); 4307159571Sgallatin TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay", 4308159571Sgallatin &mxge_intr_coal_delay); 4309159571Sgallatin TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable", 4310159571Sgallatin &mxge_nvidia_ecrc_enable); 4311164513Sgallatin TUNABLE_INT_FETCH("hw.mxge.force_firmware", 4312164513Sgallatin &mxge_force_firmware); 4313159612Sgallatin TUNABLE_INT_FETCH("hw.mxge.deassert_wait", 4314159612Sgallatin &mxge_deassert_wait); 4315159612Sgallatin TUNABLE_INT_FETCH("hw.mxge.verbose", 4316159612Sgallatin &mxge_verbose); 4317166373Sgallatin TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks); 4318175365Sgallatin TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc); 4319175365Sgallatin TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type); 4320195818Sgallatin TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type); 4321194836Sgallatin TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu); 4322197391Sgallatin TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle); 4323155852Sgallatin 4324159612Sgallatin if (bootverbose) 4325159612Sgallatin mxge_verbose = 1; 4326159571Sgallatin if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000) 4327159571Sgallatin mxge_intr_coal_delay = 30; 4328166373Sgallatin if (mxge_ticks == 0) 4329175365Sgallatin mxge_ticks = hz / 2; 4330159571Sgallatin sc->pause = mxge_flow_control; 4331175365Sgallatin if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4 4332194761Sgallatin || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) { 4333202121Sgallatin mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 4334175365Sgallatin } 4335194836Sgallatin if (mxge_initial_mtu > ETHERMTU_JUMBO || 4336194836Sgallatin mxge_initial_mtu < ETHER_MIN_LEN) 4337194836Sgallatin mxge_initial_mtu = ETHERMTU_JUMBO; 4338197391Sgallatin 4339197391Sgallatin if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE) 4340197391Sgallatin mxge_throttle = MXGE_MAX_THROTTLE; 4341197391Sgallatin if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE) 4342197391Sgallatin mxge_throttle = MXGE_MIN_THROTTLE; 4343197391Sgallatin sc->throttle = mxge_throttle; 4344175365Sgallatin} 4345169840Sgallatin 4346175365Sgallatin 4347175365Sgallatinstatic void 4348175365Sgallatinmxge_free_slices(mxge_softc_t *sc) 4349175365Sgallatin{ 4350175365Sgallatin struct mxge_slice_state *ss; 4351175365Sgallatin int i; 4352175365Sgallatin 4353175365Sgallatin 4354175365Sgallatin if (sc->ss == NULL) 4355175365Sgallatin return; 4356175365Sgallatin 4357175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4358175365Sgallatin ss = &sc->ss[i]; 4359175365Sgallatin if (ss->fw_stats != NULL) { 4360175365Sgallatin mxge_dma_free(&ss->fw_stats_dma); 4361175365Sgallatin ss->fw_stats = NULL; 4362191562Sgallatin#ifdef IFNET_BUF_RING 4363191562Sgallatin if (ss->tx.br != NULL) { 4364191562Sgallatin drbr_free(ss->tx.br, M_DEVBUF); 4365191562Sgallatin ss->tx.br = NULL; 4366191562Sgallatin } 4367191562Sgallatin#endif 4368175365Sgallatin mtx_destroy(&ss->tx.mtx); 4369175365Sgallatin } 4370175365Sgallatin if (ss->rx_done.entry != NULL) { 4371175365Sgallatin mxge_dma_free(&ss->rx_done.dma); 4372175365Sgallatin ss->rx_done.entry = NULL; 4373175365Sgallatin } 4374175365Sgallatin } 4375175365Sgallatin free(sc->ss, M_DEVBUF); 4376175365Sgallatin sc->ss = NULL; 4377155852Sgallatin} 4378155852Sgallatin 4379175365Sgallatinstatic int 4380175365Sgallatinmxge_alloc_slices(mxge_softc_t *sc) 4381175365Sgallatin{ 4382175365Sgallatin mxge_cmd_t cmd; 4383175365Sgallatin struct mxge_slice_state *ss; 4384175365Sgallatin size_t bytes; 4385175365Sgallatin int err, i, max_intr_slots; 4386175365Sgallatin 4387175365Sgallatin err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 4388175365Sgallatin if (err != 0) { 4389175365Sgallatin device_printf(sc->dev, "Cannot determine rx ring size\n"); 4390175365Sgallatin return err; 4391175365Sgallatin } 4392175365Sgallatin sc->rx_ring_size = cmd.data0; 4393175365Sgallatin max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t)); 4394175365Sgallatin 4395175365Sgallatin bytes = sizeof (*sc->ss) * sc->num_slices; 4396175365Sgallatin sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 4397175365Sgallatin if (sc->ss == NULL) 4398175365Sgallatin return (ENOMEM); 4399175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4400175365Sgallatin ss = &sc->ss[i]; 4401175365Sgallatin 4402175365Sgallatin ss->sc = sc; 4403175365Sgallatin 4404175365Sgallatin /* allocate per-slice rx interrupt queues */ 4405175365Sgallatin 4406175365Sgallatin bytes = max_intr_slots * sizeof (*ss->rx_done.entry); 4407175365Sgallatin err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096); 4408175365Sgallatin if (err != 0) 4409175365Sgallatin goto abort; 4410175365Sgallatin ss->rx_done.entry = ss->rx_done.dma.addr; 4411175365Sgallatin bzero(ss->rx_done.entry, bytes); 4412175365Sgallatin 4413175365Sgallatin /* 4414175365Sgallatin * allocate the per-slice firmware stats; stats 4415175365Sgallatin * (including tx) are used used only on the first 4416175365Sgallatin * slice for now 4417175365Sgallatin */ 4418191562Sgallatin#ifndef IFNET_BUF_RING 4419175365Sgallatin if (i > 0) 4420175365Sgallatin continue; 4421191562Sgallatin#endif 4422175365Sgallatin 4423175365Sgallatin bytes = sizeof (*ss->fw_stats); 4424175365Sgallatin err = mxge_dma_alloc(sc, &ss->fw_stats_dma, 4425175365Sgallatin sizeof (*ss->fw_stats), 64); 4426175365Sgallatin if (err != 0) 4427175365Sgallatin goto abort; 4428175365Sgallatin ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr; 4429175365Sgallatin snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name), 4430175365Sgallatin "%s:tx(%d)", device_get_nameunit(sc->dev), i); 4431175365Sgallatin mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF); 4432191562Sgallatin#ifdef IFNET_BUF_RING 4433191562Sgallatin ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, 4434191562Sgallatin &ss->tx.mtx); 4435191562Sgallatin#endif 4436175365Sgallatin } 4437175365Sgallatin 4438175365Sgallatin return (0); 4439175365Sgallatin 4440175365Sgallatinabort: 4441175365Sgallatin mxge_free_slices(sc); 4442175365Sgallatin return (ENOMEM); 4443175365Sgallatin} 4444175365Sgallatin 4445175365Sgallatinstatic void 4446175365Sgallatinmxge_slice_probe(mxge_softc_t *sc) 4447175365Sgallatin{ 4448175365Sgallatin mxge_cmd_t cmd; 4449175365Sgallatin char *old_fw; 4450175365Sgallatin int msix_cnt, status, max_intr_slots; 4451175365Sgallatin 4452175365Sgallatin sc->num_slices = 1; 4453175365Sgallatin /* 4454175365Sgallatin * don't enable multiple slices if they are not enabled, 4455175365Sgallatin * or if this is not an SMP system 4456175365Sgallatin */ 4457175365Sgallatin 4458175365Sgallatin if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2) 4459175365Sgallatin return; 4460175365Sgallatin 4461175365Sgallatin /* see how many MSI-X interrupts are available */ 4462175365Sgallatin msix_cnt = pci_msix_count(sc->dev); 4463175365Sgallatin if (msix_cnt < 2) 4464175365Sgallatin return; 4465175365Sgallatin 4466175365Sgallatin /* now load the slice aware firmware see what it supports */ 4467175365Sgallatin old_fw = sc->fw_name; 4468175365Sgallatin if (old_fw == mxge_fw_aligned) 4469175365Sgallatin sc->fw_name = mxge_fw_rss_aligned; 4470175365Sgallatin else 4471175365Sgallatin sc->fw_name = mxge_fw_rss_unaligned; 4472175365Sgallatin status = mxge_load_firmware(sc, 0); 4473175365Sgallatin if (status != 0) { 4474175365Sgallatin device_printf(sc->dev, "Falling back to a single slice\n"); 4475175365Sgallatin return; 4476175365Sgallatin } 4477175365Sgallatin 4478175365Sgallatin /* try to send a reset command to the card to see if it 4479175365Sgallatin is alive */ 4480175365Sgallatin memset(&cmd, 0, sizeof (cmd)); 4481175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); 4482175365Sgallatin if (status != 0) { 4483175365Sgallatin device_printf(sc->dev, "failed reset\n"); 4484175365Sgallatin goto abort_with_fw; 4485175365Sgallatin } 4486175365Sgallatin 4487175365Sgallatin /* get rx ring size */ 4488175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); 4489175365Sgallatin if (status != 0) { 4490175365Sgallatin device_printf(sc->dev, "Cannot determine rx ring size\n"); 4491175365Sgallatin goto abort_with_fw; 4492175365Sgallatin } 4493175365Sgallatin max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t)); 4494175365Sgallatin 4495175365Sgallatin /* tell it the size of the interrupt queues */ 4496175365Sgallatin cmd.data0 = max_intr_slots * sizeof (struct mcp_slot); 4497175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); 4498175365Sgallatin if (status != 0) { 4499175365Sgallatin device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 4500175365Sgallatin goto abort_with_fw; 4501175365Sgallatin } 4502175365Sgallatin 4503175365Sgallatin /* ask the maximum number of slices it supports */ 4504175365Sgallatin status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd); 4505175365Sgallatin if (status != 0) { 4506175365Sgallatin device_printf(sc->dev, 4507175365Sgallatin "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n"); 4508175365Sgallatin goto abort_with_fw; 4509175365Sgallatin } 4510175365Sgallatin sc->num_slices = cmd.data0; 4511175365Sgallatin if (sc->num_slices > msix_cnt) 4512175365Sgallatin sc->num_slices = msix_cnt; 4513175365Sgallatin 4514175365Sgallatin if (mxge_max_slices == -1) { 4515175365Sgallatin /* cap to number of CPUs in system */ 4516175365Sgallatin if (sc->num_slices > mp_ncpus) 4517175365Sgallatin sc->num_slices = mp_ncpus; 4518175365Sgallatin } else { 4519175365Sgallatin if (sc->num_slices > mxge_max_slices) 4520175365Sgallatin sc->num_slices = mxge_max_slices; 4521175365Sgallatin } 4522175365Sgallatin /* make sure it is a power of two */ 4523175365Sgallatin while (sc->num_slices & (sc->num_slices - 1)) 4524175365Sgallatin sc->num_slices--; 4525175365Sgallatin 4526175365Sgallatin if (mxge_verbose) 4527175365Sgallatin device_printf(sc->dev, "using %d slices\n", 4528175365Sgallatin sc->num_slices); 4529175365Sgallatin 4530175365Sgallatin return; 4531175365Sgallatin 4532175365Sgallatinabort_with_fw: 4533175365Sgallatin sc->fw_name = old_fw; 4534175365Sgallatin (void) mxge_load_firmware(sc, 0); 4535175365Sgallatin} 4536175365Sgallatin 4537175365Sgallatinstatic int 4538175365Sgallatinmxge_add_msix_irqs(mxge_softc_t *sc) 4539175365Sgallatin{ 4540175365Sgallatin size_t bytes; 4541175365Sgallatin int count, err, i, rid; 4542175365Sgallatin 4543175365Sgallatin rid = PCIR_BAR(2); 4544175365Sgallatin sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 4545175365Sgallatin &rid, RF_ACTIVE); 4546175365Sgallatin 4547175365Sgallatin if (sc->msix_table_res == NULL) { 4548175365Sgallatin device_printf(sc->dev, "couldn't alloc MSIX table res\n"); 4549175365Sgallatin return ENXIO; 4550175365Sgallatin } 4551175365Sgallatin 4552175365Sgallatin count = sc->num_slices; 4553175365Sgallatin err = pci_alloc_msix(sc->dev, &count); 4554175365Sgallatin if (err != 0) { 4555175365Sgallatin device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d" 4556175365Sgallatin "err = %d \n", sc->num_slices, err); 4557175365Sgallatin goto abort_with_msix_table; 4558175365Sgallatin } 4559175365Sgallatin if (count < sc->num_slices) { 4560175365Sgallatin device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n", 4561175365Sgallatin count, sc->num_slices); 4562175365Sgallatin device_printf(sc->dev, 4563175365Sgallatin "Try setting hw.mxge.max_slices to %d\n", 4564175365Sgallatin count); 4565175365Sgallatin err = ENOSPC; 4566175365Sgallatin goto abort_with_msix; 4567175365Sgallatin } 4568175365Sgallatin bytes = sizeof (*sc->msix_irq_res) * sc->num_slices; 4569175365Sgallatin sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 4570175365Sgallatin if (sc->msix_irq_res == NULL) { 4571175365Sgallatin err = ENOMEM; 4572175365Sgallatin goto abort_with_msix; 4573175365Sgallatin } 4574175365Sgallatin 4575175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4576175365Sgallatin rid = i + 1; 4577175365Sgallatin sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev, 4578175365Sgallatin SYS_RES_IRQ, 4579175365Sgallatin &rid, RF_ACTIVE); 4580175365Sgallatin if (sc->msix_irq_res[i] == NULL) { 4581175365Sgallatin device_printf(sc->dev, "couldn't allocate IRQ res" 4582175365Sgallatin " for message %d\n", i); 4583175365Sgallatin err = ENXIO; 4584175365Sgallatin goto abort_with_res; 4585175365Sgallatin } 4586175365Sgallatin } 4587175365Sgallatin 4588175365Sgallatin bytes = sizeof (*sc->msix_ih) * sc->num_slices; 4589175365Sgallatin sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); 4590175365Sgallatin 4591175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4592175365Sgallatin err = bus_setup_intr(sc->dev, sc->msix_irq_res[i], 4593175365Sgallatin INTR_TYPE_NET | INTR_MPSAFE, 4594176261Sgallatin#if __FreeBSD_version > 700030 4595176261Sgallatin NULL, 4596176261Sgallatin#endif 4597176261Sgallatin mxge_intr, &sc->ss[i], &sc->msix_ih[i]); 4598175365Sgallatin if (err != 0) { 4599175365Sgallatin device_printf(sc->dev, "couldn't setup intr for " 4600175365Sgallatin "message %d\n", i); 4601175365Sgallatin goto abort_with_intr; 4602175365Sgallatin } 4603208379Sgallatin bus_describe_intr(sc->dev, sc->msix_irq_res[i], 4604208379Sgallatin sc->msix_ih[i], "s%d", i); 4605175365Sgallatin } 4606175365Sgallatin 4607175365Sgallatin if (mxge_verbose) { 4608175365Sgallatin device_printf(sc->dev, "using %d msix IRQs:", 4609175365Sgallatin sc->num_slices); 4610175365Sgallatin for (i = 0; i < sc->num_slices; i++) 4611175365Sgallatin printf(" %ld", rman_get_start(sc->msix_irq_res[i])); 4612175365Sgallatin printf("\n"); 4613175365Sgallatin } 4614175365Sgallatin return (0); 4615175365Sgallatin 4616175365Sgallatinabort_with_intr: 4617175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4618175365Sgallatin if (sc->msix_ih[i] != NULL) { 4619175365Sgallatin bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 4620175365Sgallatin sc->msix_ih[i]); 4621175365Sgallatin sc->msix_ih[i] = NULL; 4622175365Sgallatin } 4623175365Sgallatin } 4624175365Sgallatin free(sc->msix_ih, M_DEVBUF); 4625175365Sgallatin 4626175365Sgallatin 4627175365Sgallatinabort_with_res: 4628175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4629175365Sgallatin rid = i + 1; 4630175365Sgallatin if (sc->msix_irq_res[i] != NULL) 4631175365Sgallatin bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 4632175365Sgallatin sc->msix_irq_res[i]); 4633175365Sgallatin sc->msix_irq_res[i] = NULL; 4634175365Sgallatin } 4635175365Sgallatin free(sc->msix_irq_res, M_DEVBUF); 4636175365Sgallatin 4637175365Sgallatin 4638175365Sgallatinabort_with_msix: 4639175365Sgallatin pci_release_msi(sc->dev); 4640175365Sgallatin 4641175365Sgallatinabort_with_msix_table: 4642175365Sgallatin bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 4643175365Sgallatin sc->msix_table_res); 4644175365Sgallatin 4645175365Sgallatin return err; 4646175365Sgallatin} 4647175365Sgallatin 4648175365Sgallatinstatic int 4649175365Sgallatinmxge_add_single_irq(mxge_softc_t *sc) 4650175365Sgallatin{ 4651175365Sgallatin int count, err, rid; 4652175365Sgallatin 4653175365Sgallatin count = pci_msi_count(sc->dev); 4654175365Sgallatin if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) { 4655175365Sgallatin rid = 1; 4656175365Sgallatin } else { 4657175365Sgallatin rid = 0; 4658176281Sgallatin sc->legacy_irq = 1; 4659175365Sgallatin } 4660175365Sgallatin sc->irq_res = bus_alloc_resource(sc->dev, SYS_RES_IRQ, &rid, 0, ~0, 4661175365Sgallatin 1, RF_SHAREABLE | RF_ACTIVE); 4662175365Sgallatin if (sc->irq_res == NULL) { 4663175365Sgallatin device_printf(sc->dev, "could not alloc interrupt\n"); 4664175365Sgallatin return ENXIO; 4665175365Sgallatin } 4666175365Sgallatin if (mxge_verbose) 4667175365Sgallatin device_printf(sc->dev, "using %s irq %ld\n", 4668176281Sgallatin sc->legacy_irq ? "INTx" : "MSI", 4669175365Sgallatin rman_get_start(sc->irq_res)); 4670175365Sgallatin err = bus_setup_intr(sc->dev, sc->irq_res, 4671175365Sgallatin INTR_TYPE_NET | INTR_MPSAFE, 4672176261Sgallatin#if __FreeBSD_version > 700030 4673176261Sgallatin NULL, 4674176261Sgallatin#endif 4675176261Sgallatin mxge_intr, &sc->ss[0], &sc->ih); 4676175365Sgallatin if (err != 0) { 4677175365Sgallatin bus_release_resource(sc->dev, SYS_RES_IRQ, 4678176281Sgallatin sc->legacy_irq ? 0 : 1, sc->irq_res); 4679176281Sgallatin if (!sc->legacy_irq) 4680175365Sgallatin pci_release_msi(sc->dev); 4681175365Sgallatin } 4682175365Sgallatin return err; 4683175365Sgallatin} 4684175365Sgallatin 4685175365Sgallatinstatic void 4686175365Sgallatinmxge_rem_msix_irqs(mxge_softc_t *sc) 4687175365Sgallatin{ 4688175365Sgallatin int i, rid; 4689175365Sgallatin 4690175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4691175365Sgallatin if (sc->msix_ih[i] != NULL) { 4692175365Sgallatin bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 4693175365Sgallatin sc->msix_ih[i]); 4694175365Sgallatin sc->msix_ih[i] = NULL; 4695175365Sgallatin } 4696175365Sgallatin } 4697175365Sgallatin free(sc->msix_ih, M_DEVBUF); 4698175365Sgallatin 4699175365Sgallatin for (i = 0; i < sc->num_slices; i++) { 4700175365Sgallatin rid = i + 1; 4701175365Sgallatin if (sc->msix_irq_res[i] != NULL) 4702175365Sgallatin bus_release_resource(sc->dev, SYS_RES_IRQ, rid, 4703175365Sgallatin sc->msix_irq_res[i]); 4704175365Sgallatin sc->msix_irq_res[i] = NULL; 4705175365Sgallatin } 4706175365Sgallatin free(sc->msix_irq_res, M_DEVBUF); 4707175365Sgallatin 4708175365Sgallatin bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), 4709175365Sgallatin sc->msix_table_res); 4710175365Sgallatin 4711175365Sgallatin pci_release_msi(sc->dev); 4712175365Sgallatin return; 4713175365Sgallatin} 4714175365Sgallatin 4715175365Sgallatinstatic void 4716175365Sgallatinmxge_rem_single_irq(mxge_softc_t *sc) 4717175365Sgallatin{ 4718175365Sgallatin bus_teardown_intr(sc->dev, sc->irq_res, sc->ih); 4719175365Sgallatin bus_release_resource(sc->dev, SYS_RES_IRQ, 4720176281Sgallatin sc->legacy_irq ? 0 : 1, sc->irq_res); 4721176281Sgallatin if (!sc->legacy_irq) 4722175365Sgallatin pci_release_msi(sc->dev); 4723175365Sgallatin} 4724175365Sgallatin 4725175365Sgallatinstatic void 4726175365Sgallatinmxge_rem_irq(mxge_softc_t *sc) 4727175365Sgallatin{ 4728175365Sgallatin if (sc->num_slices > 1) 4729175365Sgallatin mxge_rem_msix_irqs(sc); 4730175365Sgallatin else 4731175365Sgallatin mxge_rem_single_irq(sc); 4732175365Sgallatin} 4733175365Sgallatin 4734175365Sgallatinstatic int 4735175365Sgallatinmxge_add_irq(mxge_softc_t *sc) 4736175365Sgallatin{ 4737175365Sgallatin int err; 4738175365Sgallatin 4739175365Sgallatin if (sc->num_slices > 1) 4740175365Sgallatin err = mxge_add_msix_irqs(sc); 4741175365Sgallatin else 4742175365Sgallatin err = mxge_add_single_irq(sc); 4743175365Sgallatin 4744175365Sgallatin if (0 && err == 0 && sc->num_slices > 1) { 4745175365Sgallatin mxge_rem_msix_irqs(sc); 4746175365Sgallatin err = mxge_add_msix_irqs(sc); 4747175365Sgallatin } 4748175365Sgallatin return err; 4749175365Sgallatin} 4750175365Sgallatin 4751175365Sgallatin 4752155852Sgallatinstatic int 4753159571Sgallatinmxge_attach(device_t dev) 4754155852Sgallatin{ 4755247011Sgallatin mxge_cmd_t cmd; 4756159571Sgallatin mxge_softc_t *sc = device_get_softc(dev); 4757155852Sgallatin struct ifnet *ifp; 4758175365Sgallatin int err, rid; 4759155852Sgallatin 4760155852Sgallatin sc->dev = dev; 4761159571Sgallatin mxge_fetch_tunables(sc); 4762155852Sgallatin 4763198250Sgallatin TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc); 4764217104Sjhb sc->tq = taskqueue_create("mxge_taskq", M_WAITOK, 4765217104Sjhb taskqueue_thread_enqueue, &sc->tq); 4766198250Sgallatin if (sc->tq == NULL) { 4767198250Sgallatin err = ENOMEM; 4768198250Sgallatin goto abort_with_nothing; 4769198250Sgallatin } 4770198250Sgallatin 4771232874Sscottl err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4772155852Sgallatin 1, /* alignment */ 4773175365Sgallatin 0, /* boundary */ 4774155852Sgallatin BUS_SPACE_MAXADDR, /* low */ 4775155852Sgallatin BUS_SPACE_MAXADDR, /* high */ 4776155852Sgallatin NULL, NULL, /* filter */ 4777162322Sgallatin 65536 + 256, /* maxsize */ 4778159612Sgallatin MXGE_MAX_SEND_DESC, /* num segs */ 4779175365Sgallatin 65536, /* maxsegsize */ 4780155852Sgallatin 0, /* flags */ 4781155852Sgallatin NULL, NULL, /* lock */ 4782155852Sgallatin &sc->parent_dmat); /* tag */ 4783155852Sgallatin 4784155852Sgallatin if (err != 0) { 4785155852Sgallatin device_printf(sc->dev, "Err %d allocating parent dmat\n", 4786155852Sgallatin err); 4787198250Sgallatin goto abort_with_tq; 4788155852Sgallatin } 4789155852Sgallatin 4790155852Sgallatin ifp = sc->ifp = if_alloc(IFT_ETHER); 4791155852Sgallatin if (ifp == NULL) { 4792155852Sgallatin device_printf(dev, "can not if_alloc()\n"); 4793155852Sgallatin err = ENOSPC; 4794155852Sgallatin goto abort_with_parent_dmat; 4795155852Sgallatin } 4796175365Sgallatin if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 4797175365Sgallatin 4798166370Sgallatin snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd", 4799166370Sgallatin device_get_nameunit(dev)); 4800166370Sgallatin mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF); 4801166370Sgallatin snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name), 4802166370Sgallatin "%s:drv", device_get_nameunit(dev)); 4803166370Sgallatin mtx_init(&sc->driver_mtx, sc->driver_mtx_name, 4804155852Sgallatin MTX_NETWORK_LOCK, MTX_DEF); 4805155852Sgallatin 4806166373Sgallatin callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0); 4807164513Sgallatin 4808166373Sgallatin mxge_setup_cfg_space(sc); 4809166373Sgallatin 4810155852Sgallatin /* Map the board into the kernel */ 4811155852Sgallatin rid = PCIR_BARS; 4812155852Sgallatin sc->mem_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 0, 4813155852Sgallatin ~0, 1, RF_ACTIVE); 4814155852Sgallatin if (sc->mem_res == NULL) { 4815155852Sgallatin device_printf(dev, "could not map memory\n"); 4816155852Sgallatin err = ENXIO; 4817155852Sgallatin goto abort_with_lock; 4818155852Sgallatin } 4819155852Sgallatin sc->sram = rman_get_virtual(sc->mem_res); 4820155852Sgallatin sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100; 4821155852Sgallatin if (sc->sram_size > rman_get_size(sc->mem_res)) { 4822155852Sgallatin device_printf(dev, "impossible memory region size %ld\n", 4823155852Sgallatin rman_get_size(sc->mem_res)); 4824155852Sgallatin err = ENXIO; 4825155852Sgallatin goto abort_with_mem_res; 4826155852Sgallatin } 4827155852Sgallatin 4828155852Sgallatin /* make NULL terminated copy of the EEPROM strings section of 4829155852Sgallatin lanai SRAM */ 4830159571Sgallatin bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE); 4831155852Sgallatin bus_space_read_region_1(rman_get_bustag(sc->mem_res), 4832155852Sgallatin rman_get_bushandle(sc->mem_res), 4833159571Sgallatin sc->sram_size - MXGE_EEPROM_STRINGS_SIZE, 4834155852Sgallatin sc->eeprom_strings, 4835159571Sgallatin MXGE_EEPROM_STRINGS_SIZE - 2); 4836159571Sgallatin err = mxge_parse_strings(sc); 4837155852Sgallatin if (err != 0) 4838155852Sgallatin goto abort_with_mem_res; 4839155852Sgallatin 4840155852Sgallatin /* Enable write combining for efficient use of PCIe bus */ 4841159571Sgallatin mxge_enable_wc(sc); 4842155852Sgallatin 4843155852Sgallatin /* Allocate the out of band dma memory */ 4844159571Sgallatin err = mxge_dma_alloc(sc, &sc->cmd_dma, 4845159571Sgallatin sizeof (mxge_cmd_t), 64); 4846155852Sgallatin if (err != 0) 4847155852Sgallatin goto abort_with_mem_res; 4848155852Sgallatin sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr; 4849159571Sgallatin err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64); 4850155852Sgallatin if (err != 0) 4851155852Sgallatin goto abort_with_cmd_dma; 4852155852Sgallatin 4853166370Sgallatin err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096); 4854166370Sgallatin if (err != 0) 4855175365Sgallatin goto abort_with_zeropad_dma; 4856155852Sgallatin 4857169376Sgallatin /* select & load the firmware */ 4858169376Sgallatin err = mxge_select_firmware(sc); 4859155852Sgallatin if (err != 0) 4860175365Sgallatin goto abort_with_dmabench; 4861159612Sgallatin sc->intr_coal_delay = mxge_intr_coal_delay; 4862175365Sgallatin 4863175365Sgallatin mxge_slice_probe(sc); 4864175365Sgallatin err = mxge_alloc_slices(sc); 4865175365Sgallatin if (err != 0) 4866175365Sgallatin goto abort_with_dmabench; 4867175365Sgallatin 4868169871Sgallatin err = mxge_reset(sc, 0); 4869155852Sgallatin if (err != 0) 4870175365Sgallatin goto abort_with_slices; 4871155852Sgallatin 4872166370Sgallatin err = mxge_alloc_rings(sc); 4873166370Sgallatin if (err != 0) { 4874166370Sgallatin device_printf(sc->dev, "failed to allocate rings\n"); 4875205255Sgallatin goto abort_with_slices; 4876166370Sgallatin } 4877166370Sgallatin 4878175365Sgallatin err = mxge_add_irq(sc); 4879166370Sgallatin if (err != 0) { 4880175365Sgallatin device_printf(sc->dev, "failed to add irq\n"); 4881166370Sgallatin goto abort_with_rings; 4882166370Sgallatin } 4883175365Sgallatin 4884241687Sglebius if_initbaudrate(ifp, IF_Gbps(10)); 4885169905Sgallatin ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 | 4886247133Sgallatin IFCAP_VLAN_MTU | IFCAP_LINKSTATE | IFCAP_TXCSUM_IPV6 | 4887247133Sgallatin IFCAP_RXCSUM_IPV6; 4888247133Sgallatin#if defined(INET) || defined(INET6) 4889194743Sgallatin ifp->if_capabilities |= IFCAP_LRO; 4890194743Sgallatin#endif 4891169905Sgallatin 4892176261Sgallatin#ifdef MXGE_NEW_VLAN_API 4893176261Sgallatin ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 4894204212Sgallatin 4895204212Sgallatin /* Only FW 1.4.32 and newer can do TSO over vlans */ 4896204212Sgallatin if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && 4897204212Sgallatin sc->fw_ver_tiny >= 32) 4898204212Sgallatin ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 4899176261Sgallatin#endif 4900169840Sgallatin sc->max_mtu = mxge_max_mtu(sc); 4901169840Sgallatin if (sc->max_mtu >= 9000) 4902169840Sgallatin ifp->if_capabilities |= IFCAP_JUMBO_MTU; 4903169840Sgallatin else 4904169840Sgallatin device_printf(dev, "MTU limited to %d. Install " 4905169871Sgallatin "latest firmware for 9000 byte jumbo support\n", 4906169840Sgallatin sc->max_mtu - ETHER_HDR_LEN); 4907162322Sgallatin ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO; 4908247011Sgallatin ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6; 4909247011Sgallatin /* check to see if f/w supports TSO for IPv6 */ 4910247011Sgallatin if (!mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd)) { 4911247011Sgallatin if (CSUM_TCP_IPV6) 4912247011Sgallatin ifp->if_capabilities |= IFCAP_TSO6; 4913247011Sgallatin sc->max_tso6_hlen = min(cmd.data0, 4914247011Sgallatin sizeof (sc->ss[0].scratch)); 4915247011Sgallatin } 4916155852Sgallatin ifp->if_capenable = ifp->if_capabilities; 4917170626Sgallatin if (sc->lro_cnt == 0) 4918170626Sgallatin ifp->if_capenable &= ~IFCAP_LRO; 4919159571Sgallatin ifp->if_init = mxge_init; 4920155852Sgallatin ifp->if_softc = sc; 4921155852Sgallatin ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 4922159571Sgallatin ifp->if_ioctl = mxge_ioctl; 4923159571Sgallatin ifp->if_start = mxge_start; 4924171917Sgallatin /* Initialise the ifmedia structure */ 4925171917Sgallatin ifmedia_init(&sc->media, 0, mxge_media_change, 4926171917Sgallatin mxge_media_status); 4927206662Sgallatin mxge_media_init(sc); 4928171917Sgallatin mxge_media_probe(sc); 4929194909Sgallatin sc->dying = 0; 4930155852Sgallatin ether_ifattach(ifp, sc->mac_addr); 4931194836Sgallatin /* ether_ifattach sets mtu to ETHERMTU */ 4932194836Sgallatin if (mxge_initial_mtu != ETHERMTU) 4933194836Sgallatin mxge_change_mtu(sc, mxge_initial_mtu); 4934155852Sgallatin 4935159571Sgallatin mxge_add_sysctls(sc); 4936191562Sgallatin#ifdef IFNET_BUF_RING 4937191562Sgallatin ifp->if_transmit = mxge_transmit; 4938191562Sgallatin ifp->if_qflush = mxge_qflush; 4939191562Sgallatin#endif 4940205255Sgallatin taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 4941205255Sgallatin device_get_nameunit(sc->dev)); 4942198303Sgallatin callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); 4943155852Sgallatin return 0; 4944155852Sgallatin 4945166370Sgallatinabort_with_rings: 4946166370Sgallatin mxge_free_rings(sc); 4947175365Sgallatinabort_with_slices: 4948175365Sgallatin mxge_free_slices(sc); 4949166370Sgallatinabort_with_dmabench: 4950166370Sgallatin mxge_dma_free(&sc->dmabench_dma); 4951155852Sgallatinabort_with_zeropad_dma: 4952159571Sgallatin mxge_dma_free(&sc->zeropad_dma); 4953155852Sgallatinabort_with_cmd_dma: 4954159571Sgallatin mxge_dma_free(&sc->cmd_dma); 4955155852Sgallatinabort_with_mem_res: 4956155852Sgallatin bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 4957155852Sgallatinabort_with_lock: 4958155852Sgallatin pci_disable_busmaster(dev); 4959166370Sgallatin mtx_destroy(&sc->cmd_mtx); 4960166370Sgallatin mtx_destroy(&sc->driver_mtx); 4961155852Sgallatin if_free(ifp); 4962155852Sgallatinabort_with_parent_dmat: 4963155852Sgallatin bus_dma_tag_destroy(sc->parent_dmat); 4964198250Sgallatinabort_with_tq: 4965198250Sgallatin if (sc->tq != NULL) { 4966198250Sgallatin taskqueue_drain(sc->tq, &sc->watchdog_task); 4967198250Sgallatin taskqueue_free(sc->tq); 4968198250Sgallatin sc->tq = NULL; 4969198250Sgallatin } 4970155852Sgallatinabort_with_nothing: 4971155852Sgallatin return err; 4972155852Sgallatin} 4973155852Sgallatin 4974155852Sgallatinstatic int 4975159571Sgallatinmxge_detach(device_t dev) 4976155852Sgallatin{ 4977159571Sgallatin mxge_softc_t *sc = device_get_softc(dev); 4978155852Sgallatin 4979176261Sgallatin if (mxge_vlans_active(sc)) { 4980169905Sgallatin device_printf(sc->dev, 4981169905Sgallatin "Detach vlans before removing module\n"); 4982169905Sgallatin return EBUSY; 4983169905Sgallatin } 4984166370Sgallatin mtx_lock(&sc->driver_mtx); 4985194909Sgallatin sc->dying = 1; 4986155852Sgallatin if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) 4987197395Sgallatin mxge_close(sc, 0); 4988166370Sgallatin mtx_unlock(&sc->driver_mtx); 4989155852Sgallatin ether_ifdetach(sc->ifp); 4990198250Sgallatin if (sc->tq != NULL) { 4991198250Sgallatin taskqueue_drain(sc->tq, &sc->watchdog_task); 4992198250Sgallatin taskqueue_free(sc->tq); 4993198250Sgallatin sc->tq = NULL; 4994198250Sgallatin } 4995180567Sgallatin callout_drain(&sc->co_hdl); 4996166373Sgallatin ifmedia_removeall(&sc->media); 4997160876Sgallatin mxge_dummy_rdma(sc, 0); 4998175365Sgallatin mxge_rem_sysctls(sc); 4999175365Sgallatin mxge_rem_irq(sc); 5000166370Sgallatin mxge_free_rings(sc); 5001175365Sgallatin mxge_free_slices(sc); 5002166370Sgallatin mxge_dma_free(&sc->dmabench_dma); 5003159571Sgallatin mxge_dma_free(&sc->zeropad_dma); 5004159571Sgallatin mxge_dma_free(&sc->cmd_dma); 5005155852Sgallatin bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); 5006155852Sgallatin pci_disable_busmaster(dev); 5007166370Sgallatin mtx_destroy(&sc->cmd_mtx); 5008166370Sgallatin mtx_destroy(&sc->driver_mtx); 5009155852Sgallatin if_free(sc->ifp); 5010155852Sgallatin bus_dma_tag_destroy(sc->parent_dmat); 5011155852Sgallatin return 0; 5012155852Sgallatin} 5013155852Sgallatin 5014155852Sgallatinstatic int 5015159571Sgallatinmxge_shutdown(device_t dev) 5016155852Sgallatin{ 5017155852Sgallatin return 0; 5018155852Sgallatin} 5019155852Sgallatin 5020155852Sgallatin/* 5021155852Sgallatin This file uses Myri10GE driver indentation. 5022155852Sgallatin 5023155852Sgallatin Local Variables: 5024155852Sgallatin c-file-style:"linux" 5025155852Sgallatin tab-width:8 5026155852Sgallatin End: 5027155852Sgallatin*/ 5028