if_vmx.c revision 320099
1254738Sbryanv/*- 2254738Sbryanv * Copyright (c) 2013 Tsubai Masanari 3254738Sbryanv * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4254738Sbryanv * 5254738Sbryanv * Permission to use, copy, modify, and distribute this software for any 6254738Sbryanv * purpose with or without fee is hereby granted, provided that the above 7254738Sbryanv * copyright notice and this permission notice appear in all copies. 8254738Sbryanv * 9254738Sbryanv * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10254738Sbryanv * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11254738Sbryanv * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12254738Sbryanv * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13254738Sbryanv * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14254738Sbryanv * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15254738Sbryanv * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16254738Sbryanv * 17254738Sbryanv * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18254738Sbryanv */ 19254738Sbryanv 20254738Sbryanv/* Driver for VMware vmxnet3 virtual ethernet devices. */ 21254738Sbryanv 22254738Sbryanv#include <sys/cdefs.h> 23254738Sbryanv__FBSDID("$FreeBSD: stable/11/sys/dev/vmware/vmxnet3/if_vmx.c 320099 2017-06-19 14:45:20Z avg $"); 24254738Sbryanv 25254738Sbryanv#include <sys/param.h> 26254738Sbryanv#include <sys/systm.h> 27257241Sglebius#include <sys/eventhandler.h> 28254738Sbryanv#include <sys/kernel.h> 29254738Sbryanv#include <sys/endian.h> 30254738Sbryanv#include <sys/sockio.h> 31254738Sbryanv#include <sys/mbuf.h> 32254738Sbryanv#include <sys/malloc.h> 33254738Sbryanv#include <sys/module.h> 34254738Sbryanv#include <sys/socket.h> 35254738Sbryanv#include <sys/sysctl.h> 36263259Sbryanv#include <sys/smp.h> 37263259Sbryanv#include <sys/taskqueue.h> 38254738Sbryanv#include <vm/vm.h> 39254738Sbryanv#include <vm/pmap.h> 40254738Sbryanv 41254738Sbryanv#include <net/ethernet.h> 42254738Sbryanv#include <net/if.h> 43257176Sglebius#include <net/if_var.h> 44254738Sbryanv#include <net/if_arp.h> 45254738Sbryanv#include <net/if_dl.h> 46254738Sbryanv#include <net/if_types.h> 47254738Sbryanv#include <net/if_media.h> 48254738Sbryanv#include <net/if_vlan_var.h> 49254738Sbryanv 50254738Sbryanv#include <net/bpf.h> 51254738Sbryanv 52254738Sbryanv#include <netinet/in_systm.h> 53254738Sbryanv#include <netinet/in.h> 54254738Sbryanv#include <netinet/ip.h> 55254738Sbryanv#include <netinet/ip6.h> 56254738Sbryanv#include <netinet6/ip6_var.h> 57254738Sbryanv#include <netinet/udp.h> 58254738Sbryanv#include <netinet/tcp.h> 59254738Sbryanv 60267253Sbryanv#include <machine/in_cksum.h> 61267253Sbryanv 62254738Sbryanv#include <machine/bus.h> 63254738Sbryanv#include <machine/resource.h> 64254738Sbryanv#include <sys/bus.h> 65254738Sbryanv#include <sys/rman.h> 66254738Sbryanv 67254738Sbryanv#include <dev/pci/pcireg.h> 68254738Sbryanv#include <dev/pci/pcivar.h> 69254738Sbryanv 70254738Sbryanv#include "if_vmxreg.h" 71254738Sbryanv#include "if_vmxvar.h" 72254738Sbryanv 73254738Sbryanv#include "opt_inet.h" 74254738Sbryanv#include "opt_inet6.h" 75254738Sbryanv 76254738Sbryanv#ifdef VMXNET3_FAILPOINTS 77254738Sbryanv#include <sys/fail.h> 78254738Sbryanvstatic SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 79254738Sbryanv "vmxnet3 fail points"); 80254738Sbryanv#define VMXNET3_FP _debug_fail_point_vmxnet3 81254738Sbryanv#endif 82254738Sbryanv 83254738Sbryanvstatic int vmxnet3_probe(device_t); 84254738Sbryanvstatic int vmxnet3_attach(device_t); 85254738Sbryanvstatic int vmxnet3_detach(device_t); 86254738Sbryanvstatic int vmxnet3_shutdown(device_t); 87254738Sbryanv 88254738Sbryanvstatic int vmxnet3_alloc_resources(struct vmxnet3_softc *); 89254738Sbryanvstatic void vmxnet3_free_resources(struct vmxnet3_softc *); 90254738Sbryanvstatic int vmxnet3_check_version(struct vmxnet3_softc *); 91254738Sbryanvstatic void vmxnet3_initial_config(struct vmxnet3_softc *); 92263259Sbryanvstatic void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 93254738Sbryanv 94254738Sbryanvstatic int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95254738Sbryanvstatic int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96254738Sbryanvstatic int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 97254738Sbryanvstatic int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 98254738Sbryanv struct vmxnet3_interrupt *); 99254738Sbryanvstatic int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 100254738Sbryanvstatic int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 101254738Sbryanvstatic int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 102254738Sbryanvstatic int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 103254738Sbryanvstatic int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 104254738Sbryanv 105254738Sbryanvstatic void vmxnet3_free_interrupt(struct vmxnet3_softc *, 106254738Sbryanv struct vmxnet3_interrupt *); 107254738Sbryanvstatic void vmxnet3_free_interrupts(struct vmxnet3_softc *); 108254738Sbryanv 109263259Sbryanv#ifndef VMXNET3_LEGACY_TX 110263259Sbryanvstatic int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 111263259Sbryanvstatic void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 112263259Sbryanvstatic void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 113263259Sbryanvstatic void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 114263259Sbryanv#endif 115263259Sbryanv 116254738Sbryanvstatic int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 117254738Sbryanvstatic int vmxnet3_init_txq(struct vmxnet3_softc *, int); 118254738Sbryanvstatic int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 119254738Sbryanvstatic void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 120254738Sbryanvstatic void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 121254738Sbryanvstatic void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 122254738Sbryanv 123254738Sbryanvstatic int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 124254738Sbryanvstatic void vmxnet3_free_shared_data(struct vmxnet3_softc *); 125254738Sbryanvstatic int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 126254738Sbryanvstatic void vmxnet3_free_txq_data(struct vmxnet3_softc *); 127254738Sbryanvstatic int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 128254738Sbryanvstatic void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 129254738Sbryanvstatic int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 130254738Sbryanvstatic void vmxnet3_free_queue_data(struct vmxnet3_softc *); 131254738Sbryanvstatic int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 132254738Sbryanvstatic void vmxnet3_init_shared_data(struct vmxnet3_softc *); 133303136Smavstatic void vmxnet3_init_hwassist(struct vmxnet3_softc *); 134254738Sbryanvstatic void vmxnet3_reinit_interface(struct vmxnet3_softc *); 135263259Sbryanvstatic void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 136254738Sbryanvstatic void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 137254738Sbryanvstatic int vmxnet3_alloc_data(struct vmxnet3_softc *); 138254738Sbryanvstatic void vmxnet3_free_data(struct vmxnet3_softc *); 139254738Sbryanvstatic int vmxnet3_setup_interface(struct vmxnet3_softc *); 140254738Sbryanv 141254738Sbryanvstatic void vmxnet3_evintr(struct vmxnet3_softc *); 142254738Sbryanvstatic void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 143254738Sbryanvstatic void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 144254738Sbryanvstatic int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 145254738Sbryanvstatic void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 146254738Sbryanv struct vmxnet3_rxring *, int); 147254738Sbryanvstatic void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 148254738Sbryanvstatic void vmxnet3_legacy_intr(void *); 149254738Sbryanvstatic void vmxnet3_txq_intr(void *); 150254738Sbryanvstatic void vmxnet3_rxq_intr(void *); 151254738Sbryanvstatic void vmxnet3_event_intr(void *); 152254738Sbryanv 153254738Sbryanvstatic void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 154254738Sbryanvstatic void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 155254738Sbryanvstatic void vmxnet3_stop(struct vmxnet3_softc *); 156254738Sbryanv 157254738Sbryanvstatic void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 158254738Sbryanvstatic int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 159254738Sbryanvstatic int vmxnet3_reinit_queues(struct vmxnet3_softc *); 160254738Sbryanvstatic int vmxnet3_enable_device(struct vmxnet3_softc *); 161254738Sbryanvstatic void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 162254738Sbryanvstatic int vmxnet3_reinit(struct vmxnet3_softc *); 163254738Sbryanvstatic void vmxnet3_init_locked(struct vmxnet3_softc *); 164254738Sbryanvstatic void vmxnet3_init(void *); 165254738Sbryanv 166263259Sbryanvstatic int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 167263259Sbryanv int *, int *, int *); 168254738Sbryanvstatic int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 169254738Sbryanv bus_dmamap_t, bus_dma_segment_t [], int *); 170254738Sbryanvstatic void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 171254738Sbryanvstatic int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 172263259Sbryanv#ifdef VMXNET3_LEGACY_TX 173254738Sbryanvstatic void vmxnet3_start_locked(struct ifnet *); 174254738Sbryanvstatic void vmxnet3_start(struct ifnet *); 175263259Sbryanv#else 176263259Sbryanvstatic int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 177263259Sbryanv struct mbuf *); 178263259Sbryanvstatic int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 179263259Sbryanvstatic void vmxnet3_txq_tq_deferred(void *, int); 180263259Sbryanv#endif 181263259Sbryanvstatic void vmxnet3_txq_start(struct vmxnet3_txqueue *); 182263259Sbryanvstatic void vmxnet3_tx_start_all(struct vmxnet3_softc *); 183254738Sbryanv 184254738Sbryanvstatic void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 185254738Sbryanv uint16_t); 186254738Sbryanvstatic void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 187254738Sbryanvstatic void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 188254738Sbryanvstatic void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 189254738Sbryanvstatic int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 190254738Sbryanvstatic int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 191272099Sglebiusstatic uint64_t vmxnet3_get_counter(struct ifnet *, ift_counter); 192254738Sbryanv 193263259Sbryanv#ifndef VMXNET3_LEGACY_TX 194263259Sbryanvstatic void vmxnet3_qflush(struct ifnet *); 195263259Sbryanv#endif 196263259Sbryanv 197254738Sbryanvstatic int vmxnet3_watchdog(struct vmxnet3_txqueue *); 198263259Sbryanvstatic void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 199254738Sbryanvstatic void vmxnet3_tick(void *); 200254738Sbryanvstatic void vmxnet3_link_status(struct vmxnet3_softc *); 201254738Sbryanvstatic void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 202254738Sbryanvstatic int vmxnet3_media_change(struct ifnet *); 203254738Sbryanvstatic void vmxnet3_set_lladdr(struct vmxnet3_softc *); 204254738Sbryanvstatic void vmxnet3_get_lladdr(struct vmxnet3_softc *); 205254738Sbryanv 206254738Sbryanvstatic void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 207254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 208254738Sbryanvstatic void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 209254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 210254738Sbryanvstatic void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 211254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 212254738Sbryanvstatic void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 213254738Sbryanv 214254738Sbryanvstatic void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 215254738Sbryanv uint32_t); 216254738Sbryanvstatic uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 217254738Sbryanvstatic void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 218254738Sbryanv uint32_t); 219254738Sbryanvstatic void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 220254738Sbryanvstatic uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 221254738Sbryanv 222254738Sbryanvstatic void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 223254738Sbryanvstatic void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 224254738Sbryanvstatic void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 225254738Sbryanvstatic void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 226254738Sbryanv 227254738Sbryanvstatic int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 228254738Sbryanv bus_size_t, struct vmxnet3_dma_alloc *); 229254738Sbryanvstatic void vmxnet3_dma_free(struct vmxnet3_softc *, 230254738Sbryanv struct vmxnet3_dma_alloc *); 231255055Sbryanvstatic int vmxnet3_tunable_int(struct vmxnet3_softc *, 232255055Sbryanv const char *, int); 233254738Sbryanv 234254738Sbryanvtypedef enum { 235254738Sbryanv VMXNET3_BARRIER_RD, 236254738Sbryanv VMXNET3_BARRIER_WR, 237254738Sbryanv VMXNET3_BARRIER_RDWR, 238254738Sbryanv} vmxnet3_barrier_t; 239254738Sbryanv 240254738Sbryanvstatic void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 241254738Sbryanv 242255055Sbryanv/* Tunables. */ 243263259Sbryanvstatic int vmxnet3_mq_disable = 0; 244263259SbryanvTUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 245263259Sbryanvstatic int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 246263259SbryanvTUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 247263259Sbryanvstatic int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 248263259SbryanvTUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 249255055Sbryanvstatic int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 250255055SbryanvTUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 251255055Sbryanvstatic int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 252255055SbryanvTUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 253255055Sbryanv 254254738Sbryanvstatic device_method_t vmxnet3_methods[] = { 255254738Sbryanv /* Device interface. */ 256254738Sbryanv DEVMETHOD(device_probe, vmxnet3_probe), 257254738Sbryanv DEVMETHOD(device_attach, vmxnet3_attach), 258254738Sbryanv DEVMETHOD(device_detach, vmxnet3_detach), 259254738Sbryanv DEVMETHOD(device_shutdown, vmxnet3_shutdown), 260254738Sbryanv 261254738Sbryanv DEVMETHOD_END 262254738Sbryanv}; 263254738Sbryanv 264254738Sbryanvstatic driver_t vmxnet3_driver = { 265254738Sbryanv "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 266254738Sbryanv}; 267254738Sbryanv 268254738Sbryanvstatic devclass_t vmxnet3_devclass; 269254738SbryanvDRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 270254738Sbryanv 271254738SbryanvMODULE_DEPEND(vmx, pci, 1, 1, 1); 272254738SbryanvMODULE_DEPEND(vmx, ether, 1, 1, 1); 273254738Sbryanv 274254738Sbryanv#define VMXNET3_VMWARE_VENDOR_ID 0x15AD 275254738Sbryanv#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 276254738Sbryanv 277254738Sbryanvstatic int 278254738Sbryanvvmxnet3_probe(device_t dev) 279254738Sbryanv{ 280254738Sbryanv 281254738Sbryanv if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 282254738Sbryanv pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 283254738Sbryanv device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 284254738Sbryanv return (BUS_PROBE_DEFAULT); 285254738Sbryanv } 286254738Sbryanv 287254738Sbryanv return (ENXIO); 288254738Sbryanv} 289254738Sbryanv 290254738Sbryanvstatic int 291254738Sbryanvvmxnet3_attach(device_t dev) 292254738Sbryanv{ 293254738Sbryanv struct vmxnet3_softc *sc; 294254738Sbryanv int error; 295254738Sbryanv 296254738Sbryanv sc = device_get_softc(dev); 297254738Sbryanv sc->vmx_dev = dev; 298254738Sbryanv 299254738Sbryanv pci_enable_busmaster(dev); 300254738Sbryanv 301254738Sbryanv VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 302254738Sbryanv callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 303254738Sbryanv 304254738Sbryanv vmxnet3_initial_config(sc); 305254738Sbryanv 306254738Sbryanv error = vmxnet3_alloc_resources(sc); 307254738Sbryanv if (error) 308254738Sbryanv goto fail; 309254738Sbryanv 310254738Sbryanv error = vmxnet3_check_version(sc); 311254738Sbryanv if (error) 312254738Sbryanv goto fail; 313254738Sbryanv 314254738Sbryanv error = vmxnet3_alloc_rxtx_queues(sc); 315254738Sbryanv if (error) 316254738Sbryanv goto fail; 317254738Sbryanv 318263259Sbryanv#ifndef VMXNET3_LEGACY_TX 319263259Sbryanv error = vmxnet3_alloc_taskqueue(sc); 320263259Sbryanv if (error) 321263259Sbryanv goto fail; 322263259Sbryanv#endif 323263259Sbryanv 324254738Sbryanv error = vmxnet3_alloc_interrupts(sc); 325254738Sbryanv if (error) 326254738Sbryanv goto fail; 327254738Sbryanv 328263259Sbryanv vmxnet3_check_multiqueue(sc); 329263259Sbryanv 330254738Sbryanv error = vmxnet3_alloc_data(sc); 331254738Sbryanv if (error) 332254738Sbryanv goto fail; 333254738Sbryanv 334254738Sbryanv error = vmxnet3_setup_interface(sc); 335254738Sbryanv if (error) 336254738Sbryanv goto fail; 337254738Sbryanv 338254738Sbryanv error = vmxnet3_setup_interrupts(sc); 339254738Sbryanv if (error) { 340254738Sbryanv ether_ifdetach(sc->vmx_ifp); 341254738Sbryanv device_printf(dev, "could not set up interrupt\n"); 342254738Sbryanv goto fail; 343254738Sbryanv } 344254738Sbryanv 345254738Sbryanv vmxnet3_setup_sysctl(sc); 346263259Sbryanv#ifndef VMXNET3_LEGACY_TX 347263259Sbryanv vmxnet3_start_taskqueue(sc); 348263259Sbryanv#endif 349254738Sbryanv 350254738Sbryanvfail: 351254738Sbryanv if (error) 352254738Sbryanv vmxnet3_detach(dev); 353254738Sbryanv 354254738Sbryanv return (error); 355254738Sbryanv} 356254738Sbryanv 357254738Sbryanvstatic int 358254738Sbryanvvmxnet3_detach(device_t dev) 359254738Sbryanv{ 360254738Sbryanv struct vmxnet3_softc *sc; 361254738Sbryanv struct ifnet *ifp; 362254738Sbryanv 363254738Sbryanv sc = device_get_softc(dev); 364254738Sbryanv ifp = sc->vmx_ifp; 365254738Sbryanv 366254738Sbryanv if (device_is_attached(dev)) { 367254738Sbryanv VMXNET3_CORE_LOCK(sc); 368254738Sbryanv vmxnet3_stop(sc); 369254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 370263259Sbryanv 371254738Sbryanv callout_drain(&sc->vmx_tick); 372263259Sbryanv#ifndef VMXNET3_LEGACY_TX 373263259Sbryanv vmxnet3_drain_taskqueue(sc); 374263259Sbryanv#endif 375263259Sbryanv 376263259Sbryanv ether_ifdetach(ifp); 377254738Sbryanv } 378254738Sbryanv 379254738Sbryanv if (sc->vmx_vlan_attach != NULL) { 380254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 381254738Sbryanv sc->vmx_vlan_attach = NULL; 382254738Sbryanv } 383254738Sbryanv if (sc->vmx_vlan_detach != NULL) { 384254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 385254738Sbryanv sc->vmx_vlan_detach = NULL; 386254738Sbryanv } 387254738Sbryanv 388263259Sbryanv#ifndef VMXNET3_LEGACY_TX 389263259Sbryanv vmxnet3_free_taskqueue(sc); 390263259Sbryanv#endif 391254738Sbryanv vmxnet3_free_interrupts(sc); 392254738Sbryanv 393254738Sbryanv if (ifp != NULL) { 394254738Sbryanv if_free(ifp); 395254738Sbryanv sc->vmx_ifp = NULL; 396254738Sbryanv } 397254738Sbryanv 398254738Sbryanv ifmedia_removeall(&sc->vmx_media); 399254738Sbryanv 400254738Sbryanv vmxnet3_free_data(sc); 401254738Sbryanv vmxnet3_free_resources(sc); 402254738Sbryanv vmxnet3_free_rxtx_queues(sc); 403254738Sbryanv 404254738Sbryanv VMXNET3_CORE_LOCK_DESTROY(sc); 405254738Sbryanv 406254738Sbryanv return (0); 407254738Sbryanv} 408254738Sbryanv 409254738Sbryanvstatic int 410254738Sbryanvvmxnet3_shutdown(device_t dev) 411254738Sbryanv{ 412254738Sbryanv 413254738Sbryanv return (0); 414254738Sbryanv} 415254738Sbryanv 416254738Sbryanvstatic int 417254738Sbryanvvmxnet3_alloc_resources(struct vmxnet3_softc *sc) 418254738Sbryanv{ 419254738Sbryanv device_t dev; 420254738Sbryanv int rid; 421254738Sbryanv 422254738Sbryanv dev = sc->vmx_dev; 423254738Sbryanv 424254738Sbryanv rid = PCIR_BAR(0); 425254738Sbryanv sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 426254738Sbryanv RF_ACTIVE); 427254738Sbryanv if (sc->vmx_res0 == NULL) { 428254738Sbryanv device_printf(dev, 429254738Sbryanv "could not map BAR0 memory\n"); 430254738Sbryanv return (ENXIO); 431254738Sbryanv } 432254738Sbryanv 433254738Sbryanv sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 434254738Sbryanv sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 435254738Sbryanv 436254738Sbryanv rid = PCIR_BAR(1); 437254738Sbryanv sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 438254738Sbryanv RF_ACTIVE); 439254738Sbryanv if (sc->vmx_res1 == NULL) { 440254738Sbryanv device_printf(dev, 441254738Sbryanv "could not map BAR1 memory\n"); 442254738Sbryanv return (ENXIO); 443254738Sbryanv } 444254738Sbryanv 445254738Sbryanv sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 446254738Sbryanv sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 447254738Sbryanv 448254738Sbryanv if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 449254738Sbryanv rid = PCIR_BAR(2); 450254738Sbryanv sc->vmx_msix_res = bus_alloc_resource_any(dev, 451254738Sbryanv SYS_RES_MEMORY, &rid, RF_ACTIVE); 452254738Sbryanv } 453254738Sbryanv 454254738Sbryanv if (sc->vmx_msix_res == NULL) 455254738Sbryanv sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 456254738Sbryanv 457254738Sbryanv return (0); 458254738Sbryanv} 459254738Sbryanv 460254738Sbryanvstatic void 461254738Sbryanvvmxnet3_free_resources(struct vmxnet3_softc *sc) 462254738Sbryanv{ 463254738Sbryanv device_t dev; 464254738Sbryanv int rid; 465254738Sbryanv 466254738Sbryanv dev = sc->vmx_dev; 467254738Sbryanv 468254738Sbryanv if (sc->vmx_res0 != NULL) { 469254738Sbryanv rid = PCIR_BAR(0); 470254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 471254738Sbryanv sc->vmx_res0 = NULL; 472254738Sbryanv } 473254738Sbryanv 474254738Sbryanv if (sc->vmx_res1 != NULL) { 475254738Sbryanv rid = PCIR_BAR(1); 476254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 477254738Sbryanv sc->vmx_res1 = NULL; 478254738Sbryanv } 479254738Sbryanv 480254738Sbryanv if (sc->vmx_msix_res != NULL) { 481254738Sbryanv rid = PCIR_BAR(2); 482254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, 483254738Sbryanv sc->vmx_msix_res); 484254738Sbryanv sc->vmx_msix_res = NULL; 485254738Sbryanv } 486254738Sbryanv} 487254738Sbryanv 488254738Sbryanvstatic int 489254738Sbryanvvmxnet3_check_version(struct vmxnet3_softc *sc) 490254738Sbryanv{ 491254738Sbryanv device_t dev; 492254738Sbryanv uint32_t version; 493254738Sbryanv 494254738Sbryanv dev = sc->vmx_dev; 495254738Sbryanv 496254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 497254738Sbryanv if ((version & 0x01) == 0) { 498254738Sbryanv device_printf(dev, "unsupported hardware version %#x\n", 499254738Sbryanv version); 500254738Sbryanv return (ENOTSUP); 501254950Sbryanv } 502254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 503254738Sbryanv 504254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 505254738Sbryanv if ((version & 0x01) == 0) { 506254738Sbryanv device_printf(dev, "unsupported UPT version %#x\n", version); 507254738Sbryanv return (ENOTSUP); 508254950Sbryanv } 509254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 510254738Sbryanv 511254738Sbryanv return (0); 512254738Sbryanv} 513254738Sbryanv 514290948Sjhbstatic int 515290948Sjhbtrunc_powerof2(int val) 516290948Sjhb{ 517290948Sjhb 518290948Sjhb return (1U << (fls(val) - 1)); 519290948Sjhb} 520290948Sjhb 521254738Sbryanvstatic void 522254738Sbryanvvmxnet3_initial_config(struct vmxnet3_softc *sc) 523254738Sbryanv{ 524263259Sbryanv int nqueue, ndesc; 525254738Sbryanv 526263259Sbryanv nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 527263259Sbryanv if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 528263259Sbryanv nqueue = VMXNET3_DEF_TX_QUEUES; 529263259Sbryanv if (nqueue > mp_ncpus) 530263259Sbryanv nqueue = mp_ncpus; 531290948Sjhb sc->vmx_max_ntxqueues = trunc_powerof2(nqueue); 532255055Sbryanv 533263259Sbryanv nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 534263259Sbryanv if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 535263259Sbryanv nqueue = VMXNET3_DEF_RX_QUEUES; 536263259Sbryanv if (nqueue > mp_ncpus) 537263259Sbryanv nqueue = mp_ncpus; 538290948Sjhb sc->vmx_max_nrxqueues = trunc_powerof2(nqueue); 539263259Sbryanv 540263259Sbryanv if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 541263259Sbryanv sc->vmx_max_nrxqueues = 1; 542263259Sbryanv sc->vmx_max_ntxqueues = 1; 543263259Sbryanv } 544263259Sbryanv 545255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 546255055Sbryanv if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 547255055Sbryanv ndesc = VMXNET3_DEF_TX_NDESC; 548255055Sbryanv if (ndesc & VMXNET3_MASK_TX_NDESC) 549255055Sbryanv ndesc &= ~VMXNET3_MASK_TX_NDESC; 550255055Sbryanv sc->vmx_ntxdescs = ndesc; 551255055Sbryanv 552255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 553255055Sbryanv if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 554255055Sbryanv ndesc = VMXNET3_DEF_RX_NDESC; 555255055Sbryanv if (ndesc & VMXNET3_MASK_RX_NDESC) 556255055Sbryanv ndesc &= ~VMXNET3_MASK_RX_NDESC; 557255055Sbryanv sc->vmx_nrxdescs = ndesc; 558254738Sbryanv sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 559254738Sbryanv} 560254738Sbryanv 561263259Sbryanvstatic void 562263259Sbryanvvmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 563263259Sbryanv{ 564263259Sbryanv 565263259Sbryanv if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 566263259Sbryanv goto out; 567263259Sbryanv 568263259Sbryanv /* BMV: Just use the maximum configured for now. */ 569263259Sbryanv sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 570263259Sbryanv sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 571263259Sbryanv 572263259Sbryanv if (sc->vmx_nrxqueues > 1) 573263259Sbryanv sc->vmx_flags |= VMXNET3_FLAG_RSS; 574263259Sbryanv 575263259Sbryanv return; 576263259Sbryanv 577263259Sbryanvout: 578263259Sbryanv sc->vmx_ntxqueues = 1; 579263259Sbryanv sc->vmx_nrxqueues = 1; 580263259Sbryanv} 581263259Sbryanv 582254738Sbryanvstatic int 583254738Sbryanvvmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 584254738Sbryanv{ 585254738Sbryanv device_t dev; 586254738Sbryanv int nmsix, cnt, required; 587254738Sbryanv 588254738Sbryanv dev = sc->vmx_dev; 589254738Sbryanv 590254738Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 591254738Sbryanv return (1); 592254738Sbryanv 593254738Sbryanv /* Allocate an additional vector for the events interrupt. */ 594263259Sbryanv required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 595254738Sbryanv 596254738Sbryanv nmsix = pci_msix_count(dev); 597254738Sbryanv if (nmsix < required) 598254738Sbryanv return (1); 599254738Sbryanv 600254738Sbryanv cnt = required; 601254738Sbryanv if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 602254738Sbryanv sc->vmx_nintrs = required; 603254738Sbryanv return (0); 604254738Sbryanv } else 605254738Sbryanv pci_release_msi(dev); 606254738Sbryanv 607263259Sbryanv /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 608263259Sbryanv 609254738Sbryanv return (1); 610254738Sbryanv} 611254738Sbryanv 612254738Sbryanvstatic int 613254738Sbryanvvmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 614254738Sbryanv{ 615254738Sbryanv device_t dev; 616254738Sbryanv int nmsi, cnt, required; 617254738Sbryanv 618254738Sbryanv dev = sc->vmx_dev; 619254738Sbryanv required = 1; 620254738Sbryanv 621254738Sbryanv nmsi = pci_msi_count(dev); 622254738Sbryanv if (nmsi < required) 623254738Sbryanv return (1); 624254738Sbryanv 625254738Sbryanv cnt = required; 626254738Sbryanv if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 627254738Sbryanv sc->vmx_nintrs = 1; 628254738Sbryanv return (0); 629254738Sbryanv } else 630254738Sbryanv pci_release_msi(dev); 631254738Sbryanv 632254738Sbryanv return (1); 633254738Sbryanv} 634254738Sbryanv 635254738Sbryanvstatic int 636254738Sbryanvvmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 637254738Sbryanv{ 638254738Sbryanv 639254738Sbryanv sc->vmx_nintrs = 1; 640254738Sbryanv return (0); 641254738Sbryanv} 642254738Sbryanv 643254738Sbryanvstatic int 644254738Sbryanvvmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 645254738Sbryanv struct vmxnet3_interrupt *intr) 646254738Sbryanv{ 647254738Sbryanv struct resource *irq; 648254738Sbryanv 649254738Sbryanv irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 650254738Sbryanv if (irq == NULL) 651254738Sbryanv return (ENXIO); 652254738Sbryanv 653254738Sbryanv intr->vmxi_irq = irq; 654254738Sbryanv intr->vmxi_rid = rid; 655254738Sbryanv 656254738Sbryanv return (0); 657254738Sbryanv} 658254738Sbryanv 659254738Sbryanvstatic int 660254738Sbryanvvmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 661254738Sbryanv{ 662254738Sbryanv int i, rid, flags, error; 663254738Sbryanv 664254738Sbryanv rid = 0; 665254738Sbryanv flags = RF_ACTIVE; 666254738Sbryanv 667254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 668254738Sbryanv flags |= RF_SHAREABLE; 669254738Sbryanv else 670254738Sbryanv rid = 1; 671254738Sbryanv 672254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 673254738Sbryanv error = vmxnet3_alloc_interrupt(sc, rid, flags, 674254738Sbryanv &sc->vmx_intrs[i]); 675254738Sbryanv if (error) 676254738Sbryanv return (error); 677254738Sbryanv } 678254738Sbryanv 679254738Sbryanv return (0); 680254738Sbryanv} 681254738Sbryanv 682254738Sbryanvstatic int 683254738Sbryanvvmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 684254738Sbryanv{ 685254738Sbryanv device_t dev; 686254738Sbryanv struct vmxnet3_txqueue *txq; 687254738Sbryanv struct vmxnet3_rxqueue *rxq; 688254738Sbryanv struct vmxnet3_interrupt *intr; 689254738Sbryanv enum intr_type type; 690254738Sbryanv int i, error; 691254738Sbryanv 692254738Sbryanv dev = sc->vmx_dev; 693254738Sbryanv intr = &sc->vmx_intrs[0]; 694254738Sbryanv type = INTR_TYPE_NET | INTR_MPSAFE; 695254738Sbryanv 696254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 697254738Sbryanv txq = &sc->vmx_txq[i]; 698254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 699254738Sbryanv vmxnet3_txq_intr, txq, &intr->vmxi_handler); 700254738Sbryanv if (error) 701254738Sbryanv return (error); 702268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 703268012Sbryanv "tq%d", i); 704254738Sbryanv txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 705254738Sbryanv } 706254738Sbryanv 707254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 708254738Sbryanv rxq = &sc->vmx_rxq[i]; 709254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 710254738Sbryanv vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 711254738Sbryanv if (error) 712254738Sbryanv return (error); 713268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 714268012Sbryanv "rq%d", i); 715254738Sbryanv rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 716254738Sbryanv } 717254738Sbryanv 718254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 719254738Sbryanv vmxnet3_event_intr, sc, &intr->vmxi_handler); 720254738Sbryanv if (error) 721254738Sbryanv return (error); 722268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 723254738Sbryanv sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 724254738Sbryanv 725254738Sbryanv return (0); 726254738Sbryanv} 727254738Sbryanv 728254738Sbryanvstatic int 729254738Sbryanvvmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 730254738Sbryanv{ 731254738Sbryanv struct vmxnet3_interrupt *intr; 732254738Sbryanv int i, error; 733254738Sbryanv 734254738Sbryanv intr = &sc->vmx_intrs[0]; 735254738Sbryanv error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 736254738Sbryanv INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 737254738Sbryanv &intr->vmxi_handler); 738254738Sbryanv 739254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 740254738Sbryanv sc->vmx_txq[i].vxtxq_intr_idx = 0; 741254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 742254738Sbryanv sc->vmx_rxq[i].vxrxq_intr_idx = 0; 743254738Sbryanv sc->vmx_event_intr_idx = 0; 744254738Sbryanv 745254738Sbryanv return (error); 746254738Sbryanv} 747254738Sbryanv 748254738Sbryanvstatic void 749254738Sbryanvvmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 750254738Sbryanv{ 751254738Sbryanv struct vmxnet3_txqueue *txq; 752254738Sbryanv struct vmxnet3_txq_shared *txs; 753254738Sbryanv struct vmxnet3_rxqueue *rxq; 754254738Sbryanv struct vmxnet3_rxq_shared *rxs; 755254738Sbryanv int i; 756254738Sbryanv 757254738Sbryanv sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 758254738Sbryanv 759254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 760254738Sbryanv txq = &sc->vmx_txq[i]; 761254738Sbryanv txs = txq->vxtxq_ts; 762254738Sbryanv txs->intr_idx = txq->vxtxq_intr_idx; 763254738Sbryanv } 764254738Sbryanv 765254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 766254738Sbryanv rxq = &sc->vmx_rxq[i]; 767254738Sbryanv rxs = rxq->vxrxq_rs; 768254738Sbryanv rxs->intr_idx = rxq->vxrxq_intr_idx; 769254738Sbryanv } 770254738Sbryanv} 771254738Sbryanv 772254738Sbryanvstatic int 773254738Sbryanvvmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 774254738Sbryanv{ 775254738Sbryanv int error; 776254738Sbryanv 777254738Sbryanv error = vmxnet3_alloc_intr_resources(sc); 778254738Sbryanv if (error) 779254738Sbryanv return (error); 780254738Sbryanv 781254738Sbryanv switch (sc->vmx_intr_type) { 782254738Sbryanv case VMXNET3_IT_MSIX: 783254738Sbryanv error = vmxnet3_setup_msix_interrupts(sc); 784254738Sbryanv break; 785254738Sbryanv case VMXNET3_IT_MSI: 786254738Sbryanv case VMXNET3_IT_LEGACY: 787254738Sbryanv error = vmxnet3_setup_legacy_interrupt(sc); 788254738Sbryanv break; 789254738Sbryanv default: 790254738Sbryanv panic("%s: invalid interrupt type %d", __func__, 791254738Sbryanv sc->vmx_intr_type); 792254738Sbryanv } 793254738Sbryanv 794254738Sbryanv if (error == 0) 795254738Sbryanv vmxnet3_set_interrupt_idx(sc); 796254738Sbryanv 797254738Sbryanv return (error); 798254738Sbryanv} 799254738Sbryanv 800254738Sbryanvstatic int 801254738Sbryanvvmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 802254738Sbryanv{ 803254738Sbryanv device_t dev; 804254738Sbryanv uint32_t config; 805254738Sbryanv int error; 806254738Sbryanv 807254738Sbryanv dev = sc->vmx_dev; 808254738Sbryanv config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 809254738Sbryanv 810254738Sbryanv sc->vmx_intr_type = config & 0x03; 811254738Sbryanv sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 812254738Sbryanv 813254738Sbryanv switch (sc->vmx_intr_type) { 814254738Sbryanv case VMXNET3_IT_AUTO: 815254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSIX; 816254738Sbryanv /* FALLTHROUGH */ 817254738Sbryanv case VMXNET3_IT_MSIX: 818254738Sbryanv error = vmxnet3_alloc_msix_interrupts(sc); 819254738Sbryanv if (error == 0) 820254738Sbryanv break; 821254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSI; 822254738Sbryanv /* FALLTHROUGH */ 823254738Sbryanv case VMXNET3_IT_MSI: 824254738Sbryanv error = vmxnet3_alloc_msi_interrupts(sc); 825254738Sbryanv if (error == 0) 826254738Sbryanv break; 827254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_LEGACY; 828254738Sbryanv /* FALLTHROUGH */ 829254738Sbryanv case VMXNET3_IT_LEGACY: 830254738Sbryanv error = vmxnet3_alloc_legacy_interrupts(sc); 831254738Sbryanv if (error == 0) 832254738Sbryanv break; 833254738Sbryanv /* FALLTHROUGH */ 834254738Sbryanv default: 835254738Sbryanv sc->vmx_intr_type = -1; 836254738Sbryanv device_printf(dev, "cannot allocate any interrupt resources\n"); 837254738Sbryanv return (ENXIO); 838254738Sbryanv } 839254738Sbryanv 840254738Sbryanv return (error); 841254738Sbryanv} 842254738Sbryanv 843254738Sbryanvstatic void 844254738Sbryanvvmxnet3_free_interrupt(struct vmxnet3_softc *sc, 845254738Sbryanv struct vmxnet3_interrupt *intr) 846254738Sbryanv{ 847254738Sbryanv device_t dev; 848254738Sbryanv 849254738Sbryanv dev = sc->vmx_dev; 850254738Sbryanv 851254738Sbryanv if (intr->vmxi_handler != NULL) { 852254738Sbryanv bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 853254738Sbryanv intr->vmxi_handler = NULL; 854254738Sbryanv } 855254738Sbryanv 856254738Sbryanv if (intr->vmxi_irq != NULL) { 857254738Sbryanv bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 858254738Sbryanv intr->vmxi_irq); 859254738Sbryanv intr->vmxi_irq = NULL; 860254738Sbryanv intr->vmxi_rid = -1; 861254738Sbryanv } 862254738Sbryanv} 863254738Sbryanv 864254738Sbryanvstatic void 865254738Sbryanvvmxnet3_free_interrupts(struct vmxnet3_softc *sc) 866254738Sbryanv{ 867254738Sbryanv int i; 868254738Sbryanv 869254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 870254738Sbryanv vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 871254738Sbryanv 872254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_MSI || 873254738Sbryanv sc->vmx_intr_type == VMXNET3_IT_MSIX) 874254738Sbryanv pci_release_msi(sc->vmx_dev); 875254738Sbryanv} 876254738Sbryanv 877263259Sbryanv#ifndef VMXNET3_LEGACY_TX 878254738Sbryanvstatic int 879263259Sbryanvvmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 880263259Sbryanv{ 881263259Sbryanv device_t dev; 882263259Sbryanv 883263259Sbryanv dev = sc->vmx_dev; 884263259Sbryanv 885263259Sbryanv sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 886263259Sbryanv taskqueue_thread_enqueue, &sc->vmx_tq); 887263259Sbryanv if (sc->vmx_tq == NULL) 888263259Sbryanv return (ENOMEM); 889263259Sbryanv 890263259Sbryanv return (0); 891263259Sbryanv} 892263259Sbryanv 893263259Sbryanvstatic void 894263259Sbryanvvmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 895263259Sbryanv{ 896263259Sbryanv device_t dev; 897263259Sbryanv int nthreads, error; 898263259Sbryanv 899263259Sbryanv dev = sc->vmx_dev; 900263259Sbryanv 901263259Sbryanv /* 902263259Sbryanv * The taskqueue is typically not frequently used, so a dedicated 903263259Sbryanv * thread for each queue is unnecessary. 904263259Sbryanv */ 905263259Sbryanv nthreads = MAX(1, sc->vmx_ntxqueues / 2); 906263259Sbryanv 907263259Sbryanv /* 908263259Sbryanv * Most drivers just ignore the return value - it only fails 909263259Sbryanv * with ENOMEM so an error is not likely. It is hard for us 910263259Sbryanv * to recover from an error here. 911263259Sbryanv */ 912263259Sbryanv error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 913263259Sbryanv "%s taskq", device_get_nameunit(dev)); 914263259Sbryanv if (error) 915263259Sbryanv device_printf(dev, "failed to start taskqueue: %d", error); 916263259Sbryanv} 917263259Sbryanv 918263259Sbryanvstatic void 919263259Sbryanvvmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 920263259Sbryanv{ 921263259Sbryanv struct vmxnet3_txqueue *txq; 922263259Sbryanv int i; 923263259Sbryanv 924263259Sbryanv if (sc->vmx_tq != NULL) { 925263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 926263259Sbryanv txq = &sc->vmx_txq[i]; 927263259Sbryanv taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 928263259Sbryanv } 929263259Sbryanv } 930263259Sbryanv} 931263259Sbryanv 932263259Sbryanvstatic void 933263259Sbryanvvmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 934263259Sbryanv{ 935263259Sbryanv if (sc->vmx_tq != NULL) { 936263259Sbryanv taskqueue_free(sc->vmx_tq); 937263259Sbryanv sc->vmx_tq = NULL; 938263259Sbryanv } 939263259Sbryanv} 940263259Sbryanv#endif 941263259Sbryanv 942263259Sbryanvstatic int 943254738Sbryanvvmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 944254738Sbryanv{ 945254738Sbryanv struct vmxnet3_rxqueue *rxq; 946254738Sbryanv struct vmxnet3_rxring *rxr; 947254738Sbryanv int i; 948254738Sbryanv 949254738Sbryanv rxq = &sc->vmx_rxq[q]; 950254738Sbryanv 951254738Sbryanv snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 952254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 953254738Sbryanv mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 954254738Sbryanv 955254738Sbryanv rxq->vxrxq_sc = sc; 956254738Sbryanv rxq->vxrxq_id = q; 957254738Sbryanv 958254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 959254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 960254738Sbryanv rxr->vxrxr_rid = i; 961254738Sbryanv rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 962254738Sbryanv rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 963254738Sbryanv sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 964254738Sbryanv if (rxr->vxrxr_rxbuf == NULL) 965254738Sbryanv return (ENOMEM); 966254950Sbryanv 967254950Sbryanv rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 968254738Sbryanv } 969254738Sbryanv 970254738Sbryanv return (0); 971254738Sbryanv} 972254738Sbryanv 973254738Sbryanvstatic int 974254738Sbryanvvmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 975254738Sbryanv{ 976254738Sbryanv struct vmxnet3_txqueue *txq; 977254738Sbryanv struct vmxnet3_txring *txr; 978254738Sbryanv 979254738Sbryanv txq = &sc->vmx_txq[q]; 980254738Sbryanv txr = &txq->vxtxq_cmd_ring; 981254738Sbryanv 982254738Sbryanv snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 983254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 984254738Sbryanv mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 985254738Sbryanv 986254738Sbryanv txq->vxtxq_sc = sc; 987254738Sbryanv txq->vxtxq_id = q; 988254738Sbryanv 989254738Sbryanv txr->vxtxr_ndesc = sc->vmx_ntxdescs; 990254738Sbryanv txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 991254738Sbryanv sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 992254738Sbryanv if (txr->vxtxr_txbuf == NULL) 993254738Sbryanv return (ENOMEM); 994254738Sbryanv 995254738Sbryanv txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 996254738Sbryanv 997263259Sbryanv#ifndef VMXNET3_LEGACY_TX 998263259Sbryanv TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 999263259Sbryanv 1000263259Sbryanv txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 1001263259Sbryanv M_NOWAIT, &txq->vxtxq_mtx); 1002263259Sbryanv if (txq->vxtxq_br == NULL) 1003263259Sbryanv return (ENOMEM); 1004263259Sbryanv#endif 1005263259Sbryanv 1006254738Sbryanv return (0); 1007254738Sbryanv} 1008254738Sbryanv 1009254738Sbryanvstatic int 1010254738Sbryanvvmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1011254738Sbryanv{ 1012254738Sbryanv int i, error; 1013254738Sbryanv 1014263259Sbryanv /* 1015263259Sbryanv * Only attempt to create multiple queues if MSIX is available. MSIX is 1016263259Sbryanv * disabled by default because its apparently broken for devices passed 1017263259Sbryanv * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1018263259Sbryanv * must be set to zero for MSIX. This check prevents us from allocating 1019263259Sbryanv * queue structures that we will not use. 1020263259Sbryanv */ 1021263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1022263259Sbryanv sc->vmx_max_nrxqueues = 1; 1023263259Sbryanv sc->vmx_max_ntxqueues = 1; 1024263259Sbryanv } 1025263259Sbryanv 1026254738Sbryanv sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1027263259Sbryanv sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1028254738Sbryanv sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1029263259Sbryanv sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1030254738Sbryanv if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1031254738Sbryanv return (ENOMEM); 1032254738Sbryanv 1033263259Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1034254738Sbryanv error = vmxnet3_init_rxq(sc, i); 1035254738Sbryanv if (error) 1036254738Sbryanv return (error); 1037254738Sbryanv } 1038254738Sbryanv 1039263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1040254738Sbryanv error = vmxnet3_init_txq(sc, i); 1041254738Sbryanv if (error) 1042254738Sbryanv return (error); 1043254738Sbryanv } 1044254738Sbryanv 1045254738Sbryanv return (0); 1046254738Sbryanv} 1047254738Sbryanv 1048254738Sbryanvstatic void 1049254738Sbryanvvmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1050254738Sbryanv{ 1051254738Sbryanv struct vmxnet3_rxring *rxr; 1052254738Sbryanv int i; 1053254738Sbryanv 1054254738Sbryanv rxq->vxrxq_sc = NULL; 1055254738Sbryanv rxq->vxrxq_id = -1; 1056254738Sbryanv 1057254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1058254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1059254738Sbryanv 1060254738Sbryanv if (rxr->vxrxr_rxbuf != NULL) { 1061254738Sbryanv free(rxr->vxrxr_rxbuf, M_DEVBUF); 1062254738Sbryanv rxr->vxrxr_rxbuf = NULL; 1063254738Sbryanv } 1064254738Sbryanv } 1065254738Sbryanv 1066254738Sbryanv if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1067254738Sbryanv mtx_destroy(&rxq->vxrxq_mtx); 1068254738Sbryanv} 1069254738Sbryanv 1070254738Sbryanvstatic void 1071254738Sbryanvvmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1072254738Sbryanv{ 1073254738Sbryanv struct vmxnet3_txring *txr; 1074254738Sbryanv 1075254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1076254738Sbryanv 1077254738Sbryanv txq->vxtxq_sc = NULL; 1078254738Sbryanv txq->vxtxq_id = -1; 1079254738Sbryanv 1080263259Sbryanv#ifndef VMXNET3_LEGACY_TX 1081263259Sbryanv if (txq->vxtxq_br != NULL) { 1082263259Sbryanv buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1083263259Sbryanv txq->vxtxq_br = NULL; 1084263259Sbryanv } 1085263259Sbryanv#endif 1086263259Sbryanv 1087254738Sbryanv if (txr->vxtxr_txbuf != NULL) { 1088254738Sbryanv free(txr->vxtxr_txbuf, M_DEVBUF); 1089254738Sbryanv txr->vxtxr_txbuf = NULL; 1090254738Sbryanv } 1091254738Sbryanv 1092254738Sbryanv if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1093254738Sbryanv mtx_destroy(&txq->vxtxq_mtx); 1094254738Sbryanv} 1095254738Sbryanv 1096254738Sbryanvstatic void 1097254738Sbryanvvmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1098254738Sbryanv{ 1099254738Sbryanv int i; 1100254738Sbryanv 1101254738Sbryanv if (sc->vmx_rxq != NULL) { 1102263259Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1103254738Sbryanv vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1104254738Sbryanv free(sc->vmx_rxq, M_DEVBUF); 1105254738Sbryanv sc->vmx_rxq = NULL; 1106254738Sbryanv } 1107254738Sbryanv 1108254738Sbryanv if (sc->vmx_txq != NULL) { 1109263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1110254738Sbryanv vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1111254738Sbryanv free(sc->vmx_txq, M_DEVBUF); 1112254738Sbryanv sc->vmx_txq = NULL; 1113254738Sbryanv } 1114254738Sbryanv} 1115254738Sbryanv 1116254738Sbryanvstatic int 1117254738Sbryanvvmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1118254738Sbryanv{ 1119254738Sbryanv device_t dev; 1120254738Sbryanv uint8_t *kva; 1121254738Sbryanv size_t size; 1122254738Sbryanv int i, error; 1123254738Sbryanv 1124254738Sbryanv dev = sc->vmx_dev; 1125254738Sbryanv 1126254738Sbryanv size = sizeof(struct vmxnet3_driver_shared); 1127254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1128254738Sbryanv if (error) { 1129254738Sbryanv device_printf(dev, "cannot alloc shared memory\n"); 1130254738Sbryanv return (error); 1131254738Sbryanv } 1132254738Sbryanv sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1133254738Sbryanv 1134254738Sbryanv size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1135254738Sbryanv sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1136254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1137254738Sbryanv if (error) { 1138254738Sbryanv device_printf(dev, "cannot alloc queue shared memory\n"); 1139254738Sbryanv return (error); 1140254738Sbryanv } 1141254738Sbryanv sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1142254738Sbryanv kva = sc->vmx_qs; 1143254738Sbryanv 1144254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1145254738Sbryanv sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1146254738Sbryanv kva += sizeof(struct vmxnet3_txq_shared); 1147254738Sbryanv } 1148254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1149254738Sbryanv sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1150254738Sbryanv kva += sizeof(struct vmxnet3_rxq_shared); 1151254738Sbryanv } 1152254738Sbryanv 1153263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1154263259Sbryanv size = sizeof(struct vmxnet3_rss_shared); 1155263259Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1156263259Sbryanv if (error) { 1157263259Sbryanv device_printf(dev, "cannot alloc rss shared memory\n"); 1158263259Sbryanv return (error); 1159263259Sbryanv } 1160263259Sbryanv sc->vmx_rss = 1161263259Sbryanv (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1162263259Sbryanv } 1163263259Sbryanv 1164254738Sbryanv return (0); 1165254738Sbryanv} 1166254738Sbryanv 1167254738Sbryanvstatic void 1168254738Sbryanvvmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1169254738Sbryanv{ 1170254738Sbryanv 1171263259Sbryanv if (sc->vmx_rss != NULL) { 1172263259Sbryanv vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1173263259Sbryanv sc->vmx_rss = NULL; 1174263259Sbryanv } 1175263259Sbryanv 1176254738Sbryanv if (sc->vmx_qs != NULL) { 1177254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1178254738Sbryanv sc->vmx_qs = NULL; 1179254738Sbryanv } 1180254738Sbryanv 1181254738Sbryanv if (sc->vmx_ds != NULL) { 1182254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1183254738Sbryanv sc->vmx_ds = NULL; 1184254738Sbryanv } 1185254738Sbryanv} 1186254738Sbryanv 1187254738Sbryanvstatic int 1188254738Sbryanvvmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1189254738Sbryanv{ 1190254738Sbryanv device_t dev; 1191254738Sbryanv struct vmxnet3_txqueue *txq; 1192254738Sbryanv struct vmxnet3_txring *txr; 1193254738Sbryanv struct vmxnet3_comp_ring *txc; 1194254738Sbryanv size_t descsz, compsz; 1195254738Sbryanv int i, q, error; 1196254738Sbryanv 1197254738Sbryanv dev = sc->vmx_dev; 1198254738Sbryanv 1199254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1200254738Sbryanv txq = &sc->vmx_txq[q]; 1201254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1202254738Sbryanv txc = &txq->vxtxq_comp_ring; 1203254738Sbryanv 1204254738Sbryanv descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1205254738Sbryanv compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1206254738Sbryanv 1207254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1208254738Sbryanv 1, 0, /* alignment, boundary */ 1209254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1210254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1211254738Sbryanv NULL, NULL, /* filter, filterarg */ 1212263259Sbryanv VMXNET3_TX_MAXSIZE, /* maxsize */ 1213254738Sbryanv VMXNET3_TX_MAXSEGS, /* nsegments */ 1214254738Sbryanv VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1215254738Sbryanv 0, /* flags */ 1216254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1217254738Sbryanv &txr->vxtxr_txtag); 1218254738Sbryanv if (error) { 1219254738Sbryanv device_printf(dev, 1220254738Sbryanv "unable to create Tx buffer tag for queue %d\n", q); 1221254738Sbryanv return (error); 1222254738Sbryanv } 1223254738Sbryanv 1224254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1225254738Sbryanv if (error) { 1226254738Sbryanv device_printf(dev, "cannot alloc Tx descriptors for " 1227254738Sbryanv "queue %d error %d\n", q, error); 1228254738Sbryanv return (error); 1229254738Sbryanv } 1230254738Sbryanv txr->vxtxr_txd = 1231254738Sbryanv (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1232254738Sbryanv 1233254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1234254738Sbryanv if (error) { 1235254738Sbryanv device_printf(dev, "cannot alloc Tx comp descriptors " 1236254738Sbryanv "for queue %d error %d\n", q, error); 1237254738Sbryanv return (error); 1238254738Sbryanv } 1239254738Sbryanv txc->vxcr_u.txcd = 1240254738Sbryanv (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1241254738Sbryanv 1242254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1243254738Sbryanv error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1244254738Sbryanv &txr->vxtxr_txbuf[i].vtxb_dmamap); 1245254738Sbryanv if (error) { 1246254738Sbryanv device_printf(dev, "unable to create Tx buf " 1247254738Sbryanv "dmamap for queue %d idx %d\n", q, i); 1248254738Sbryanv return (error); 1249254738Sbryanv } 1250254738Sbryanv } 1251254738Sbryanv } 1252254738Sbryanv 1253254738Sbryanv return (0); 1254254738Sbryanv} 1255254738Sbryanv 1256254738Sbryanvstatic void 1257254738Sbryanvvmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1258254738Sbryanv{ 1259254738Sbryanv device_t dev; 1260254738Sbryanv struct vmxnet3_txqueue *txq; 1261254738Sbryanv struct vmxnet3_txring *txr; 1262254738Sbryanv struct vmxnet3_comp_ring *txc; 1263254738Sbryanv struct vmxnet3_txbuf *txb; 1264254738Sbryanv int i, q; 1265254738Sbryanv 1266254738Sbryanv dev = sc->vmx_dev; 1267254738Sbryanv 1268254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1269254738Sbryanv txq = &sc->vmx_txq[q]; 1270254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1271254738Sbryanv txc = &txq->vxtxq_comp_ring; 1272254738Sbryanv 1273254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1274254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 1275254738Sbryanv if (txb->vtxb_dmamap != NULL) { 1276254738Sbryanv bus_dmamap_destroy(txr->vxtxr_txtag, 1277254738Sbryanv txb->vtxb_dmamap); 1278254738Sbryanv txb->vtxb_dmamap = NULL; 1279254738Sbryanv } 1280254738Sbryanv } 1281254738Sbryanv 1282254738Sbryanv if (txc->vxcr_u.txcd != NULL) { 1283254738Sbryanv vmxnet3_dma_free(sc, &txc->vxcr_dma); 1284254738Sbryanv txc->vxcr_u.txcd = NULL; 1285254738Sbryanv } 1286254738Sbryanv 1287254738Sbryanv if (txr->vxtxr_txd != NULL) { 1288254738Sbryanv vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1289254738Sbryanv txr->vxtxr_txd = NULL; 1290254738Sbryanv } 1291254738Sbryanv 1292254738Sbryanv if (txr->vxtxr_txtag != NULL) { 1293254738Sbryanv bus_dma_tag_destroy(txr->vxtxr_txtag); 1294254738Sbryanv txr->vxtxr_txtag = NULL; 1295254738Sbryanv } 1296254738Sbryanv } 1297254738Sbryanv} 1298254738Sbryanv 1299254738Sbryanvstatic int 1300254738Sbryanvvmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1301254738Sbryanv{ 1302254738Sbryanv device_t dev; 1303254738Sbryanv struct vmxnet3_rxqueue *rxq; 1304254738Sbryanv struct vmxnet3_rxring *rxr; 1305254738Sbryanv struct vmxnet3_comp_ring *rxc; 1306254738Sbryanv int descsz, compsz; 1307254738Sbryanv int i, j, q, error; 1308254738Sbryanv 1309254738Sbryanv dev = sc->vmx_dev; 1310254738Sbryanv 1311254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1312254738Sbryanv rxq = &sc->vmx_rxq[q]; 1313254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1314254738Sbryanv compsz = 0; 1315254738Sbryanv 1316254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1317254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1318254738Sbryanv 1319254738Sbryanv descsz = rxr->vxrxr_ndesc * 1320254738Sbryanv sizeof(struct vmxnet3_rxdesc); 1321254738Sbryanv compsz += rxr->vxrxr_ndesc * 1322254738Sbryanv sizeof(struct vmxnet3_rxcompdesc); 1323254738Sbryanv 1324254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1325254738Sbryanv 1, 0, /* alignment, boundary */ 1326254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1327254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1328254738Sbryanv NULL, NULL, /* filter, filterarg */ 1329254738Sbryanv MJUMPAGESIZE, /* maxsize */ 1330254738Sbryanv 1, /* nsegments */ 1331254738Sbryanv MJUMPAGESIZE, /* maxsegsize */ 1332254738Sbryanv 0, /* flags */ 1333254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1334254738Sbryanv &rxr->vxrxr_rxtag); 1335254738Sbryanv if (error) { 1336254738Sbryanv device_printf(dev, 1337254738Sbryanv "unable to create Rx buffer tag for " 1338254738Sbryanv "queue %d\n", q); 1339254738Sbryanv return (error); 1340254738Sbryanv } 1341254738Sbryanv 1342254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, 1343254738Sbryanv &rxr->vxrxr_dma); 1344254738Sbryanv if (error) { 1345254738Sbryanv device_printf(dev, "cannot allocate Rx " 1346254738Sbryanv "descriptors for queue %d/%d error %d\n", 1347254738Sbryanv i, q, error); 1348254738Sbryanv return (error); 1349254738Sbryanv } 1350254738Sbryanv rxr->vxrxr_rxd = 1351254738Sbryanv (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1352254738Sbryanv } 1353254738Sbryanv 1354254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1355254738Sbryanv if (error) { 1356254738Sbryanv device_printf(dev, "cannot alloc Rx comp descriptors " 1357254738Sbryanv "for queue %d error %d\n", q, error); 1358254738Sbryanv return (error); 1359254738Sbryanv } 1360254738Sbryanv rxc->vxcr_u.rxcd = 1361254738Sbryanv (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1362254738Sbryanv 1363254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1364254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1365254738Sbryanv 1366254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1367254738Sbryanv &rxr->vxrxr_spare_dmap); 1368254738Sbryanv if (error) { 1369254738Sbryanv device_printf(dev, "unable to create spare " 1370254738Sbryanv "dmamap for queue %d/%d error %d\n", 1371254738Sbryanv q, i, error); 1372254738Sbryanv return (error); 1373254738Sbryanv } 1374254738Sbryanv 1375254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1376254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1377254738Sbryanv &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1378254738Sbryanv if (error) { 1379254738Sbryanv device_printf(dev, "unable to create " 1380254738Sbryanv "dmamap for queue %d/%d slot %d " 1381254738Sbryanv "error %d\n", 1382254738Sbryanv q, i, j, error); 1383254738Sbryanv return (error); 1384254738Sbryanv } 1385254738Sbryanv } 1386254738Sbryanv } 1387254738Sbryanv } 1388254738Sbryanv 1389254738Sbryanv return (0); 1390254738Sbryanv} 1391254738Sbryanv 1392254738Sbryanvstatic void 1393254738Sbryanvvmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1394254738Sbryanv{ 1395254738Sbryanv device_t dev; 1396254738Sbryanv struct vmxnet3_rxqueue *rxq; 1397254738Sbryanv struct vmxnet3_rxring *rxr; 1398254738Sbryanv struct vmxnet3_comp_ring *rxc; 1399254738Sbryanv struct vmxnet3_rxbuf *rxb; 1400254738Sbryanv int i, j, q; 1401254738Sbryanv 1402254738Sbryanv dev = sc->vmx_dev; 1403254738Sbryanv 1404254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1405254738Sbryanv rxq = &sc->vmx_rxq[q]; 1406254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1407254738Sbryanv 1408254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1409254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1410254738Sbryanv 1411254738Sbryanv if (rxr->vxrxr_spare_dmap != NULL) { 1412254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1413254738Sbryanv rxr->vxrxr_spare_dmap); 1414254738Sbryanv rxr->vxrxr_spare_dmap = NULL; 1415254738Sbryanv } 1416254738Sbryanv 1417254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1418254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 1419254738Sbryanv if (rxb->vrxb_dmamap != NULL) { 1420254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1421254738Sbryanv rxb->vrxb_dmamap); 1422254738Sbryanv rxb->vrxb_dmamap = NULL; 1423254738Sbryanv } 1424254738Sbryanv } 1425254738Sbryanv } 1426254738Sbryanv 1427254738Sbryanv if (rxc->vxcr_u.rxcd != NULL) { 1428254738Sbryanv vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1429254738Sbryanv rxc->vxcr_u.rxcd = NULL; 1430254738Sbryanv } 1431254738Sbryanv 1432254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1433254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1434254738Sbryanv 1435254738Sbryanv if (rxr->vxrxr_rxd != NULL) { 1436254738Sbryanv vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1437254738Sbryanv rxr->vxrxr_rxd = NULL; 1438254738Sbryanv } 1439254738Sbryanv 1440254738Sbryanv if (rxr->vxrxr_rxtag != NULL) { 1441254738Sbryanv bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1442254738Sbryanv rxr->vxrxr_rxtag = NULL; 1443254738Sbryanv } 1444254738Sbryanv } 1445254738Sbryanv } 1446254738Sbryanv} 1447254738Sbryanv 1448254738Sbryanvstatic int 1449254738Sbryanvvmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1450254738Sbryanv{ 1451254738Sbryanv int error; 1452254738Sbryanv 1453254738Sbryanv error = vmxnet3_alloc_txq_data(sc); 1454254738Sbryanv if (error) 1455254738Sbryanv return (error); 1456254738Sbryanv 1457254738Sbryanv error = vmxnet3_alloc_rxq_data(sc); 1458254738Sbryanv if (error) 1459254738Sbryanv return (error); 1460254738Sbryanv 1461254738Sbryanv return (0); 1462254738Sbryanv} 1463254738Sbryanv 1464254738Sbryanvstatic void 1465254738Sbryanvvmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1466254738Sbryanv{ 1467254738Sbryanv 1468254950Sbryanv if (sc->vmx_rxq != NULL) 1469254950Sbryanv vmxnet3_free_rxq_data(sc); 1470254950Sbryanv 1471254950Sbryanv if (sc->vmx_txq != NULL) 1472254950Sbryanv vmxnet3_free_txq_data(sc); 1473254738Sbryanv} 1474254738Sbryanv 1475254738Sbryanvstatic int 1476254738Sbryanvvmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1477254738Sbryanv{ 1478254738Sbryanv int error; 1479254738Sbryanv 1480254738Sbryanv error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1481254738Sbryanv 32, &sc->vmx_mcast_dma); 1482254738Sbryanv if (error) 1483254738Sbryanv device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1484254738Sbryanv else 1485254738Sbryanv sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1486254738Sbryanv 1487254738Sbryanv return (error); 1488254738Sbryanv} 1489254738Sbryanv 1490254738Sbryanvstatic void 1491254738Sbryanvvmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1492254738Sbryanv{ 1493254738Sbryanv 1494254738Sbryanv if (sc->vmx_mcast != NULL) { 1495254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1496254738Sbryanv sc->vmx_mcast = NULL; 1497254738Sbryanv } 1498254738Sbryanv} 1499254738Sbryanv 1500254738Sbryanvstatic void 1501254738Sbryanvvmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1502254738Sbryanv{ 1503254738Sbryanv struct vmxnet3_driver_shared *ds; 1504254738Sbryanv struct vmxnet3_txqueue *txq; 1505254738Sbryanv struct vmxnet3_txq_shared *txs; 1506254738Sbryanv struct vmxnet3_rxqueue *rxq; 1507254738Sbryanv struct vmxnet3_rxq_shared *rxs; 1508254738Sbryanv int i; 1509254738Sbryanv 1510254738Sbryanv ds = sc->vmx_ds; 1511254738Sbryanv 1512254738Sbryanv /* 1513254738Sbryanv * Initialize fields of the shared data that remains the same across 1514254738Sbryanv * reinits. Note the shared data is zero'd when allocated. 1515254738Sbryanv */ 1516254738Sbryanv 1517254738Sbryanv ds->magic = VMXNET3_REV1_MAGIC; 1518254738Sbryanv 1519254738Sbryanv /* DriverInfo */ 1520254738Sbryanv ds->version = VMXNET3_DRIVER_VERSION; 1521256308Sbryanv ds->guest = VMXNET3_GOS_FREEBSD | 1522254738Sbryanv#ifdef __LP64__ 1523254738Sbryanv VMXNET3_GOS_64BIT; 1524254738Sbryanv#else 1525254738Sbryanv VMXNET3_GOS_32BIT; 1526254738Sbryanv#endif 1527254738Sbryanv ds->vmxnet3_revision = 1; 1528254738Sbryanv ds->upt_version = 1; 1529254738Sbryanv 1530254738Sbryanv /* Misc. conf */ 1531254738Sbryanv ds->driver_data = vtophys(sc); 1532254738Sbryanv ds->driver_data_len = sizeof(struct vmxnet3_softc); 1533254738Sbryanv ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1534254738Sbryanv ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1535254738Sbryanv ds->nrxsg_max = sc->vmx_max_rxsegs; 1536254738Sbryanv 1537263259Sbryanv /* RSS conf */ 1538263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1539263259Sbryanv ds->rss.version = 1; 1540263259Sbryanv ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1541263259Sbryanv ds->rss.len = sc->vmx_rss_dma.dma_size; 1542263259Sbryanv } 1543263259Sbryanv 1544254738Sbryanv /* Interrupt control. */ 1545254738Sbryanv ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1546254738Sbryanv ds->nintr = sc->vmx_nintrs; 1547254738Sbryanv ds->evintr = sc->vmx_event_intr_idx; 1548254738Sbryanv ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1549254738Sbryanv 1550254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 1551254738Sbryanv ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1552254738Sbryanv 1553254738Sbryanv /* Receive filter. */ 1554254738Sbryanv ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1555254738Sbryanv ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1556254738Sbryanv 1557254738Sbryanv /* Tx queues */ 1558254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1559254738Sbryanv txq = &sc->vmx_txq[i]; 1560254738Sbryanv txs = txq->vxtxq_ts; 1561254738Sbryanv 1562254738Sbryanv txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1563254950Sbryanv txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1564254738Sbryanv txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1565254950Sbryanv txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1566254738Sbryanv txs->driver_data = vtophys(txq); 1567254738Sbryanv txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1568254738Sbryanv } 1569254738Sbryanv 1570254738Sbryanv /* Rx queues */ 1571254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1572254738Sbryanv rxq = &sc->vmx_rxq[i]; 1573254738Sbryanv rxs = rxq->vxrxq_rs; 1574254738Sbryanv 1575254738Sbryanv rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1576254738Sbryanv rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1577254738Sbryanv rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1578254738Sbryanv rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1579254738Sbryanv rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1580254950Sbryanv rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1581254738Sbryanv rxs->driver_data = vtophys(rxq); 1582254738Sbryanv rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1583254738Sbryanv } 1584254738Sbryanv} 1585254738Sbryanv 1586254738Sbryanvstatic void 1587303136Smavvmxnet3_init_hwassist(struct vmxnet3_softc *sc) 1588303136Smav{ 1589303136Smav struct ifnet *ifp = sc->vmx_ifp; 1590303136Smav uint64_t hwassist; 1591303136Smav 1592303136Smav hwassist = 0; 1593303136Smav if (ifp->if_capenable & IFCAP_TXCSUM) 1594303136Smav hwassist |= VMXNET3_CSUM_OFFLOAD; 1595303136Smav if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1596303136Smav hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1597303136Smav if (ifp->if_capenable & IFCAP_TSO4) 1598303136Smav hwassist |= CSUM_IP_TSO; 1599303136Smav if (ifp->if_capenable & IFCAP_TSO6) 1600303136Smav hwassist |= CSUM_IP6_TSO; 1601303136Smav ifp->if_hwassist = hwassist; 1602303136Smav} 1603303136Smav 1604303136Smavstatic void 1605254738Sbryanvvmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1606254738Sbryanv{ 1607254738Sbryanv struct ifnet *ifp; 1608254738Sbryanv 1609254738Sbryanv ifp = sc->vmx_ifp; 1610254738Sbryanv 1611254738Sbryanv /* Use the current MAC address. */ 1612254738Sbryanv bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1613254738Sbryanv vmxnet3_set_lladdr(sc); 1614254738Sbryanv 1615303136Smav vmxnet3_init_hwassist(sc); 1616254738Sbryanv} 1617254738Sbryanv 1618254738Sbryanvstatic void 1619263259Sbryanvvmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1620263259Sbryanv{ 1621263259Sbryanv /* 1622263259Sbryanv * Use the same key as the Linux driver until FreeBSD can do 1623263259Sbryanv * RSS (presumably Toeplitz) in software. 1624263259Sbryanv */ 1625263259Sbryanv static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1626263259Sbryanv 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1627263259Sbryanv 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1628263259Sbryanv 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1629263259Sbryanv 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1630263259Sbryanv 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1631263259Sbryanv }; 1632263259Sbryanv 1633263259Sbryanv struct vmxnet3_driver_shared *ds; 1634263259Sbryanv struct vmxnet3_rss_shared *rss; 1635263259Sbryanv int i; 1636263259Sbryanv 1637263259Sbryanv ds = sc->vmx_ds; 1638263259Sbryanv rss = sc->vmx_rss; 1639263259Sbryanv 1640263259Sbryanv rss->hash_type = 1641263259Sbryanv UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1642263259Sbryanv UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1643263259Sbryanv rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1644263259Sbryanv rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1645263259Sbryanv rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1646263259Sbryanv memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1647263259Sbryanv 1648263259Sbryanv for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1649263259Sbryanv rss->ind_table[i] = i % sc->vmx_nrxqueues; 1650263259Sbryanv} 1651263259Sbryanv 1652263259Sbryanvstatic void 1653254738Sbryanvvmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1654254738Sbryanv{ 1655254738Sbryanv struct ifnet *ifp; 1656254738Sbryanv struct vmxnet3_driver_shared *ds; 1657254738Sbryanv 1658254738Sbryanv ifp = sc->vmx_ifp; 1659254738Sbryanv ds = sc->vmx_ds; 1660254738Sbryanv 1661263259Sbryanv ds->mtu = ifp->if_mtu; 1662263259Sbryanv ds->ntxqueue = sc->vmx_ntxqueues; 1663263259Sbryanv ds->nrxqueue = sc->vmx_nrxqueues; 1664263259Sbryanv 1665254738Sbryanv ds->upt_features = 0; 1666255055Sbryanv if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1667255055Sbryanv ds->upt_features |= UPT1_F_CSUM; 1668254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1669254738Sbryanv ds->upt_features |= UPT1_F_VLAN; 1670254738Sbryanv if (ifp->if_capenable & IFCAP_LRO) 1671254738Sbryanv ds->upt_features |= UPT1_F_LRO; 1672254738Sbryanv 1673263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1674263259Sbryanv ds->upt_features |= UPT1_F_RSS; 1675263259Sbryanv vmxnet3_reinit_rss_shared_data(sc); 1676263259Sbryanv } 1677254738Sbryanv 1678254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1679254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1680254738Sbryanv (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1681254738Sbryanv} 1682254738Sbryanv 1683254738Sbryanvstatic int 1684254738Sbryanvvmxnet3_alloc_data(struct vmxnet3_softc *sc) 1685254738Sbryanv{ 1686254738Sbryanv int error; 1687254738Sbryanv 1688254738Sbryanv error = vmxnet3_alloc_shared_data(sc); 1689254738Sbryanv if (error) 1690254738Sbryanv return (error); 1691254738Sbryanv 1692254738Sbryanv error = vmxnet3_alloc_queue_data(sc); 1693254738Sbryanv if (error) 1694254738Sbryanv return (error); 1695254738Sbryanv 1696254738Sbryanv error = vmxnet3_alloc_mcast_table(sc); 1697254738Sbryanv if (error) 1698254738Sbryanv return (error); 1699254738Sbryanv 1700254738Sbryanv vmxnet3_init_shared_data(sc); 1701254738Sbryanv 1702254738Sbryanv return (0); 1703254738Sbryanv} 1704254738Sbryanv 1705254738Sbryanvstatic void 1706254738Sbryanvvmxnet3_free_data(struct vmxnet3_softc *sc) 1707254738Sbryanv{ 1708254738Sbryanv 1709254738Sbryanv vmxnet3_free_mcast_table(sc); 1710254738Sbryanv vmxnet3_free_queue_data(sc); 1711254738Sbryanv vmxnet3_free_shared_data(sc); 1712254738Sbryanv} 1713254738Sbryanv 1714254738Sbryanvstatic int 1715254738Sbryanvvmxnet3_setup_interface(struct vmxnet3_softc *sc) 1716254738Sbryanv{ 1717254738Sbryanv device_t dev; 1718254738Sbryanv struct ifnet *ifp; 1719254738Sbryanv 1720254738Sbryanv dev = sc->vmx_dev; 1721254738Sbryanv 1722254738Sbryanv ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1723254738Sbryanv if (ifp == NULL) { 1724254738Sbryanv device_printf(dev, "cannot allocate ifnet structure\n"); 1725254738Sbryanv return (ENOSPC); 1726254738Sbryanv } 1727254738Sbryanv 1728254738Sbryanv if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1729263259Sbryanv#if __FreeBSD_version < 1000025 1730263259Sbryanv ifp->if_baudrate = 1000000000; 1731263259Sbryanv#elif __FreeBSD_version < 1100011 1732263259Sbryanv if_initbaudrate(ifp, IF_Gbps(10)); 1733263259Sbryanv#else 1734263102Sglebius ifp->if_baudrate = IF_Gbps(10); 1735263259Sbryanv#endif 1736254738Sbryanv ifp->if_softc = sc; 1737254738Sbryanv ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1738254738Sbryanv ifp->if_init = vmxnet3_init; 1739254738Sbryanv ifp->if_ioctl = vmxnet3_ioctl; 1740272099Sglebius ifp->if_get_counter = vmxnet3_get_counter; 1741271946Shselasky ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1742271946Shselasky ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1743271946Shselasky ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1744263259Sbryanv 1745263259Sbryanv#ifdef VMXNET3_LEGACY_TX 1746254738Sbryanv ifp->if_start = vmxnet3_start; 1747254738Sbryanv ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1748254738Sbryanv IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1749254738Sbryanv IFQ_SET_READY(&ifp->if_snd); 1750263259Sbryanv#else 1751263259Sbryanv ifp->if_transmit = vmxnet3_txq_mq_start; 1752263259Sbryanv ifp->if_qflush = vmxnet3_qflush; 1753263259Sbryanv#endif 1754254738Sbryanv 1755254738Sbryanv vmxnet3_get_lladdr(sc); 1756254738Sbryanv ether_ifattach(ifp, sc->vmx_lladdr); 1757254738Sbryanv 1758254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1759254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1760254738Sbryanv ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1761255055Sbryanv ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1762255055Sbryanv IFCAP_VLAN_HWCSUM; 1763254738Sbryanv ifp->if_capenable = ifp->if_capabilities; 1764254738Sbryanv 1765255055Sbryanv /* These capabilities are not enabled by default. */ 1766255055Sbryanv ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1767254738Sbryanv 1768254738Sbryanv sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1769254738Sbryanv vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1770254738Sbryanv sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1771254738Sbryanv vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1772254738Sbryanv 1773254738Sbryanv ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1774254738Sbryanv vmxnet3_media_status); 1775254738Sbryanv ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1776254738Sbryanv ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1777254738Sbryanv 1778254738Sbryanv return (0); 1779254738Sbryanv} 1780254738Sbryanv 1781254738Sbryanvstatic void 1782254738Sbryanvvmxnet3_evintr(struct vmxnet3_softc *sc) 1783254738Sbryanv{ 1784254738Sbryanv device_t dev; 1785254738Sbryanv struct ifnet *ifp; 1786254738Sbryanv struct vmxnet3_txq_shared *ts; 1787254738Sbryanv struct vmxnet3_rxq_shared *rs; 1788254738Sbryanv uint32_t event; 1789254738Sbryanv int reset; 1790254738Sbryanv 1791254738Sbryanv dev = sc->vmx_dev; 1792254738Sbryanv ifp = sc->vmx_ifp; 1793254738Sbryanv reset = 0; 1794254738Sbryanv 1795254738Sbryanv VMXNET3_CORE_LOCK(sc); 1796254738Sbryanv 1797254738Sbryanv /* Clear events. */ 1798254738Sbryanv event = sc->vmx_ds->event; 1799254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1800254738Sbryanv 1801263259Sbryanv if (event & VMXNET3_EVENT_LINK) { 1802254738Sbryanv vmxnet3_link_status(sc); 1803263259Sbryanv if (sc->vmx_link_active != 0) 1804263259Sbryanv vmxnet3_tx_start_all(sc); 1805263259Sbryanv } 1806254738Sbryanv 1807254738Sbryanv if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1808254738Sbryanv reset = 1; 1809254738Sbryanv vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1810254738Sbryanv ts = sc->vmx_txq[0].vxtxq_ts; 1811254738Sbryanv if (ts->stopped != 0) 1812254738Sbryanv device_printf(dev, "Tx queue error %#x\n", ts->error); 1813254738Sbryanv rs = sc->vmx_rxq[0].vxrxq_rs; 1814254738Sbryanv if (rs->stopped != 0) 1815254738Sbryanv device_printf(dev, "Rx queue error %#x\n", rs->error); 1816254738Sbryanv device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1817254738Sbryanv } 1818254738Sbryanv 1819254738Sbryanv if (event & VMXNET3_EVENT_DIC) 1820254738Sbryanv device_printf(dev, "device implementation change event\n"); 1821254738Sbryanv if (event & VMXNET3_EVENT_DEBUG) 1822254738Sbryanv device_printf(dev, "debug event\n"); 1823254738Sbryanv 1824254738Sbryanv if (reset != 0) { 1825254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1826254738Sbryanv vmxnet3_init_locked(sc); 1827254738Sbryanv } 1828254738Sbryanv 1829254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 1830254738Sbryanv} 1831254738Sbryanv 1832254738Sbryanvstatic void 1833254738Sbryanvvmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1834254738Sbryanv{ 1835254738Sbryanv struct vmxnet3_softc *sc; 1836254738Sbryanv struct ifnet *ifp; 1837254738Sbryanv struct vmxnet3_txring *txr; 1838254738Sbryanv struct vmxnet3_comp_ring *txc; 1839254738Sbryanv struct vmxnet3_txcompdesc *txcd; 1840254738Sbryanv struct vmxnet3_txbuf *txb; 1841263259Sbryanv struct mbuf *m; 1842254738Sbryanv u_int sop; 1843254738Sbryanv 1844254738Sbryanv sc = txq->vxtxq_sc; 1845254738Sbryanv ifp = sc->vmx_ifp; 1846254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1847254738Sbryanv txc = &txq->vxtxq_comp_ring; 1848254738Sbryanv 1849254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 1850254738Sbryanv 1851254738Sbryanv for (;;) { 1852254738Sbryanv txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1853254738Sbryanv if (txcd->gen != txc->vxcr_gen) 1854254738Sbryanv break; 1855254950Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1856254738Sbryanv 1857254738Sbryanv if (++txc->vxcr_next == txc->vxcr_ndesc) { 1858254738Sbryanv txc->vxcr_next = 0; 1859254738Sbryanv txc->vxcr_gen ^= 1; 1860254738Sbryanv } 1861254738Sbryanv 1862254738Sbryanv sop = txr->vxtxr_next; 1863254738Sbryanv txb = &txr->vxtxr_txbuf[sop]; 1864254738Sbryanv 1865263259Sbryanv if ((m = txb->vtxb_m) != NULL) { 1866254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1867254738Sbryanv BUS_DMASYNC_POSTWRITE); 1868254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1869254738Sbryanv 1870263259Sbryanv txq->vxtxq_stats.vmtxs_opackets++; 1871263259Sbryanv txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1872263259Sbryanv if (m->m_flags & M_MCAST) 1873263259Sbryanv txq->vxtxq_stats.vmtxs_omcasts++; 1874263259Sbryanv 1875263259Sbryanv m_freem(m); 1876254738Sbryanv txb->vtxb_m = NULL; 1877254738Sbryanv } 1878254738Sbryanv 1879254738Sbryanv txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1880254738Sbryanv } 1881254738Sbryanv 1882254738Sbryanv if (txr->vxtxr_head == txr->vxtxr_next) 1883254738Sbryanv txq->vxtxq_watchdog = 0; 1884254738Sbryanv} 1885254738Sbryanv 1886254738Sbryanvstatic int 1887254738Sbryanvvmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1888254738Sbryanv{ 1889254738Sbryanv struct ifnet *ifp; 1890254738Sbryanv struct mbuf *m; 1891254738Sbryanv struct vmxnet3_rxdesc *rxd; 1892254738Sbryanv struct vmxnet3_rxbuf *rxb; 1893254738Sbryanv bus_dma_tag_t tag; 1894254738Sbryanv bus_dmamap_t dmap; 1895254738Sbryanv bus_dma_segment_t segs[1]; 1896254738Sbryanv int idx, clsize, btype, flags, nsegs, error; 1897254738Sbryanv 1898254738Sbryanv ifp = sc->vmx_ifp; 1899254738Sbryanv tag = rxr->vxrxr_rxtag; 1900254738Sbryanv dmap = rxr->vxrxr_spare_dmap; 1901254738Sbryanv idx = rxr->vxrxr_fill; 1902254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 1903254738Sbryanv rxb = &rxr->vxrxr_rxbuf[idx]; 1904254738Sbryanv 1905254738Sbryanv#ifdef VMXNET3_FAILPOINTS 1906254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1907254738Sbryanv if (rxr->vxrxr_rid != 0) 1908254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1909254738Sbryanv#endif 1910254738Sbryanv 1911254738Sbryanv if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1912254738Sbryanv flags = M_PKTHDR; 1913254738Sbryanv clsize = MCLBYTES; 1914254738Sbryanv btype = VMXNET3_BTYPE_HEAD; 1915254738Sbryanv } else { 1916254738Sbryanv#if __FreeBSD_version < 902001 1917254738Sbryanv /* 1918254738Sbryanv * These mbufs will never be used for the start of a frame. 1919254738Sbryanv * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1920254738Sbryanv * required the mbuf to always be a packet header. Avoid 1921254738Sbryanv * unnecessary mbuf initialization in newer versions where 1922254738Sbryanv * that is not the case. 1923254738Sbryanv */ 1924254738Sbryanv flags = M_PKTHDR; 1925254738Sbryanv#else 1926254738Sbryanv flags = 0; 1927254738Sbryanv#endif 1928254738Sbryanv clsize = MJUMPAGESIZE; 1929254738Sbryanv btype = VMXNET3_BTYPE_BODY; 1930254738Sbryanv } 1931254738Sbryanv 1932254738Sbryanv m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1933254738Sbryanv if (m == NULL) { 1934254738Sbryanv sc->vmx_stats.vmst_mgetcl_failed++; 1935254738Sbryanv return (ENOBUFS); 1936254738Sbryanv } 1937254738Sbryanv 1938254738Sbryanv if (btype == VMXNET3_BTYPE_HEAD) { 1939254738Sbryanv m->m_len = m->m_pkthdr.len = clsize; 1940254738Sbryanv m_adj(m, ETHER_ALIGN); 1941254738Sbryanv } else 1942254738Sbryanv m->m_len = clsize; 1943254738Sbryanv 1944254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1945254738Sbryanv BUS_DMA_NOWAIT); 1946254738Sbryanv if (error) { 1947254738Sbryanv m_freem(m); 1948254950Sbryanv sc->vmx_stats.vmst_mbuf_load_failed++; 1949254738Sbryanv return (error); 1950254738Sbryanv } 1951254738Sbryanv KASSERT(nsegs == 1, 1952254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1953254738Sbryanv#if __FreeBSD_version < 902001 1954254738Sbryanv if (btype == VMXNET3_BTYPE_BODY) 1955254738Sbryanv m->m_flags &= ~M_PKTHDR; 1956254738Sbryanv#endif 1957254738Sbryanv 1958254738Sbryanv if (rxb->vrxb_m != NULL) { 1959254738Sbryanv bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1960254738Sbryanv bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1961254738Sbryanv } 1962254738Sbryanv 1963254738Sbryanv rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1964254738Sbryanv rxb->vrxb_dmamap = dmap; 1965254738Sbryanv rxb->vrxb_m = m; 1966254738Sbryanv 1967254738Sbryanv rxd->addr = segs[0].ds_addr; 1968254738Sbryanv rxd->len = segs[0].ds_len; 1969254738Sbryanv rxd->btype = btype; 1970254738Sbryanv rxd->gen = rxr->vxrxr_gen; 1971254738Sbryanv 1972254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 1973254738Sbryanv return (0); 1974254738Sbryanv} 1975254738Sbryanv 1976254738Sbryanvstatic void 1977254738Sbryanvvmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1978254738Sbryanv struct vmxnet3_rxring *rxr, int idx) 1979254738Sbryanv{ 1980254738Sbryanv struct vmxnet3_rxdesc *rxd; 1981254738Sbryanv 1982254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 1983254738Sbryanv rxd->gen = rxr->vxrxr_gen; 1984254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 1985254738Sbryanv} 1986254738Sbryanv 1987254738Sbryanvstatic void 1988254738Sbryanvvmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1989254738Sbryanv{ 1990254738Sbryanv struct vmxnet3_softc *sc; 1991254738Sbryanv struct vmxnet3_rxring *rxr; 1992254738Sbryanv struct vmxnet3_comp_ring *rxc; 1993254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 1994254738Sbryanv int idx, eof; 1995254738Sbryanv 1996254738Sbryanv sc = rxq->vxrxq_sc; 1997254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1998254738Sbryanv 1999254738Sbryanv do { 2000254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2001254738Sbryanv if (rxcd->gen != rxc->vxcr_gen) 2002254738Sbryanv break; /* Not expected. */ 2003254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2004254738Sbryanv 2005254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2006254738Sbryanv rxc->vxcr_next = 0; 2007254738Sbryanv rxc->vxcr_gen ^= 1; 2008254738Sbryanv } 2009254738Sbryanv 2010254738Sbryanv idx = rxcd->rxd_idx; 2011254738Sbryanv eof = rxcd->eop; 2012254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2013254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2014254738Sbryanv else 2015254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2016254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2017254738Sbryanv } while (!eof); 2018254738Sbryanv} 2019254738Sbryanv 2020254738Sbryanvstatic void 2021254738Sbryanvvmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2022254738Sbryanv{ 2023254738Sbryanv 2024254738Sbryanv if (rxcd->ipv4) { 2025254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2026254738Sbryanv if (rxcd->ipcsum_ok) 2027254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2028254738Sbryanv } 2029254738Sbryanv 2030254738Sbryanv if (!rxcd->fragment) { 2031254738Sbryanv if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2032254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2033254738Sbryanv CSUM_PSEUDO_HDR; 2034254738Sbryanv m->m_pkthdr.csum_data = 0xFFFF; 2035254738Sbryanv } 2036254738Sbryanv } 2037254738Sbryanv} 2038254738Sbryanv 2039254738Sbryanvstatic void 2040254738Sbryanvvmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2041254738Sbryanv struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2042254738Sbryanv{ 2043254738Sbryanv struct vmxnet3_softc *sc; 2044254738Sbryanv struct ifnet *ifp; 2045254738Sbryanv 2046254738Sbryanv sc = rxq->vxrxq_sc; 2047254738Sbryanv ifp = sc->vmx_ifp; 2048254738Sbryanv 2049254738Sbryanv if (rxcd->error) { 2050263259Sbryanv rxq->vxrxq_stats.vmrxs_ierrors++; 2051254738Sbryanv m_freem(m); 2052254738Sbryanv return; 2053254738Sbryanv } 2054254738Sbryanv 2055263259Sbryanv#ifdef notyet 2056263259Sbryanv switch (rxcd->rss_type) { 2057263259Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV4: 2058263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2059263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2060263259Sbryanv break; 2061263259Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2062263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2063263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2064263259Sbryanv break; 2065263259Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV6: 2066263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2067263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2068263259Sbryanv break; 2069263259Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2070263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2071263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2072263259Sbryanv break; 2073263259Sbryanv default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2074263259Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2075263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2076263259Sbryanv break; 2077263259Sbryanv } 2078263259Sbryanv#else 2079263259Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2080275358Shselasky M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2081263259Sbryanv#endif 2082263259Sbryanv 2083254738Sbryanv if (!rxcd->no_csum) 2084254738Sbryanv vmxnet3_rx_csum(rxcd, m); 2085254738Sbryanv if (rxcd->vlan) { 2086254738Sbryanv m->m_flags |= M_VLANTAG; 2087254738Sbryanv m->m_pkthdr.ether_vtag = rxcd->vtag; 2088254738Sbryanv } 2089254738Sbryanv 2090263259Sbryanv rxq->vxrxq_stats.vmrxs_ipackets++; 2091263259Sbryanv rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2092263259Sbryanv 2093254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2094254738Sbryanv (*ifp->if_input)(ifp, m); 2095254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2096254738Sbryanv} 2097254738Sbryanv 2098254738Sbryanvstatic void 2099254738Sbryanvvmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2100254738Sbryanv{ 2101254738Sbryanv struct vmxnet3_softc *sc; 2102254738Sbryanv struct ifnet *ifp; 2103254738Sbryanv struct vmxnet3_rxring *rxr; 2104254738Sbryanv struct vmxnet3_comp_ring *rxc; 2105254738Sbryanv struct vmxnet3_rxdesc *rxd; 2106254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 2107254738Sbryanv struct mbuf *m, *m_head, *m_tail; 2108254738Sbryanv int idx, length; 2109254738Sbryanv 2110254738Sbryanv sc = rxq->vxrxq_sc; 2111254738Sbryanv ifp = sc->vmx_ifp; 2112254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2113254738Sbryanv 2114254738Sbryanv VMXNET3_RXQ_LOCK_ASSERT(rxq); 2115254738Sbryanv 2116254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2117254738Sbryanv return; 2118254738Sbryanv 2119267662Sbryanv m_head = rxq->vxrxq_mhead; 2120267662Sbryanv rxq->vxrxq_mhead = NULL; 2121267662Sbryanv m_tail = rxq->vxrxq_mtail; 2122267662Sbryanv rxq->vxrxq_mtail = NULL; 2123267662Sbryanv MPASS(m_head == NULL || m_tail != NULL); 2124267662Sbryanv 2125254738Sbryanv for (;;) { 2126254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2127267662Sbryanv if (rxcd->gen != rxc->vxcr_gen) { 2128267662Sbryanv rxq->vxrxq_mhead = m_head; 2129267662Sbryanv rxq->vxrxq_mtail = m_tail; 2130254738Sbryanv break; 2131267662Sbryanv } 2132254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2133254738Sbryanv 2134254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2135254738Sbryanv rxc->vxcr_next = 0; 2136254738Sbryanv rxc->vxcr_gen ^= 1; 2137254738Sbryanv } 2138254738Sbryanv 2139254738Sbryanv idx = rxcd->rxd_idx; 2140254738Sbryanv length = rxcd->len; 2141254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2142254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2143254738Sbryanv else 2144254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2145254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 2146254738Sbryanv 2147254738Sbryanv m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2148254738Sbryanv KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2149254738Sbryanv __func__, rxcd->qid, idx)); 2150254738Sbryanv 2151254738Sbryanv /* 2152254738Sbryanv * The host may skip descriptors. We detect this when this 2153254738Sbryanv * descriptor does not match the previous fill index. Catch 2154254738Sbryanv * up with the host now. 2155254738Sbryanv */ 2156254738Sbryanv if (__predict_false(rxr->vxrxr_fill != idx)) { 2157254738Sbryanv while (rxr->vxrxr_fill != idx) { 2158254738Sbryanv rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2159254738Sbryanv rxr->vxrxr_gen; 2160254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 2161254738Sbryanv } 2162254738Sbryanv } 2163254738Sbryanv 2164254738Sbryanv if (rxcd->sop) { 2165254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2166254738Sbryanv ("%s: start of frame w/o head buffer", __func__)); 2167254738Sbryanv KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2168254738Sbryanv ("%s: start of frame not in ring 0", __func__)); 2169254738Sbryanv KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2170254738Sbryanv ("%s: start of frame at unexcepted index %d (%d)", 2171254738Sbryanv __func__, idx, sc->vmx_rx_max_chain)); 2172254738Sbryanv KASSERT(m_head == NULL, 2173254738Sbryanv ("%s: duplicate start of frame?", __func__)); 2174254738Sbryanv 2175254738Sbryanv if (length == 0) { 2176254738Sbryanv /* Just ignore this descriptor. */ 2177254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2178254738Sbryanv goto nextp; 2179254738Sbryanv } 2180254738Sbryanv 2181254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2182263259Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2183254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2184254738Sbryanv if (!rxcd->eop) 2185254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2186254738Sbryanv goto nextp; 2187254738Sbryanv } 2188254738Sbryanv 2189254738Sbryanv m->m_pkthdr.rcvif = ifp; 2190254738Sbryanv m->m_pkthdr.len = m->m_len = length; 2191254738Sbryanv m->m_pkthdr.csum_flags = 0; 2192254738Sbryanv m_head = m_tail = m; 2193254738Sbryanv 2194254738Sbryanv } else { 2195254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2196254738Sbryanv ("%s: non start of frame w/o body buffer", __func__)); 2197320099Savg 2198320099Savg if (m_head == NULL && m_tail == NULL) { 2199320099Savg /* 2200320099Savg * This is a continuation of a packet that we 2201320099Savg * started to drop, but could not drop entirely 2202320099Savg * because this segment was still owned by the 2203320099Savg * host. So, drop the remainder now. 2204320099Savg */ 2205320099Savg vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2206320099Savg if (!rxcd->eop) 2207320099Savg vmxnet3_rxq_discard_chain(rxq); 2208320099Savg goto nextp; 2209320099Savg } 2210320099Savg 2211254738Sbryanv KASSERT(m_head != NULL, 2212254738Sbryanv ("%s: frame not started?", __func__)); 2213254738Sbryanv 2214254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2215263259Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2216254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2217254738Sbryanv if (!rxcd->eop) 2218254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2219254738Sbryanv m_freem(m_head); 2220254738Sbryanv m_head = m_tail = NULL; 2221254738Sbryanv goto nextp; 2222254738Sbryanv } 2223254738Sbryanv 2224254738Sbryanv m->m_len = length; 2225254738Sbryanv m_head->m_pkthdr.len += length; 2226254738Sbryanv m_tail->m_next = m; 2227254738Sbryanv m_tail = m; 2228254738Sbryanv } 2229254738Sbryanv 2230254738Sbryanv if (rxcd->eop) { 2231254738Sbryanv vmxnet3_rxq_input(rxq, rxcd, m_head); 2232254738Sbryanv m_head = m_tail = NULL; 2233254738Sbryanv 2234254738Sbryanv /* Must recheck after dropping the Rx lock. */ 2235254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2236254738Sbryanv break; 2237254738Sbryanv } 2238254738Sbryanv 2239254738Sbryanvnextp: 2240254738Sbryanv if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2241254738Sbryanv int qid = rxcd->qid; 2242254738Sbryanv bus_size_t r; 2243254738Sbryanv 2244254738Sbryanv idx = (idx + 1) % rxr->vxrxr_ndesc; 2245254738Sbryanv if (qid >= sc->vmx_nrxqueues) { 2246254738Sbryanv qid -= sc->vmx_nrxqueues; 2247254738Sbryanv r = VMXNET3_BAR0_RXH2(qid); 2248254738Sbryanv } else 2249254738Sbryanv r = VMXNET3_BAR0_RXH1(qid); 2250254738Sbryanv vmxnet3_write_bar0(sc, r, idx); 2251254738Sbryanv } 2252254738Sbryanv } 2253254738Sbryanv} 2254254738Sbryanv 2255254738Sbryanvstatic void 2256254738Sbryanvvmxnet3_legacy_intr(void *xsc) 2257254738Sbryanv{ 2258254738Sbryanv struct vmxnet3_softc *sc; 2259254738Sbryanv struct vmxnet3_rxqueue *rxq; 2260254738Sbryanv struct vmxnet3_txqueue *txq; 2261254738Sbryanv 2262254738Sbryanv sc = xsc; 2263254738Sbryanv rxq = &sc->vmx_rxq[0]; 2264254738Sbryanv txq = &sc->vmx_txq[0]; 2265254738Sbryanv 2266254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2267254738Sbryanv if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2268254738Sbryanv return; 2269254738Sbryanv } 2270254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2271254738Sbryanv vmxnet3_disable_all_intrs(sc); 2272254738Sbryanv 2273254738Sbryanv if (sc->vmx_ds->event != 0) 2274254738Sbryanv vmxnet3_evintr(sc); 2275254738Sbryanv 2276254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2277254738Sbryanv vmxnet3_rxq_eof(rxq); 2278254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2279254738Sbryanv 2280254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2281254738Sbryanv vmxnet3_txq_eof(txq); 2282263259Sbryanv vmxnet3_txq_start(txq); 2283254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2284254738Sbryanv 2285254738Sbryanv vmxnet3_enable_all_intrs(sc); 2286254738Sbryanv} 2287254738Sbryanv 2288254738Sbryanvstatic void 2289254738Sbryanvvmxnet3_txq_intr(void *xtxq) 2290254738Sbryanv{ 2291254738Sbryanv struct vmxnet3_softc *sc; 2292254738Sbryanv struct vmxnet3_txqueue *txq; 2293254738Sbryanv 2294254738Sbryanv txq = xtxq; 2295254738Sbryanv sc = txq->vxtxq_sc; 2296254738Sbryanv 2297254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2298254738Sbryanv vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2299254738Sbryanv 2300254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2301254738Sbryanv vmxnet3_txq_eof(txq); 2302263259Sbryanv vmxnet3_txq_start(txq); 2303254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2304254738Sbryanv 2305254738Sbryanv vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2306254738Sbryanv} 2307254738Sbryanv 2308254738Sbryanvstatic void 2309254738Sbryanvvmxnet3_rxq_intr(void *xrxq) 2310254738Sbryanv{ 2311254738Sbryanv struct vmxnet3_softc *sc; 2312254738Sbryanv struct vmxnet3_rxqueue *rxq; 2313254738Sbryanv 2314254738Sbryanv rxq = xrxq; 2315254738Sbryanv sc = rxq->vxrxq_sc; 2316254738Sbryanv 2317254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2318254738Sbryanv vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2319254738Sbryanv 2320254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2321254738Sbryanv vmxnet3_rxq_eof(rxq); 2322254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2323254738Sbryanv 2324254738Sbryanv vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2325254738Sbryanv} 2326254738Sbryanv 2327254738Sbryanvstatic void 2328254738Sbryanvvmxnet3_event_intr(void *xsc) 2329254738Sbryanv{ 2330254738Sbryanv struct vmxnet3_softc *sc; 2331254738Sbryanv 2332254738Sbryanv sc = xsc; 2333254738Sbryanv 2334254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2335254738Sbryanv vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2336254738Sbryanv 2337254738Sbryanv if (sc->vmx_ds->event != 0) 2338254738Sbryanv vmxnet3_evintr(sc); 2339254738Sbryanv 2340254738Sbryanv vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2341254738Sbryanv} 2342254738Sbryanv 2343254738Sbryanvstatic void 2344254738Sbryanvvmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2345254738Sbryanv{ 2346254738Sbryanv struct vmxnet3_txring *txr; 2347254738Sbryanv struct vmxnet3_txbuf *txb; 2348254738Sbryanv int i; 2349254738Sbryanv 2350254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2351254738Sbryanv 2352254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 2353254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 2354254738Sbryanv 2355254738Sbryanv if (txb->vtxb_m == NULL) 2356254738Sbryanv continue; 2357254738Sbryanv 2358254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2359254738Sbryanv BUS_DMASYNC_POSTWRITE); 2360254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2361254738Sbryanv m_freem(txb->vtxb_m); 2362254738Sbryanv txb->vtxb_m = NULL; 2363254738Sbryanv } 2364254738Sbryanv} 2365254738Sbryanv 2366254738Sbryanvstatic void 2367254738Sbryanvvmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2368254738Sbryanv{ 2369254738Sbryanv struct vmxnet3_rxring *rxr; 2370254738Sbryanv struct vmxnet3_rxbuf *rxb; 2371254738Sbryanv int i, j; 2372254738Sbryanv 2373267662Sbryanv if (rxq->vxrxq_mhead != NULL) { 2374267662Sbryanv m_freem(rxq->vxrxq_mhead); 2375267662Sbryanv rxq->vxrxq_mhead = NULL; 2376267662Sbryanv rxq->vxrxq_mtail = NULL; 2377267662Sbryanv } 2378267662Sbryanv 2379254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2380254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2381254738Sbryanv 2382254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2383254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 2384254738Sbryanv 2385254738Sbryanv if (rxb->vrxb_m == NULL) 2386254738Sbryanv continue; 2387263259Sbryanv 2388254738Sbryanv bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2389254738Sbryanv BUS_DMASYNC_POSTREAD); 2390254738Sbryanv bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2391254738Sbryanv m_freem(rxb->vrxb_m); 2392254738Sbryanv rxb->vrxb_m = NULL; 2393254738Sbryanv } 2394254738Sbryanv } 2395254738Sbryanv} 2396254738Sbryanv 2397254738Sbryanvstatic void 2398254738Sbryanvvmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2399254738Sbryanv{ 2400254738Sbryanv struct vmxnet3_rxqueue *rxq; 2401254738Sbryanv struct vmxnet3_txqueue *txq; 2402254738Sbryanv int i; 2403254738Sbryanv 2404254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 2405254738Sbryanv rxq = &sc->vmx_rxq[i]; 2406254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2407254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2408254738Sbryanv } 2409254738Sbryanv 2410254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 2411254738Sbryanv txq = &sc->vmx_txq[i]; 2412254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2413254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2414254738Sbryanv } 2415254738Sbryanv} 2416254738Sbryanv 2417254738Sbryanvstatic void 2418254738Sbryanvvmxnet3_stop(struct vmxnet3_softc *sc) 2419254738Sbryanv{ 2420254738Sbryanv struct ifnet *ifp; 2421254738Sbryanv int q; 2422254738Sbryanv 2423254738Sbryanv ifp = sc->vmx_ifp; 2424254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 2425254738Sbryanv 2426254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2427254738Sbryanv sc->vmx_link_active = 0; 2428254738Sbryanv callout_stop(&sc->vmx_tick); 2429254738Sbryanv 2430254738Sbryanv /* Disable interrupts. */ 2431254738Sbryanv vmxnet3_disable_all_intrs(sc); 2432254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2433254738Sbryanv 2434254738Sbryanv vmxnet3_stop_rendezvous(sc); 2435254738Sbryanv 2436254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2437254738Sbryanv vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2438254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) 2439254738Sbryanv vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2440254738Sbryanv 2441254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2442254738Sbryanv} 2443254738Sbryanv 2444254738Sbryanvstatic void 2445254738Sbryanvvmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2446254738Sbryanv{ 2447254738Sbryanv struct vmxnet3_txring *txr; 2448254738Sbryanv struct vmxnet3_comp_ring *txc; 2449254738Sbryanv 2450254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2451254738Sbryanv txr->vxtxr_head = 0; 2452254738Sbryanv txr->vxtxr_next = 0; 2453254738Sbryanv txr->vxtxr_gen = VMXNET3_INIT_GEN; 2454254738Sbryanv bzero(txr->vxtxr_txd, 2455254738Sbryanv txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2456254738Sbryanv 2457254738Sbryanv txc = &txq->vxtxq_comp_ring; 2458254738Sbryanv txc->vxcr_next = 0; 2459254738Sbryanv txc->vxcr_gen = VMXNET3_INIT_GEN; 2460254738Sbryanv bzero(txc->vxcr_u.txcd, 2461254738Sbryanv txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2462254738Sbryanv} 2463254738Sbryanv 2464254738Sbryanvstatic int 2465254738Sbryanvvmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2466254738Sbryanv{ 2467254738Sbryanv struct ifnet *ifp; 2468254738Sbryanv struct vmxnet3_rxring *rxr; 2469254738Sbryanv struct vmxnet3_comp_ring *rxc; 2470254738Sbryanv int i, populate, idx, frame_size, error; 2471254738Sbryanv 2472254738Sbryanv ifp = sc->vmx_ifp; 2473254950Sbryanv frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2474254950Sbryanv ifp->if_mtu; 2475254738Sbryanv 2476254738Sbryanv /* 2477254950Sbryanv * If the MTU causes us to exceed what a regular sized cluster can 2478254950Sbryanv * handle, we allocate a second MJUMPAGESIZE cluster after it in 2479254950Sbryanv * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2480254738Sbryanv * 2481254950Sbryanv * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2482254950Sbryanv * our life easier. We do not support changing the ring size after 2483254950Sbryanv * the attach. 2484254738Sbryanv */ 2485254950Sbryanv if (frame_size <= MCLBYTES) 2486254738Sbryanv sc->vmx_rx_max_chain = 1; 2487254738Sbryanv else 2488254738Sbryanv sc->vmx_rx_max_chain = 2; 2489254738Sbryanv 2490254738Sbryanv /* 2491254738Sbryanv * Only populate ring 1 if the configuration will take advantage 2492254738Sbryanv * of it. That is either when LRO is enabled or the frame size 2493254738Sbryanv * exceeds what ring 0 can contain. 2494254738Sbryanv */ 2495254738Sbryanv if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2496254738Sbryanv frame_size <= MCLBYTES + MJUMPAGESIZE) 2497254738Sbryanv populate = 1; 2498254738Sbryanv else 2499254738Sbryanv populate = VMXNET3_RXRINGS_PERQ; 2500254738Sbryanv 2501254738Sbryanv for (i = 0; i < populate; i++) { 2502254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2503254738Sbryanv rxr->vxrxr_fill = 0; 2504254738Sbryanv rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2505254738Sbryanv bzero(rxr->vxrxr_rxd, 2506254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2507254738Sbryanv 2508254738Sbryanv for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2509254738Sbryanv error = vmxnet3_newbuf(sc, rxr); 2510254738Sbryanv if (error) 2511254738Sbryanv return (error); 2512254738Sbryanv } 2513254738Sbryanv } 2514254738Sbryanv 2515254738Sbryanv for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2516254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2517254738Sbryanv rxr->vxrxr_fill = 0; 2518254738Sbryanv rxr->vxrxr_gen = 0; 2519254738Sbryanv bzero(rxr->vxrxr_rxd, 2520254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2521254738Sbryanv } 2522254738Sbryanv 2523254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2524254738Sbryanv rxc->vxcr_next = 0; 2525254738Sbryanv rxc->vxcr_gen = VMXNET3_INIT_GEN; 2526254738Sbryanv bzero(rxc->vxcr_u.rxcd, 2527254738Sbryanv rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2528254738Sbryanv 2529254738Sbryanv return (0); 2530254738Sbryanv} 2531254738Sbryanv 2532254738Sbryanvstatic int 2533254738Sbryanvvmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2534254738Sbryanv{ 2535254738Sbryanv device_t dev; 2536254738Sbryanv int q, error; 2537254738Sbryanv 2538254738Sbryanv dev = sc->vmx_dev; 2539254738Sbryanv 2540254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2541254738Sbryanv vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2542254738Sbryanv 2543254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2544254738Sbryanv error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2545254738Sbryanv if (error) { 2546254738Sbryanv device_printf(dev, "cannot populate Rx queue %d\n", q); 2547254738Sbryanv return (error); 2548254738Sbryanv } 2549254738Sbryanv } 2550254738Sbryanv 2551254738Sbryanv return (0); 2552254738Sbryanv} 2553254738Sbryanv 2554254738Sbryanvstatic int 2555254738Sbryanvvmxnet3_enable_device(struct vmxnet3_softc *sc) 2556254738Sbryanv{ 2557254738Sbryanv int q; 2558254738Sbryanv 2559254738Sbryanv if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2560254738Sbryanv device_printf(sc->vmx_dev, "device enable command failed!\n"); 2561254738Sbryanv return (1); 2562254738Sbryanv } 2563254738Sbryanv 2564254738Sbryanv /* Reset the Rx queue heads. */ 2565254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2566254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2567254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2568254738Sbryanv } 2569254738Sbryanv 2570254738Sbryanv return (0); 2571254738Sbryanv} 2572254738Sbryanv 2573254738Sbryanvstatic void 2574254738Sbryanvvmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2575254738Sbryanv{ 2576254738Sbryanv struct ifnet *ifp; 2577254738Sbryanv 2578254738Sbryanv ifp = sc->vmx_ifp; 2579254738Sbryanv 2580254738Sbryanv vmxnet3_set_rxfilter(sc); 2581254738Sbryanv 2582254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2583254738Sbryanv bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2584254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2585254738Sbryanv else 2586254738Sbryanv bzero(sc->vmx_ds->vlan_filter, 2587254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2588254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2589254738Sbryanv} 2590254738Sbryanv 2591254738Sbryanvstatic int 2592254738Sbryanvvmxnet3_reinit(struct vmxnet3_softc *sc) 2593254738Sbryanv{ 2594254738Sbryanv 2595254738Sbryanv vmxnet3_reinit_interface(sc); 2596254738Sbryanv vmxnet3_reinit_shared_data(sc); 2597254738Sbryanv 2598254738Sbryanv if (vmxnet3_reinit_queues(sc) != 0) 2599254738Sbryanv return (ENXIO); 2600254738Sbryanv 2601254738Sbryanv if (vmxnet3_enable_device(sc) != 0) 2602254738Sbryanv return (ENXIO); 2603254738Sbryanv 2604254738Sbryanv vmxnet3_reinit_rxfilters(sc); 2605254738Sbryanv 2606254738Sbryanv return (0); 2607254738Sbryanv} 2608254738Sbryanv 2609254738Sbryanvstatic void 2610254738Sbryanvvmxnet3_init_locked(struct vmxnet3_softc *sc) 2611254738Sbryanv{ 2612254738Sbryanv struct ifnet *ifp; 2613254738Sbryanv 2614254738Sbryanv ifp = sc->vmx_ifp; 2615254738Sbryanv 2616254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2617254738Sbryanv return; 2618254738Sbryanv 2619254738Sbryanv vmxnet3_stop(sc); 2620254738Sbryanv 2621254738Sbryanv if (vmxnet3_reinit(sc) != 0) { 2622254738Sbryanv vmxnet3_stop(sc); 2623254738Sbryanv return; 2624254738Sbryanv } 2625254738Sbryanv 2626254738Sbryanv ifp->if_drv_flags |= IFF_DRV_RUNNING; 2627254738Sbryanv vmxnet3_link_status(sc); 2628254738Sbryanv 2629254738Sbryanv vmxnet3_enable_all_intrs(sc); 2630254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2631254738Sbryanv} 2632254738Sbryanv 2633254738Sbryanvstatic void 2634254738Sbryanvvmxnet3_init(void *xsc) 2635254738Sbryanv{ 2636254738Sbryanv struct vmxnet3_softc *sc; 2637254738Sbryanv 2638254738Sbryanv sc = xsc; 2639254738Sbryanv 2640254738Sbryanv VMXNET3_CORE_LOCK(sc); 2641254738Sbryanv vmxnet3_init_locked(sc); 2642254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 2643254738Sbryanv} 2644254738Sbryanv 2645254738Sbryanv/* 2646254738Sbryanv * BMV: Much of this can go away once we finally have offsets in 2647254738Sbryanv * the mbuf packet header. Bug andre@. 2648254738Sbryanv */ 2649254738Sbryanvstatic int 2650263259Sbryanvvmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2651263259Sbryanv int *etype, int *proto, int *start) 2652254738Sbryanv{ 2653254738Sbryanv struct ether_vlan_header *evh; 2654254738Sbryanv int offset; 2655267253Sbryanv#if defined(INET) 2656267632Shselasky struct ip *ip = NULL; 2657267632Shselasky struct ip iphdr; 2658267253Sbryanv#endif 2659267253Sbryanv#if defined(INET6) 2660267632Shselasky struct ip6_hdr *ip6 = NULL; 2661267632Shselasky struct ip6_hdr ip6hdr; 2662267253Sbryanv#endif 2663254738Sbryanv 2664254738Sbryanv evh = mtod(m, struct ether_vlan_header *); 2665254738Sbryanv if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2666254738Sbryanv /* BMV: We should handle nested VLAN tags too. */ 2667254738Sbryanv *etype = ntohs(evh->evl_proto); 2668254738Sbryanv offset = sizeof(struct ether_vlan_header); 2669254738Sbryanv } else { 2670254738Sbryanv *etype = ntohs(evh->evl_encap_proto); 2671254738Sbryanv offset = sizeof(struct ether_header); 2672254738Sbryanv } 2673254738Sbryanv 2674254738Sbryanv switch (*etype) { 2675254738Sbryanv#if defined(INET) 2676267253Sbryanv case ETHERTYPE_IP: 2677254738Sbryanv if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2678254738Sbryanv m_copydata(m, offset, sizeof(struct ip), 2679254738Sbryanv (caddr_t) &iphdr); 2680254738Sbryanv ip = &iphdr; 2681254738Sbryanv } else 2682263259Sbryanv ip = mtodo(m, offset); 2683254738Sbryanv *proto = ip->ip_p; 2684254738Sbryanv *start = offset + (ip->ip_hl << 2); 2685254738Sbryanv break; 2686254738Sbryanv#endif 2687254738Sbryanv#if defined(INET6) 2688254738Sbryanv case ETHERTYPE_IPV6: 2689267253Sbryanv if (__predict_false(m->m_len < 2690267253Sbryanv offset + sizeof(struct ip6_hdr))) { 2691267253Sbryanv m_copydata(m, offset, sizeof(struct ip6_hdr), 2692267253Sbryanv (caddr_t) &ip6hdr); 2693267253Sbryanv ip6 = &ip6hdr; 2694267253Sbryanv } else 2695267253Sbryanv ip6 = mtodo(m, offset); 2696254738Sbryanv *proto = -1; 2697254738Sbryanv *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2698254738Sbryanv /* Assert the network stack sent us a valid packet. */ 2699254738Sbryanv KASSERT(*start > offset, 2700254738Sbryanv ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2701254738Sbryanv *start, offset, *proto)); 2702254738Sbryanv break; 2703254738Sbryanv#endif 2704254738Sbryanv default: 2705254738Sbryanv return (EINVAL); 2706254738Sbryanv } 2707254738Sbryanv 2708254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2709254738Sbryanv struct tcphdr *tcp, tcphdr; 2710267253Sbryanv uint16_t sum; 2711254738Sbryanv 2712254738Sbryanv if (__predict_false(*proto != IPPROTO_TCP)) { 2713254738Sbryanv /* Likely failed to correctly parse the mbuf. */ 2714254738Sbryanv return (EINVAL); 2715254738Sbryanv } 2716254738Sbryanv 2717263259Sbryanv txq->vxtxq_stats.vmtxs_tso++; 2718263259Sbryanv 2719267253Sbryanv switch (*etype) { 2720267253Sbryanv#if defined(INET) 2721267253Sbryanv case ETHERTYPE_IP: 2722267253Sbryanv sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2723267253Sbryanv htons(IPPROTO_TCP)); 2724267253Sbryanv break; 2725267253Sbryanv#endif 2726267253Sbryanv#if defined(INET6) 2727267253Sbryanv case ETHERTYPE_IPV6: 2728267253Sbryanv sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2729267253Sbryanv break; 2730267253Sbryanv#endif 2731267253Sbryanv default: 2732267253Sbryanv sum = 0; 2733267253Sbryanv break; 2734267253Sbryanv } 2735267253Sbryanv 2736267253Sbryanv if (m->m_len < *start + sizeof(struct tcphdr)) { 2737267253Sbryanv m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2738267253Sbryanv sizeof(uint16_t), (caddr_t) &sum); 2739267253Sbryanv m_copydata(m, *start, sizeof(struct tcphdr), 2740267253Sbryanv (caddr_t) &tcphdr); 2741267253Sbryanv tcp = &tcphdr; 2742267253Sbryanv } else { 2743267253Sbryanv tcp = mtodo(m, *start); 2744267253Sbryanv tcp->th_sum = sum; 2745267253Sbryanv } 2746267253Sbryanv 2747263259Sbryanv /* 2748263259Sbryanv * For TSO, the size of the protocol header is also 2749263259Sbryanv * included in the descriptor header size. 2750263259Sbryanv */ 2751254738Sbryanv *start += (tcp->th_off << 2); 2752263259Sbryanv } else 2753263259Sbryanv txq->vxtxq_stats.vmtxs_csum++; 2754254738Sbryanv 2755254738Sbryanv return (0); 2756254738Sbryanv} 2757254738Sbryanv 2758254738Sbryanvstatic int 2759254738Sbryanvvmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2760254738Sbryanv bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2761254738Sbryanv{ 2762254738Sbryanv struct vmxnet3_txring *txr; 2763254738Sbryanv struct mbuf *m; 2764254738Sbryanv bus_dma_tag_t tag; 2765263259Sbryanv int error; 2766254738Sbryanv 2767254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2768254738Sbryanv m = *m0; 2769254738Sbryanv tag = txr->vxtxr_txtag; 2770254738Sbryanv 2771254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2772254738Sbryanv if (error == 0 || error != EFBIG) 2773254738Sbryanv return (error); 2774254738Sbryanv 2775263259Sbryanv m = m_defrag(m, M_NOWAIT); 2776254738Sbryanv if (m != NULL) { 2777254738Sbryanv *m0 = m; 2778254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2779254738Sbryanv } else 2780254738Sbryanv error = ENOBUFS; 2781254738Sbryanv 2782254738Sbryanv if (error) { 2783254738Sbryanv m_freem(*m0); 2784254738Sbryanv *m0 = NULL; 2785263259Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2786254738Sbryanv } else 2787263259Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2788254738Sbryanv 2789254738Sbryanv return (error); 2790254738Sbryanv} 2791254738Sbryanv 2792254738Sbryanvstatic void 2793254738Sbryanvvmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2794254738Sbryanv{ 2795254738Sbryanv struct vmxnet3_txring *txr; 2796254738Sbryanv 2797254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2798254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2799254738Sbryanv} 2800254738Sbryanv 2801254738Sbryanvstatic int 2802254738Sbryanvvmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2803254738Sbryanv{ 2804254738Sbryanv struct vmxnet3_softc *sc; 2805254738Sbryanv struct vmxnet3_txring *txr; 2806254738Sbryanv struct vmxnet3_txdesc *txd, *sop; 2807254738Sbryanv struct mbuf *m; 2808254738Sbryanv bus_dmamap_t dmap; 2809254738Sbryanv bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2810254738Sbryanv int i, gen, nsegs, etype, proto, start, error; 2811254738Sbryanv 2812254738Sbryanv sc = txq->vxtxq_sc; 2813254738Sbryanv start = 0; 2814254738Sbryanv txd = NULL; 2815254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2816254738Sbryanv dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2817254738Sbryanv 2818254738Sbryanv error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2819254738Sbryanv if (error) 2820254738Sbryanv return (error); 2821254738Sbryanv 2822254738Sbryanv m = *m0; 2823254738Sbryanv M_ASSERTPKTHDR(m); 2824254738Sbryanv KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2825254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2826254738Sbryanv 2827254738Sbryanv if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2828263259Sbryanv txq->vxtxq_stats.vmtxs_full++; 2829254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2830254738Sbryanv return (ENOSPC); 2831254738Sbryanv } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2832263259Sbryanv error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2833254738Sbryanv if (error) { 2834263259Sbryanv txq->vxtxq_stats.vmtxs_offload_failed++; 2835254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2836254738Sbryanv m_freem(m); 2837254738Sbryanv *m0 = NULL; 2838254738Sbryanv return (error); 2839254738Sbryanv } 2840254738Sbryanv } 2841254738Sbryanv 2842267252Sbryanv txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2843254738Sbryanv sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2844254738Sbryanv gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2845254738Sbryanv 2846254738Sbryanv for (i = 0; i < nsegs; i++) { 2847254738Sbryanv txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2848254738Sbryanv 2849254738Sbryanv txd->addr = segs[i].ds_addr; 2850254738Sbryanv txd->len = segs[i].ds_len; 2851254738Sbryanv txd->gen = gen; 2852254738Sbryanv txd->dtype = 0; 2853254738Sbryanv txd->offload_mode = VMXNET3_OM_NONE; 2854254738Sbryanv txd->offload_pos = 0; 2855254738Sbryanv txd->hlen = 0; 2856254738Sbryanv txd->eop = 0; 2857254738Sbryanv txd->compreq = 0; 2858254738Sbryanv txd->vtag_mode = 0; 2859254738Sbryanv txd->vtag = 0; 2860254738Sbryanv 2861254738Sbryanv if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2862254738Sbryanv txr->vxtxr_head = 0; 2863254738Sbryanv txr->vxtxr_gen ^= 1; 2864254738Sbryanv } 2865254738Sbryanv gen = txr->vxtxr_gen; 2866254738Sbryanv } 2867254738Sbryanv txd->eop = 1; 2868254738Sbryanv txd->compreq = 1; 2869254738Sbryanv 2870254738Sbryanv if (m->m_flags & M_VLANTAG) { 2871254738Sbryanv sop->vtag_mode = 1; 2872254738Sbryanv sop->vtag = m->m_pkthdr.ether_vtag; 2873254738Sbryanv } 2874254738Sbryanv 2875254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2876254738Sbryanv sop->offload_mode = VMXNET3_OM_TSO; 2877254738Sbryanv sop->hlen = start; 2878254738Sbryanv sop->offload_pos = m->m_pkthdr.tso_segsz; 2879254738Sbryanv } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2880254738Sbryanv VMXNET3_CSUM_OFFLOAD_IPV6)) { 2881254738Sbryanv sop->offload_mode = VMXNET3_OM_CSUM; 2882254738Sbryanv sop->hlen = start; 2883254738Sbryanv sop->offload_pos = start + m->m_pkthdr.csum_data; 2884254738Sbryanv } 2885254738Sbryanv 2886254738Sbryanv /* Finally, change the ownership. */ 2887254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2888254738Sbryanv sop->gen ^= 1; 2889254738Sbryanv 2890267663Sbryanv txq->vxtxq_ts->npending += nsegs; 2891267663Sbryanv if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2892254738Sbryanv txq->vxtxq_ts->npending = 0; 2893254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2894254738Sbryanv txr->vxtxr_head); 2895254738Sbryanv } 2896254738Sbryanv 2897254738Sbryanv return (0); 2898254738Sbryanv} 2899254738Sbryanv 2900263259Sbryanv#ifdef VMXNET3_LEGACY_TX 2901263259Sbryanv 2902263259Sbryanvstatic void 2903254738Sbryanvvmxnet3_start_locked(struct ifnet *ifp) 2904254738Sbryanv{ 2905254738Sbryanv struct vmxnet3_softc *sc; 2906254738Sbryanv struct vmxnet3_txqueue *txq; 2907254738Sbryanv struct vmxnet3_txring *txr; 2908254738Sbryanv struct mbuf *m_head; 2909255055Sbryanv int tx, avail; 2910254738Sbryanv 2911254738Sbryanv sc = ifp->if_softc; 2912254738Sbryanv txq = &sc->vmx_txq[0]; 2913254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2914254738Sbryanv tx = 0; 2915254738Sbryanv 2916254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 2917254738Sbryanv 2918254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2919254738Sbryanv sc->vmx_link_active == 0) 2920254738Sbryanv return; 2921254738Sbryanv 2922255055Sbryanv while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2923255055Sbryanv if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2924255055Sbryanv break; 2925255055Sbryanv 2926254738Sbryanv IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2927254738Sbryanv if (m_head == NULL) 2928254738Sbryanv break; 2929254738Sbryanv 2930255055Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 2931255055Sbryanv if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2932255055Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2933255055Sbryanv break; 2934255055Sbryanv } 2935255055Sbryanv 2936254738Sbryanv if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2937254738Sbryanv if (m_head != NULL) 2938254738Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2939254738Sbryanv break; 2940254738Sbryanv } 2941254738Sbryanv 2942254738Sbryanv tx++; 2943254738Sbryanv ETHER_BPF_MTAP(ifp, m_head); 2944254738Sbryanv } 2945254738Sbryanv 2946267661Sbryanv if (tx > 0) 2947254738Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2948254738Sbryanv} 2949254738Sbryanv 2950254738Sbryanvstatic void 2951254738Sbryanvvmxnet3_start(struct ifnet *ifp) 2952254738Sbryanv{ 2953254738Sbryanv struct vmxnet3_softc *sc; 2954254738Sbryanv struct vmxnet3_txqueue *txq; 2955254738Sbryanv 2956254738Sbryanv sc = ifp->if_softc; 2957254738Sbryanv txq = &sc->vmx_txq[0]; 2958254738Sbryanv 2959254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2960254738Sbryanv vmxnet3_start_locked(ifp); 2961254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2962254738Sbryanv} 2963254738Sbryanv 2964263259Sbryanv#else /* !VMXNET3_LEGACY_TX */ 2965263259Sbryanv 2966263259Sbryanvstatic int 2967263259Sbryanvvmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2968263259Sbryanv{ 2969263259Sbryanv struct vmxnet3_softc *sc; 2970263259Sbryanv struct vmxnet3_txring *txr; 2971263259Sbryanv struct buf_ring *br; 2972263259Sbryanv struct ifnet *ifp; 2973263259Sbryanv int tx, avail, error; 2974263259Sbryanv 2975263259Sbryanv sc = txq->vxtxq_sc; 2976263259Sbryanv br = txq->vxtxq_br; 2977263259Sbryanv ifp = sc->vmx_ifp; 2978263259Sbryanv txr = &txq->vxtxq_cmd_ring; 2979263259Sbryanv tx = 0; 2980263259Sbryanv error = 0; 2981263259Sbryanv 2982263259Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 2983263259Sbryanv 2984263259Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2985263259Sbryanv sc->vmx_link_active == 0) { 2986263259Sbryanv if (m != NULL) 2987263259Sbryanv error = drbr_enqueue(ifp, br, m); 2988263259Sbryanv return (error); 2989263259Sbryanv } 2990263259Sbryanv 2991263259Sbryanv if (m != NULL) { 2992263259Sbryanv error = drbr_enqueue(ifp, br, m); 2993263259Sbryanv if (error) 2994263259Sbryanv return (error); 2995263259Sbryanv } 2996263259Sbryanv 2997263259Sbryanv while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2998263259Sbryanv m = drbr_peek(ifp, br); 2999263259Sbryanv if (m == NULL) 3000263259Sbryanv break; 3001263259Sbryanv 3002263259Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 3003263259Sbryanv if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 3004263259Sbryanv drbr_putback(ifp, br, m); 3005263259Sbryanv break; 3006263259Sbryanv } 3007263259Sbryanv 3008267663Sbryanv if (vmxnet3_txq_encap(txq, &m) != 0) { 3009263259Sbryanv if (m != NULL) 3010263259Sbryanv drbr_putback(ifp, br, m); 3011263259Sbryanv else 3012263259Sbryanv drbr_advance(ifp, br); 3013263259Sbryanv break; 3014263259Sbryanv } 3015263259Sbryanv drbr_advance(ifp, br); 3016263259Sbryanv 3017263259Sbryanv tx++; 3018263259Sbryanv ETHER_BPF_MTAP(ifp, m); 3019263259Sbryanv } 3020263259Sbryanv 3021267661Sbryanv if (tx > 0) 3022263259Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3023263259Sbryanv 3024267187Sluigi return (0); 3025263259Sbryanv} 3026263259Sbryanv 3027263259Sbryanvstatic int 3028263259Sbryanvvmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 3029263259Sbryanv{ 3030263259Sbryanv struct vmxnet3_softc *sc; 3031263259Sbryanv struct vmxnet3_txqueue *txq; 3032263259Sbryanv int i, ntxq, error; 3033263259Sbryanv 3034263259Sbryanv sc = ifp->if_softc; 3035263259Sbryanv ntxq = sc->vmx_ntxqueues; 3036263259Sbryanv 3037275358Shselasky /* check if flowid is set */ 3038275358Shselasky if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3039263259Sbryanv i = m->m_pkthdr.flowid % ntxq; 3040263259Sbryanv else 3041263259Sbryanv i = curcpu % ntxq; 3042263259Sbryanv 3043263259Sbryanv txq = &sc->vmx_txq[i]; 3044263259Sbryanv 3045263259Sbryanv if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3046263259Sbryanv error = vmxnet3_txq_mq_start_locked(txq, m); 3047263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3048263259Sbryanv } else { 3049263259Sbryanv error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3050263259Sbryanv taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3051263259Sbryanv } 3052263259Sbryanv 3053263259Sbryanv return (error); 3054263259Sbryanv} 3055263259Sbryanv 3056254738Sbryanvstatic void 3057263259Sbryanvvmxnet3_txq_tq_deferred(void *xtxq, int pending) 3058263259Sbryanv{ 3059263259Sbryanv struct vmxnet3_softc *sc; 3060263259Sbryanv struct vmxnet3_txqueue *txq; 3061263259Sbryanv 3062263259Sbryanv txq = xtxq; 3063263259Sbryanv sc = txq->vxtxq_sc; 3064263259Sbryanv 3065263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3066263259Sbryanv if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3067263259Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3068263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3069263259Sbryanv} 3070263259Sbryanv 3071263259Sbryanv#endif /* VMXNET3_LEGACY_TX */ 3072263259Sbryanv 3073263259Sbryanvstatic void 3074263259Sbryanvvmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3075263259Sbryanv{ 3076263259Sbryanv struct vmxnet3_softc *sc; 3077263259Sbryanv struct ifnet *ifp; 3078263259Sbryanv 3079263259Sbryanv sc = txq->vxtxq_sc; 3080263259Sbryanv ifp = sc->vmx_ifp; 3081263259Sbryanv 3082263259Sbryanv#ifdef VMXNET3_LEGACY_TX 3083263259Sbryanv if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3084263259Sbryanv vmxnet3_start_locked(ifp); 3085263259Sbryanv#else 3086263259Sbryanv if (!drbr_empty(ifp, txq->vxtxq_br)) 3087263259Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3088263259Sbryanv#endif 3089263259Sbryanv} 3090263259Sbryanv 3091263259Sbryanvstatic void 3092263259Sbryanvvmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3093263259Sbryanv{ 3094263259Sbryanv struct vmxnet3_txqueue *txq; 3095263259Sbryanv int i; 3096263259Sbryanv 3097263259Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3098263259Sbryanv 3099263259Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3100263259Sbryanv txq = &sc->vmx_txq[i]; 3101263259Sbryanv 3102263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3103263259Sbryanv vmxnet3_txq_start(txq); 3104263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3105263259Sbryanv } 3106263259Sbryanv} 3107263259Sbryanv 3108263259Sbryanvstatic void 3109254738Sbryanvvmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3110254738Sbryanv{ 3111254738Sbryanv struct ifnet *ifp; 3112254738Sbryanv int idx, bit; 3113254738Sbryanv 3114254738Sbryanv ifp = sc->vmx_ifp; 3115254738Sbryanv idx = (tag >> 5) & 0x7F; 3116254738Sbryanv bit = tag & 0x1F; 3117254738Sbryanv 3118254738Sbryanv if (tag == 0 || tag > 4095) 3119254738Sbryanv return; 3120254738Sbryanv 3121254738Sbryanv VMXNET3_CORE_LOCK(sc); 3122254738Sbryanv 3123254738Sbryanv /* Update our private VLAN bitvector. */ 3124254738Sbryanv if (add) 3125254738Sbryanv sc->vmx_vlan_filter[idx] |= (1 << bit); 3126254738Sbryanv else 3127254738Sbryanv sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3128254738Sbryanv 3129254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3130254738Sbryanv if (add) 3131254738Sbryanv sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3132254738Sbryanv else 3133254738Sbryanv sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3134254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3135254738Sbryanv } 3136254738Sbryanv 3137254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3138254738Sbryanv} 3139254738Sbryanv 3140254738Sbryanvstatic void 3141254738Sbryanvvmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3142254738Sbryanv{ 3143254738Sbryanv 3144254738Sbryanv if (ifp->if_softc == arg) 3145254738Sbryanv vmxnet3_update_vlan_filter(arg, 1, tag); 3146254738Sbryanv} 3147254738Sbryanv 3148254738Sbryanvstatic void 3149254738Sbryanvvmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3150254738Sbryanv{ 3151254738Sbryanv 3152254738Sbryanv if (ifp->if_softc == arg) 3153254738Sbryanv vmxnet3_update_vlan_filter(arg, 0, tag); 3154254738Sbryanv} 3155254738Sbryanv 3156254738Sbryanvstatic void 3157254738Sbryanvvmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3158254738Sbryanv{ 3159254738Sbryanv struct ifnet *ifp; 3160254738Sbryanv struct vmxnet3_driver_shared *ds; 3161254738Sbryanv struct ifmultiaddr *ifma; 3162254738Sbryanv u_int mode; 3163254738Sbryanv 3164254738Sbryanv ifp = sc->vmx_ifp; 3165254738Sbryanv ds = sc->vmx_ds; 3166254738Sbryanv 3167263259Sbryanv mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3168254738Sbryanv if (ifp->if_flags & IFF_PROMISC) 3169254738Sbryanv mode |= VMXNET3_RXMODE_PROMISC; 3170254738Sbryanv if (ifp->if_flags & IFF_ALLMULTI) 3171254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3172254738Sbryanv else { 3173254738Sbryanv int cnt = 0, overflow = 0; 3174254738Sbryanv 3175254738Sbryanv if_maddr_rlock(ifp); 3176254738Sbryanv TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3177254738Sbryanv if (ifma->ifma_addr->sa_family != AF_LINK) 3178254738Sbryanv continue; 3179254738Sbryanv else if (cnt == VMXNET3_MULTICAST_MAX) { 3180254738Sbryanv overflow = 1; 3181254738Sbryanv break; 3182254738Sbryanv } 3183254738Sbryanv 3184254738Sbryanv bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3185254738Sbryanv &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3186254738Sbryanv cnt++; 3187254738Sbryanv } 3188254738Sbryanv if_maddr_runlock(ifp); 3189254738Sbryanv 3190254738Sbryanv if (overflow != 0) { 3191254738Sbryanv cnt = 0; 3192254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3193254738Sbryanv } else if (cnt > 0) 3194254738Sbryanv mode |= VMXNET3_RXMODE_MCAST; 3195254738Sbryanv ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3196254738Sbryanv } 3197254738Sbryanv 3198254738Sbryanv ds->rxmode = mode; 3199254738Sbryanv 3200254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3201254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3202254738Sbryanv} 3203254738Sbryanv 3204254738Sbryanvstatic int 3205254738Sbryanvvmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3206254738Sbryanv{ 3207254738Sbryanv struct ifnet *ifp; 3208254738Sbryanv 3209254738Sbryanv ifp = sc->vmx_ifp; 3210254738Sbryanv 3211254738Sbryanv if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3212254738Sbryanv return (EINVAL); 3213254738Sbryanv 3214254738Sbryanv ifp->if_mtu = mtu; 3215254738Sbryanv 3216254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3217254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3218254738Sbryanv vmxnet3_init_locked(sc); 3219254738Sbryanv } 3220254738Sbryanv 3221254738Sbryanv return (0); 3222254738Sbryanv} 3223254738Sbryanv 3224254738Sbryanvstatic int 3225254738Sbryanvvmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3226254738Sbryanv{ 3227254738Sbryanv struct vmxnet3_softc *sc; 3228254738Sbryanv struct ifreq *ifr; 3229254738Sbryanv int reinit, mask, error; 3230254738Sbryanv 3231254738Sbryanv sc = ifp->if_softc; 3232254738Sbryanv ifr = (struct ifreq *) data; 3233254738Sbryanv error = 0; 3234254738Sbryanv 3235254738Sbryanv switch (cmd) { 3236254738Sbryanv case SIOCSIFMTU: 3237254738Sbryanv if (ifp->if_mtu != ifr->ifr_mtu) { 3238254738Sbryanv VMXNET3_CORE_LOCK(sc); 3239254738Sbryanv error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3240254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3241254738Sbryanv } 3242254738Sbryanv break; 3243254738Sbryanv 3244254738Sbryanv case SIOCSIFFLAGS: 3245254738Sbryanv VMXNET3_CORE_LOCK(sc); 3246254738Sbryanv if (ifp->if_flags & IFF_UP) { 3247254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3248254738Sbryanv if ((ifp->if_flags ^ sc->vmx_if_flags) & 3249254738Sbryanv (IFF_PROMISC | IFF_ALLMULTI)) { 3250254738Sbryanv vmxnet3_set_rxfilter(sc); 3251254738Sbryanv } 3252254738Sbryanv } else 3253254738Sbryanv vmxnet3_init_locked(sc); 3254254738Sbryanv } else { 3255254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3256254738Sbryanv vmxnet3_stop(sc); 3257254738Sbryanv } 3258254738Sbryanv sc->vmx_if_flags = ifp->if_flags; 3259254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3260254738Sbryanv break; 3261254738Sbryanv 3262254738Sbryanv case SIOCADDMULTI: 3263254738Sbryanv case SIOCDELMULTI: 3264254738Sbryanv VMXNET3_CORE_LOCK(sc); 3265254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3266254738Sbryanv vmxnet3_set_rxfilter(sc); 3267254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3268254738Sbryanv break; 3269254738Sbryanv 3270254738Sbryanv case SIOCSIFMEDIA: 3271254738Sbryanv case SIOCGIFMEDIA: 3272254738Sbryanv error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3273254738Sbryanv break; 3274254738Sbryanv 3275254738Sbryanv case SIOCSIFCAP: 3276254738Sbryanv VMXNET3_CORE_LOCK(sc); 3277254738Sbryanv mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3278254738Sbryanv 3279254738Sbryanv if (mask & IFCAP_TXCSUM) 3280254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM; 3281254738Sbryanv if (mask & IFCAP_TXCSUM_IPV6) 3282254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3283254738Sbryanv if (mask & IFCAP_TSO4) 3284254738Sbryanv ifp->if_capenable ^= IFCAP_TSO4; 3285254738Sbryanv if (mask & IFCAP_TSO6) 3286254738Sbryanv ifp->if_capenable ^= IFCAP_TSO6; 3287254738Sbryanv 3288254738Sbryanv if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3289255055Sbryanv IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3290255055Sbryanv /* Changing these features requires us to reinit. */ 3291254738Sbryanv reinit = 1; 3292254738Sbryanv 3293254738Sbryanv if (mask & IFCAP_RXCSUM) 3294254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM; 3295254738Sbryanv if (mask & IFCAP_RXCSUM_IPV6) 3296254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3297254738Sbryanv if (mask & IFCAP_LRO) 3298254738Sbryanv ifp->if_capenable ^= IFCAP_LRO; 3299255055Sbryanv if (mask & IFCAP_VLAN_HWTAGGING) 3300255055Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3301254738Sbryanv if (mask & IFCAP_VLAN_HWFILTER) 3302254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3303254738Sbryanv } else 3304254738Sbryanv reinit = 0; 3305254738Sbryanv 3306254738Sbryanv if (mask & IFCAP_VLAN_HWTSO) 3307254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3308254738Sbryanv 3309254738Sbryanv if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3310254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3311254738Sbryanv vmxnet3_init_locked(sc); 3312303136Smav } else { 3313303136Smav vmxnet3_init_hwassist(sc); 3314254738Sbryanv } 3315254738Sbryanv 3316254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3317254738Sbryanv VLAN_CAPABILITIES(ifp); 3318254738Sbryanv break; 3319254738Sbryanv 3320254738Sbryanv default: 3321254738Sbryanv error = ether_ioctl(ifp, cmd, data); 3322254738Sbryanv break; 3323254738Sbryanv } 3324254738Sbryanv 3325254738Sbryanv VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3326254738Sbryanv 3327254738Sbryanv return (error); 3328254738Sbryanv} 3329254738Sbryanv 3330263259Sbryanv#ifndef VMXNET3_LEGACY_TX 3331263259Sbryanvstatic void 3332263259Sbryanvvmxnet3_qflush(struct ifnet *ifp) 3333263259Sbryanv{ 3334263259Sbryanv struct vmxnet3_softc *sc; 3335263259Sbryanv struct vmxnet3_txqueue *txq; 3336263259Sbryanv struct mbuf *m; 3337263259Sbryanv int i; 3338263259Sbryanv 3339263259Sbryanv sc = ifp->if_softc; 3340263259Sbryanv 3341263259Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3342263259Sbryanv txq = &sc->vmx_txq[i]; 3343263259Sbryanv 3344263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3345263259Sbryanv while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3346263259Sbryanv m_freem(m); 3347263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3348263259Sbryanv } 3349263259Sbryanv 3350263259Sbryanv if_qflush(ifp); 3351263259Sbryanv} 3352263259Sbryanv#endif 3353263259Sbryanv 3354254738Sbryanvstatic int 3355254738Sbryanvvmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3356254738Sbryanv{ 3357254738Sbryanv struct vmxnet3_softc *sc; 3358254738Sbryanv 3359254738Sbryanv sc = txq->vxtxq_sc; 3360254738Sbryanv 3361254738Sbryanv VMXNET3_TXQ_LOCK(txq); 3362254738Sbryanv if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3363254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3364254738Sbryanv return (0); 3365254738Sbryanv } 3366254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3367254738Sbryanv 3368254738Sbryanv if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3369254738Sbryanv txq->vxtxq_id); 3370254738Sbryanv return (1); 3371254738Sbryanv} 3372254738Sbryanv 3373254738Sbryanvstatic void 3374263259Sbryanvvmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3375254738Sbryanv{ 3376254738Sbryanv 3377254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3378254738Sbryanv} 3379254738Sbryanv 3380272099Sglebiusstatic uint64_t 3381272099Sglebiusvmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt) 3382263259Sbryanv{ 3383272099Sglebius struct vmxnet3_softc *sc; 3384272099Sglebius uint64_t rv; 3385263259Sbryanv 3386272099Sglebius sc = if_getsoftc(ifp); 3387272099Sglebius rv = 0; 3388263259Sbryanv 3389263259Sbryanv /* 3390263259Sbryanv * With the exception of if_ierrors, these ifnet statistics are 3391263259Sbryanv * only updated in the driver, so just set them to our accumulated 3392263259Sbryanv * values. if_ierrors is updated in ether_input() for malformed 3393263259Sbryanv * frames that we should have already discarded. 3394263259Sbryanv */ 3395272099Sglebius switch (cnt) { 3396272099Sglebius case IFCOUNTER_IPACKETS: 3397272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3398272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets; 3399272099Sglebius return (rv); 3400272099Sglebius case IFCOUNTER_IQDROPS: 3401272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3402272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops; 3403272099Sglebius return (rv); 3404272099Sglebius case IFCOUNTER_IERRORS: 3405272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3406272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors; 3407272099Sglebius return (rv); 3408272099Sglebius case IFCOUNTER_OPACKETS: 3409272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3410272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets; 3411272099Sglebius return (rv); 3412263259Sbryanv#ifndef VMXNET3_LEGACY_TX 3413272099Sglebius case IFCOUNTER_OBYTES: 3414272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3415272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes; 3416272099Sglebius return (rv); 3417272099Sglebius case IFCOUNTER_OMCASTS: 3418272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3419272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts; 3420272099Sglebius return (rv); 3421263259Sbryanv#endif 3422272099Sglebius default: 3423272099Sglebius return (if_get_counter_default(ifp, cnt)); 3424272099Sglebius } 3425263259Sbryanv} 3426263259Sbryanv 3427263259Sbryanvstatic void 3428254738Sbryanvvmxnet3_tick(void *xsc) 3429254738Sbryanv{ 3430254738Sbryanv struct vmxnet3_softc *sc; 3431254738Sbryanv struct ifnet *ifp; 3432254738Sbryanv int i, timedout; 3433254738Sbryanv 3434254738Sbryanv sc = xsc; 3435254738Sbryanv ifp = sc->vmx_ifp; 3436254738Sbryanv timedout = 0; 3437254738Sbryanv 3438254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3439254738Sbryanv 3440263259Sbryanv vmxnet3_refresh_host_stats(sc); 3441263259Sbryanv 3442254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3443254738Sbryanv timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3444254738Sbryanv 3445254738Sbryanv if (timedout != 0) { 3446254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3447254738Sbryanv vmxnet3_init_locked(sc); 3448254738Sbryanv } else 3449254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3450254738Sbryanv} 3451254738Sbryanv 3452254738Sbryanvstatic int 3453254738Sbryanvvmxnet3_link_is_up(struct vmxnet3_softc *sc) 3454254738Sbryanv{ 3455254738Sbryanv uint32_t status; 3456254738Sbryanv 3457254738Sbryanv /* Also update the link speed while here. */ 3458254738Sbryanv status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3459254738Sbryanv sc->vmx_link_speed = status >> 16; 3460254738Sbryanv return !!(status & 0x1); 3461254738Sbryanv} 3462254738Sbryanv 3463254738Sbryanvstatic void 3464254738Sbryanvvmxnet3_link_status(struct vmxnet3_softc *sc) 3465254738Sbryanv{ 3466254738Sbryanv struct ifnet *ifp; 3467254738Sbryanv int link; 3468254738Sbryanv 3469254738Sbryanv ifp = sc->vmx_ifp; 3470254738Sbryanv link = vmxnet3_link_is_up(sc); 3471254738Sbryanv 3472254738Sbryanv if (link != 0 && sc->vmx_link_active == 0) { 3473254738Sbryanv sc->vmx_link_active = 1; 3474254738Sbryanv if_link_state_change(ifp, LINK_STATE_UP); 3475254738Sbryanv } else if (link == 0 && sc->vmx_link_active != 0) { 3476254738Sbryanv sc->vmx_link_active = 0; 3477254738Sbryanv if_link_state_change(ifp, LINK_STATE_DOWN); 3478254738Sbryanv } 3479254738Sbryanv} 3480254738Sbryanv 3481254738Sbryanvstatic void 3482254738Sbryanvvmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3483254738Sbryanv{ 3484254738Sbryanv struct vmxnet3_softc *sc; 3485254738Sbryanv 3486254738Sbryanv sc = ifp->if_softc; 3487254738Sbryanv 3488254738Sbryanv ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3489254738Sbryanv ifmr->ifm_status = IFM_AVALID; 3490254738Sbryanv 3491254738Sbryanv VMXNET3_CORE_LOCK(sc); 3492254738Sbryanv if (vmxnet3_link_is_up(sc) != 0) 3493254738Sbryanv ifmr->ifm_status |= IFM_ACTIVE; 3494254738Sbryanv else 3495254738Sbryanv ifmr->ifm_status |= IFM_NONE; 3496254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3497254738Sbryanv} 3498254738Sbryanv 3499254738Sbryanvstatic int 3500254738Sbryanvvmxnet3_media_change(struct ifnet *ifp) 3501254738Sbryanv{ 3502254738Sbryanv 3503254738Sbryanv /* Ignore. */ 3504254738Sbryanv return (0); 3505254738Sbryanv} 3506254738Sbryanv 3507254738Sbryanvstatic void 3508254738Sbryanvvmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3509254738Sbryanv{ 3510254738Sbryanv uint32_t ml, mh; 3511254738Sbryanv 3512254738Sbryanv ml = sc->vmx_lladdr[0]; 3513254738Sbryanv ml |= sc->vmx_lladdr[1] << 8; 3514254738Sbryanv ml |= sc->vmx_lladdr[2] << 16; 3515254738Sbryanv ml |= sc->vmx_lladdr[3] << 24; 3516254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3517254738Sbryanv 3518254738Sbryanv mh = sc->vmx_lladdr[4]; 3519254738Sbryanv mh |= sc->vmx_lladdr[5] << 8; 3520254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3521254738Sbryanv} 3522254738Sbryanv 3523254738Sbryanvstatic void 3524254738Sbryanvvmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3525254738Sbryanv{ 3526254738Sbryanv uint32_t ml, mh; 3527254738Sbryanv 3528254738Sbryanv ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3529254738Sbryanv mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3530254738Sbryanv 3531254738Sbryanv sc->vmx_lladdr[0] = ml; 3532254738Sbryanv sc->vmx_lladdr[1] = ml >> 8; 3533254738Sbryanv sc->vmx_lladdr[2] = ml >> 16; 3534254738Sbryanv sc->vmx_lladdr[3] = ml >> 24; 3535254738Sbryanv sc->vmx_lladdr[4] = mh; 3536254738Sbryanv sc->vmx_lladdr[5] = mh >> 8; 3537254738Sbryanv} 3538254738Sbryanv 3539254738Sbryanvstatic void 3540254738Sbryanvvmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3541254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3542254738Sbryanv{ 3543254738Sbryanv struct sysctl_oid *node, *txsnode; 3544254738Sbryanv struct sysctl_oid_list *list, *txslist; 3545254738Sbryanv struct vmxnet3_txq_stats *stats; 3546254738Sbryanv struct UPT1_TxStats *txstats; 3547254738Sbryanv char namebuf[16]; 3548254738Sbryanv 3549254738Sbryanv stats = &txq->vxtxq_stats; 3550254738Sbryanv txstats = &txq->vxtxq_ts->stats; 3551254738Sbryanv 3552254738Sbryanv snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3553254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3554254738Sbryanv NULL, "Transmit Queue"); 3555254738Sbryanv txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3556254738Sbryanv 3557263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3558263259Sbryanv &stats->vmtxs_opackets, "Transmit packets"); 3559263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3560263259Sbryanv &stats->vmtxs_obytes, "Transmit bytes"); 3561263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3562263259Sbryanv &stats->vmtxs_omcasts, "Transmit multicasts"); 3563263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3564263259Sbryanv &stats->vmtxs_csum, "Transmit checksum offloaded"); 3565263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3566263259Sbryanv &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3567254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3568263259Sbryanv &stats->vmtxs_full, "Transmit ring full"); 3569254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3570263259Sbryanv &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3571254738Sbryanv 3572254738Sbryanv /* 3573254738Sbryanv * Add statistics reported by the host. These are updated once 3574254738Sbryanv * per second. 3575254738Sbryanv */ 3576254738Sbryanv txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3577254738Sbryanv NULL, "Host Statistics"); 3578254738Sbryanv txslist = SYSCTL_CHILDREN(txsnode); 3579254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3580254738Sbryanv &txstats->TSO_packets, "TSO packets"); 3581254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3582254738Sbryanv &txstats->TSO_bytes, "TSO bytes"); 3583254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3584254738Sbryanv &txstats->ucast_packets, "Unicast packets"); 3585254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3586254738Sbryanv &txstats->ucast_bytes, "Unicast bytes"); 3587254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3588254738Sbryanv &txstats->mcast_packets, "Multicast packets"); 3589254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3590254738Sbryanv &txstats->mcast_bytes, "Multicast bytes"); 3591254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3592254738Sbryanv &txstats->error, "Errors"); 3593254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3594254738Sbryanv &txstats->discard, "Discards"); 3595254738Sbryanv} 3596254738Sbryanv 3597254738Sbryanvstatic void 3598254738Sbryanvvmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3599254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3600254738Sbryanv{ 3601254738Sbryanv struct sysctl_oid *node, *rxsnode; 3602254738Sbryanv struct sysctl_oid_list *list, *rxslist; 3603254738Sbryanv struct vmxnet3_rxq_stats *stats; 3604254738Sbryanv struct UPT1_RxStats *rxstats; 3605254738Sbryanv char namebuf[16]; 3606254738Sbryanv 3607254738Sbryanv stats = &rxq->vxrxq_stats; 3608254738Sbryanv rxstats = &rxq->vxrxq_rs->stats; 3609254738Sbryanv 3610254738Sbryanv snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3611254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3612254738Sbryanv NULL, "Receive Queue"); 3613254738Sbryanv rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3614254738Sbryanv 3615263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3616263259Sbryanv &stats->vmrxs_ipackets, "Receive packets"); 3617263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3618263259Sbryanv &stats->vmrxs_ibytes, "Receive bytes"); 3619263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3620263259Sbryanv &stats->vmrxs_iqdrops, "Receive drops"); 3621263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3622263259Sbryanv &stats->vmrxs_ierrors, "Receive errors"); 3623263259Sbryanv 3624254738Sbryanv /* 3625254738Sbryanv * Add statistics reported by the host. These are updated once 3626254738Sbryanv * per second. 3627254738Sbryanv */ 3628254738Sbryanv rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3629254738Sbryanv NULL, "Host Statistics"); 3630254738Sbryanv rxslist = SYSCTL_CHILDREN(rxsnode); 3631254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3632254738Sbryanv &rxstats->LRO_packets, "LRO packets"); 3633254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3634254738Sbryanv &rxstats->LRO_bytes, "LRO bytes"); 3635254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3636254738Sbryanv &rxstats->ucast_packets, "Unicast packets"); 3637254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3638254738Sbryanv &rxstats->ucast_bytes, "Unicast bytes"); 3639254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3640254738Sbryanv &rxstats->mcast_packets, "Multicast packets"); 3641254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3642254738Sbryanv &rxstats->mcast_bytes, "Multicast bytes"); 3643254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3644254738Sbryanv &rxstats->bcast_packets, "Broadcast packets"); 3645254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3646254738Sbryanv &rxstats->bcast_bytes, "Broadcast bytes"); 3647254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3648254738Sbryanv &rxstats->nobuffer, "No buffer"); 3649254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3650254738Sbryanv &rxstats->error, "Errors"); 3651254738Sbryanv} 3652254738Sbryanv 3653254738Sbryanvstatic void 3654254738Sbryanvvmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3655254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3656254738Sbryanv{ 3657254738Sbryanv struct sysctl_oid *node; 3658254738Sbryanv struct sysctl_oid_list *list; 3659254738Sbryanv int i; 3660254738Sbryanv 3661254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3662254738Sbryanv struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3663254738Sbryanv 3664254738Sbryanv node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3665254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3666254738Sbryanv list = SYSCTL_CHILDREN(node); 3667254738Sbryanv 3668254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3669254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3670254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3671254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3672254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3673254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3674254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3675254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3676254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3677254738Sbryanv &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3678254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3679254738Sbryanv &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3680254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3681254738Sbryanv &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3682254738Sbryanv } 3683254738Sbryanv 3684254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 3685254738Sbryanv struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3686254738Sbryanv 3687254738Sbryanv node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3688254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3689254738Sbryanv list = SYSCTL_CHILDREN(node); 3690254738Sbryanv 3691254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3692254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3693254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3694254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3695254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3696254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3697254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3698254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3699254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3700254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3701254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3702254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3703254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3704254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3705254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3706254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3707254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3708254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3709254738Sbryanv } 3710254738Sbryanv} 3711254738Sbryanv 3712254738Sbryanvstatic void 3713254738Sbryanvvmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3714254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3715254738Sbryanv{ 3716254738Sbryanv int i; 3717254738Sbryanv 3718254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3719254738Sbryanv vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3720254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 3721254738Sbryanv vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3722254738Sbryanv 3723254738Sbryanv vmxnet3_setup_debug_sysctl(sc, ctx, child); 3724254738Sbryanv} 3725254738Sbryanv 3726254738Sbryanvstatic void 3727254738Sbryanvvmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3728254738Sbryanv{ 3729254738Sbryanv device_t dev; 3730254738Sbryanv struct vmxnet3_statistics *stats; 3731254738Sbryanv struct sysctl_ctx_list *ctx; 3732254738Sbryanv struct sysctl_oid *tree; 3733254738Sbryanv struct sysctl_oid_list *child; 3734254738Sbryanv 3735254738Sbryanv dev = sc->vmx_dev; 3736254738Sbryanv ctx = device_get_sysctl_ctx(dev); 3737254738Sbryanv tree = device_get_sysctl_tree(dev); 3738254738Sbryanv child = SYSCTL_CHILDREN(tree); 3739254738Sbryanv 3740263259Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3741263259Sbryanv &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3742263259Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3743263259Sbryanv &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3744254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3745254738Sbryanv &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3746254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3747254738Sbryanv &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3748254738Sbryanv 3749254738Sbryanv stats = &sc->vmx_stats; 3750263259Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3751263259Sbryanv &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3752263259Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3753263259Sbryanv &stats->vmst_defrag_failed, 0, 3754263259Sbryanv "Tx mbuf dropped because defrag failed"); 3755254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3756254738Sbryanv &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3757254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3758254738Sbryanv &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3759254738Sbryanv 3760254738Sbryanv vmxnet3_setup_queue_sysctl(sc, ctx, child); 3761254738Sbryanv} 3762254738Sbryanv 3763254738Sbryanvstatic void 3764254738Sbryanvvmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3765254738Sbryanv{ 3766254738Sbryanv 3767254738Sbryanv bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3768254738Sbryanv} 3769254738Sbryanv 3770254738Sbryanvstatic uint32_t 3771254738Sbryanvvmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3772254738Sbryanv{ 3773254738Sbryanv 3774254738Sbryanv return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3775254738Sbryanv} 3776254738Sbryanv 3777254738Sbryanvstatic void 3778254738Sbryanvvmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3779254738Sbryanv{ 3780254738Sbryanv 3781254738Sbryanv bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3782254738Sbryanv} 3783254738Sbryanv 3784254738Sbryanvstatic void 3785254738Sbryanvvmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3786254738Sbryanv{ 3787254738Sbryanv 3788254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3789254738Sbryanv} 3790254738Sbryanv 3791254738Sbryanvstatic uint32_t 3792254738Sbryanvvmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3793254738Sbryanv{ 3794254738Sbryanv 3795254738Sbryanv vmxnet3_write_cmd(sc, cmd); 3796254738Sbryanv bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3797254738Sbryanv BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3798254738Sbryanv return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3799254738Sbryanv} 3800254738Sbryanv 3801254738Sbryanvstatic void 3802254738Sbryanvvmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3803254738Sbryanv{ 3804254738Sbryanv 3805254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3806254738Sbryanv} 3807254738Sbryanv 3808254738Sbryanvstatic void 3809254738Sbryanvvmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3810254738Sbryanv{ 3811254738Sbryanv 3812254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3813254738Sbryanv} 3814254738Sbryanv 3815254738Sbryanvstatic void 3816254738Sbryanvvmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3817254738Sbryanv{ 3818254738Sbryanv int i; 3819254738Sbryanv 3820254738Sbryanv sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3821254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3822254738Sbryanv vmxnet3_enable_intr(sc, i); 3823254738Sbryanv} 3824254738Sbryanv 3825254738Sbryanvstatic void 3826254738Sbryanvvmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3827254738Sbryanv{ 3828254738Sbryanv int i; 3829254738Sbryanv 3830254738Sbryanv sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3831254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3832254738Sbryanv vmxnet3_disable_intr(sc, i); 3833254738Sbryanv} 3834254738Sbryanv 3835254738Sbryanvstatic void 3836254738Sbryanvvmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3837254738Sbryanv{ 3838254738Sbryanv bus_addr_t *baddr = arg; 3839254738Sbryanv 3840254738Sbryanv if (error == 0) 3841254738Sbryanv *baddr = segs->ds_addr; 3842254738Sbryanv} 3843254738Sbryanv 3844254738Sbryanvstatic int 3845254738Sbryanvvmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3846254738Sbryanv struct vmxnet3_dma_alloc *dma) 3847254738Sbryanv{ 3848254738Sbryanv device_t dev; 3849254738Sbryanv int error; 3850254738Sbryanv 3851254738Sbryanv dev = sc->vmx_dev; 3852254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3853254738Sbryanv 3854254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 3855254738Sbryanv align, 0, /* alignment, bounds */ 3856254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 3857254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 3858254738Sbryanv NULL, NULL, /* filter, filterarg */ 3859254738Sbryanv size, /* maxsize */ 3860254738Sbryanv 1, /* nsegments */ 3861254738Sbryanv size, /* maxsegsize */ 3862254738Sbryanv BUS_DMA_ALLOCNOW, /* flags */ 3863254738Sbryanv NULL, /* lockfunc */ 3864254738Sbryanv NULL, /* lockfuncarg */ 3865254738Sbryanv &dma->dma_tag); 3866254738Sbryanv if (error) { 3867254738Sbryanv device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3868254738Sbryanv goto fail; 3869254738Sbryanv } 3870254738Sbryanv 3871254738Sbryanv error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3872254738Sbryanv BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3873254738Sbryanv if (error) { 3874254738Sbryanv device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3875254738Sbryanv goto fail; 3876254738Sbryanv } 3877254738Sbryanv 3878254738Sbryanv error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3879254738Sbryanv size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3880254738Sbryanv if (error) { 3881254738Sbryanv device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3882254738Sbryanv goto fail; 3883254738Sbryanv } 3884254738Sbryanv 3885254738Sbryanv dma->dma_size = size; 3886254738Sbryanv 3887254738Sbryanvfail: 3888254738Sbryanv if (error) 3889254738Sbryanv vmxnet3_dma_free(sc, dma); 3890254738Sbryanv 3891254738Sbryanv return (error); 3892254738Sbryanv} 3893254738Sbryanv 3894254738Sbryanvstatic void 3895254738Sbryanvvmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3896254738Sbryanv{ 3897254738Sbryanv 3898254738Sbryanv if (dma->dma_tag != NULL) { 3899267488Sbryanv if (dma->dma_paddr != 0) { 3900254738Sbryanv bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3901254738Sbryanv BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3902254738Sbryanv bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3903254738Sbryanv } 3904254738Sbryanv 3905254738Sbryanv if (dma->dma_vaddr != NULL) { 3906254738Sbryanv bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3907254738Sbryanv dma->dma_map); 3908254738Sbryanv } 3909254738Sbryanv 3910254738Sbryanv bus_dma_tag_destroy(dma->dma_tag); 3911254738Sbryanv } 3912254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3913254738Sbryanv} 3914254738Sbryanv 3915255055Sbryanvstatic int 3916255055Sbryanvvmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3917255055Sbryanv{ 3918255055Sbryanv char path[64]; 3919255055Sbryanv 3920255055Sbryanv snprintf(path, sizeof(path), 3921255055Sbryanv "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3922255055Sbryanv TUNABLE_INT_FETCH(path, &def); 3923255055Sbryanv 3924255055Sbryanv return (def); 3925255055Sbryanv} 3926255055Sbryanv 3927254738Sbryanv/* 3928254738Sbryanv * Since this is a purely paravirtualized device, we do not have 3929254738Sbryanv * to worry about DMA coherency. But at times, we must make sure 3930254738Sbryanv * both the compiler and CPU do not reorder memory operations. 3931254738Sbryanv */ 3932254738Sbryanvstatic inline void 3933254738Sbryanvvmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3934254738Sbryanv{ 3935254738Sbryanv 3936254738Sbryanv switch (type) { 3937254738Sbryanv case VMXNET3_BARRIER_RD: 3938254738Sbryanv rmb(); 3939254738Sbryanv break; 3940254738Sbryanv case VMXNET3_BARRIER_WR: 3941254738Sbryanv wmb(); 3942254738Sbryanv break; 3943254738Sbryanv case VMXNET3_BARRIER_RDWR: 3944254738Sbryanv mb(); 3945254738Sbryanv break; 3946254738Sbryanv default: 3947254738Sbryanv panic("%s: bad barrier type %d", __func__, type); 3948254738Sbryanv } 3949254738Sbryanv} 3950