1254738Sbryanv/*- 2254738Sbryanv * Copyright (c) 2013 Tsubai Masanari 3254738Sbryanv * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4254738Sbryanv * 5254738Sbryanv * Permission to use, copy, modify, and distribute this software for any 6254738Sbryanv * purpose with or without fee is hereby granted, provided that the above 7254738Sbryanv * copyright notice and this permission notice appear in all copies. 8254738Sbryanv * 9254738Sbryanv * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10254738Sbryanv * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11254738Sbryanv * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12254738Sbryanv * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13254738Sbryanv * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14254738Sbryanv * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15254738Sbryanv * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16254738Sbryanv * 17254738Sbryanv * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18254738Sbryanv */ 19254738Sbryanv 20254738Sbryanv/* Driver for VMware vmxnet3 virtual ethernet devices. */ 21254738Sbryanv 22254738Sbryanv#include <sys/cdefs.h> 23254738Sbryanv__FBSDID("$FreeBSD: releng/10.3/sys/dev/vmware/vmxnet3/if_vmx.c 292568 2015-12-21 20:40:17Z jhb $"); 24254738Sbryanv 25254738Sbryanv#include <sys/param.h> 26254738Sbryanv#include <sys/systm.h> 27254738Sbryanv#include <sys/kernel.h> 28254738Sbryanv#include <sys/endian.h> 29254738Sbryanv#include <sys/sockio.h> 30254738Sbryanv#include <sys/mbuf.h> 31254738Sbryanv#include <sys/malloc.h> 32254738Sbryanv#include <sys/module.h> 33254738Sbryanv#include <sys/socket.h> 34254738Sbryanv#include <sys/sysctl.h> 35264866Sbryanv#include <sys/smp.h> 36264866Sbryanv#include <sys/taskqueue.h> 37254738Sbryanv#include <vm/vm.h> 38254738Sbryanv#include <vm/pmap.h> 39254738Sbryanv 40254738Sbryanv#include <net/ethernet.h> 41254738Sbryanv#include <net/if.h> 42254738Sbryanv#include <net/if_arp.h> 43254738Sbryanv#include <net/if_dl.h> 44254738Sbryanv#include <net/if_types.h> 45254738Sbryanv#include <net/if_media.h> 46254738Sbryanv#include <net/if_vlan_var.h> 47254738Sbryanv 48254738Sbryanv#include <net/bpf.h> 49254738Sbryanv 50254738Sbryanv#include <netinet/in_systm.h> 51254738Sbryanv#include <netinet/in.h> 52254738Sbryanv#include <netinet/ip.h> 53254738Sbryanv#include <netinet/ip6.h> 54254738Sbryanv#include <netinet6/ip6_var.h> 55254738Sbryanv#include <netinet/udp.h> 56254738Sbryanv#include <netinet/tcp.h> 57254738Sbryanv 58267382Sbryanv#include <machine/in_cksum.h> 59267382Sbryanv 60254738Sbryanv#include <machine/bus.h> 61254738Sbryanv#include <machine/resource.h> 62254738Sbryanv#include <sys/bus.h> 63254738Sbryanv#include <sys/rman.h> 64254738Sbryanv 65254738Sbryanv#include <dev/pci/pcireg.h> 66254738Sbryanv#include <dev/pci/pcivar.h> 67254738Sbryanv 68254738Sbryanv#include "if_vmxreg.h" 69254738Sbryanv#include "if_vmxvar.h" 70254738Sbryanv 71254738Sbryanv#include "opt_inet.h" 72254738Sbryanv#include "opt_inet6.h" 73254738Sbryanv 74254738Sbryanv#ifdef VMXNET3_FAILPOINTS 75254738Sbryanv#include <sys/fail.h> 76254738Sbryanvstatic SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 77254738Sbryanv "vmxnet3 fail points"); 78254738Sbryanv#define VMXNET3_FP _debug_fail_point_vmxnet3 79254738Sbryanv#endif 80254738Sbryanv 81254738Sbryanvstatic int vmxnet3_probe(device_t); 82254738Sbryanvstatic int vmxnet3_attach(device_t); 83254738Sbryanvstatic int vmxnet3_detach(device_t); 84254738Sbryanvstatic int vmxnet3_shutdown(device_t); 85254738Sbryanv 86254738Sbryanvstatic int vmxnet3_alloc_resources(struct vmxnet3_softc *); 87254738Sbryanvstatic void vmxnet3_free_resources(struct vmxnet3_softc *); 88254738Sbryanvstatic int vmxnet3_check_version(struct vmxnet3_softc *); 89254738Sbryanvstatic void vmxnet3_initial_config(struct vmxnet3_softc *); 90264866Sbryanvstatic void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 91254738Sbryanv 92254738Sbryanvstatic int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 93254738Sbryanvstatic int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 94254738Sbryanvstatic int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 95254738Sbryanvstatic int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 96254738Sbryanv struct vmxnet3_interrupt *); 97254738Sbryanvstatic int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 98254738Sbryanvstatic int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 99254738Sbryanvstatic int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 100254738Sbryanvstatic int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 101254738Sbryanvstatic int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 102254738Sbryanv 103254738Sbryanvstatic void vmxnet3_free_interrupt(struct vmxnet3_softc *, 104254738Sbryanv struct vmxnet3_interrupt *); 105254738Sbryanvstatic void vmxnet3_free_interrupts(struct vmxnet3_softc *); 106254738Sbryanv 107264866Sbryanv#ifndef VMXNET3_LEGACY_TX 108264866Sbryanvstatic int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 109264866Sbryanvstatic void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 110264866Sbryanvstatic void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 111264866Sbryanvstatic void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 112264866Sbryanv#endif 113264866Sbryanv 114254738Sbryanvstatic int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 115254738Sbryanvstatic int vmxnet3_init_txq(struct vmxnet3_softc *, int); 116254738Sbryanvstatic int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 117254738Sbryanvstatic void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 118254738Sbryanvstatic void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 119254738Sbryanvstatic void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 120254738Sbryanv 121254738Sbryanvstatic int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 122254738Sbryanvstatic void vmxnet3_free_shared_data(struct vmxnet3_softc *); 123254738Sbryanvstatic int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 124254738Sbryanvstatic void vmxnet3_free_txq_data(struct vmxnet3_softc *); 125254738Sbryanvstatic int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 126254738Sbryanvstatic void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 127254738Sbryanvstatic int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 128254738Sbryanvstatic void vmxnet3_free_queue_data(struct vmxnet3_softc *); 129254738Sbryanvstatic int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 130254738Sbryanvstatic void vmxnet3_init_shared_data(struct vmxnet3_softc *); 131254738Sbryanvstatic void vmxnet3_reinit_interface(struct vmxnet3_softc *); 132264866Sbryanvstatic void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 133254738Sbryanvstatic void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 134254738Sbryanvstatic int vmxnet3_alloc_data(struct vmxnet3_softc *); 135254738Sbryanvstatic void vmxnet3_free_data(struct vmxnet3_softc *); 136254738Sbryanvstatic int vmxnet3_setup_interface(struct vmxnet3_softc *); 137254738Sbryanv 138254738Sbryanvstatic void vmxnet3_evintr(struct vmxnet3_softc *); 139254738Sbryanvstatic void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 140254738Sbryanvstatic void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 141254738Sbryanvstatic int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 142254738Sbryanvstatic void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 143254738Sbryanv struct vmxnet3_rxring *, int); 144254738Sbryanvstatic void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 145254738Sbryanvstatic void vmxnet3_legacy_intr(void *); 146254738Sbryanvstatic void vmxnet3_txq_intr(void *); 147254738Sbryanvstatic void vmxnet3_rxq_intr(void *); 148254738Sbryanvstatic void vmxnet3_event_intr(void *); 149254738Sbryanv 150254738Sbryanvstatic void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 151254738Sbryanvstatic void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 152254738Sbryanvstatic void vmxnet3_stop(struct vmxnet3_softc *); 153254738Sbryanv 154254738Sbryanvstatic void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 155254738Sbryanvstatic int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 156254738Sbryanvstatic int vmxnet3_reinit_queues(struct vmxnet3_softc *); 157254738Sbryanvstatic int vmxnet3_enable_device(struct vmxnet3_softc *); 158254738Sbryanvstatic void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 159254738Sbryanvstatic int vmxnet3_reinit(struct vmxnet3_softc *); 160254738Sbryanvstatic void vmxnet3_init_locked(struct vmxnet3_softc *); 161254738Sbryanvstatic void vmxnet3_init(void *); 162254738Sbryanv 163264866Sbryanvstatic int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 164264866Sbryanv int *, int *, int *); 165254738Sbryanvstatic int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 166254738Sbryanv bus_dmamap_t, bus_dma_segment_t [], int *); 167254738Sbryanvstatic void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 168254738Sbryanvstatic int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 169264866Sbryanv#ifdef VMXNET3_LEGACY_TX 170254738Sbryanvstatic void vmxnet3_start_locked(struct ifnet *); 171254738Sbryanvstatic void vmxnet3_start(struct ifnet *); 172264866Sbryanv#else 173264866Sbryanvstatic int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 174264866Sbryanv struct mbuf *); 175264866Sbryanvstatic int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 176264866Sbryanvstatic void vmxnet3_txq_tq_deferred(void *, int); 177264866Sbryanv#endif 178264866Sbryanvstatic void vmxnet3_txq_start(struct vmxnet3_txqueue *); 179264866Sbryanvstatic void vmxnet3_tx_start_all(struct vmxnet3_softc *); 180254738Sbryanv 181254738Sbryanvstatic void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 182254738Sbryanv uint16_t); 183254738Sbryanvstatic void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 184254738Sbryanvstatic void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 185254738Sbryanvstatic void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 186254738Sbryanvstatic int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 187254738Sbryanvstatic int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 188254738Sbryanv 189264866Sbryanv#ifndef VMXNET3_LEGACY_TX 190264866Sbryanvstatic void vmxnet3_qflush(struct ifnet *); 191264866Sbryanv#endif 192264866Sbryanv 193254738Sbryanvstatic int vmxnet3_watchdog(struct vmxnet3_txqueue *); 194264866Sbryanvstatic void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 195264866Sbryanvstatic void vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *, 196264866Sbryanv struct vmxnet3_txq_stats *); 197264866Sbryanvstatic void vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *, 198264866Sbryanv struct vmxnet3_rxq_stats *); 199254738Sbryanvstatic void vmxnet3_tick(void *); 200254738Sbryanvstatic void vmxnet3_link_status(struct vmxnet3_softc *); 201254738Sbryanvstatic void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 202254738Sbryanvstatic int vmxnet3_media_change(struct ifnet *); 203254738Sbryanvstatic void vmxnet3_set_lladdr(struct vmxnet3_softc *); 204254738Sbryanvstatic void vmxnet3_get_lladdr(struct vmxnet3_softc *); 205254738Sbryanv 206254738Sbryanvstatic void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 207254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 208254738Sbryanvstatic void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 209254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 210254738Sbryanvstatic void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 211254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 212254738Sbryanvstatic void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 213254738Sbryanv 214254738Sbryanvstatic void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 215254738Sbryanv uint32_t); 216254738Sbryanvstatic uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 217254738Sbryanvstatic void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 218254738Sbryanv uint32_t); 219254738Sbryanvstatic void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 220254738Sbryanvstatic uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 221254738Sbryanv 222254738Sbryanvstatic void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 223254738Sbryanvstatic void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 224254738Sbryanvstatic void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 225254738Sbryanvstatic void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 226254738Sbryanv 227254738Sbryanvstatic int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 228254738Sbryanv bus_size_t, struct vmxnet3_dma_alloc *); 229254738Sbryanvstatic void vmxnet3_dma_free(struct vmxnet3_softc *, 230254738Sbryanv struct vmxnet3_dma_alloc *); 231255055Sbryanvstatic int vmxnet3_tunable_int(struct vmxnet3_softc *, 232255055Sbryanv const char *, int); 233254738Sbryanv 234254738Sbryanvtypedef enum { 235254738Sbryanv VMXNET3_BARRIER_RD, 236254738Sbryanv VMXNET3_BARRIER_WR, 237254738Sbryanv VMXNET3_BARRIER_RDWR, 238254738Sbryanv} vmxnet3_barrier_t; 239254738Sbryanv 240254738Sbryanvstatic void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 241254738Sbryanv 242255055Sbryanv/* Tunables. */ 243264866Sbryanvstatic int vmxnet3_mq_disable = 0; 244264866SbryanvTUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 245264866Sbryanvstatic int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 246264866SbryanvTUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 247264866Sbryanvstatic int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 248264866SbryanvTUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 249255055Sbryanvstatic int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 250255055SbryanvTUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 251255055Sbryanvstatic int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 252255055SbryanvTUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 253255055Sbryanv 254254738Sbryanvstatic device_method_t vmxnet3_methods[] = { 255254738Sbryanv /* Device interface. */ 256254738Sbryanv DEVMETHOD(device_probe, vmxnet3_probe), 257254738Sbryanv DEVMETHOD(device_attach, vmxnet3_attach), 258254738Sbryanv DEVMETHOD(device_detach, vmxnet3_detach), 259254738Sbryanv DEVMETHOD(device_shutdown, vmxnet3_shutdown), 260254738Sbryanv 261254738Sbryanv DEVMETHOD_END 262254738Sbryanv}; 263254738Sbryanv 264254738Sbryanvstatic driver_t vmxnet3_driver = { 265254738Sbryanv "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 266254738Sbryanv}; 267254738Sbryanv 268254738Sbryanvstatic devclass_t vmxnet3_devclass; 269254738SbryanvDRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 270254738Sbryanv 271254738SbryanvMODULE_DEPEND(vmx, pci, 1, 1, 1); 272254738SbryanvMODULE_DEPEND(vmx, ether, 1, 1, 1); 273254738Sbryanv 274254738Sbryanv#define VMXNET3_VMWARE_VENDOR_ID 0x15AD 275254738Sbryanv#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 276254738Sbryanv 277254738Sbryanvstatic int 278254738Sbryanvvmxnet3_probe(device_t dev) 279254738Sbryanv{ 280254738Sbryanv 281254738Sbryanv if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 282254738Sbryanv pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 283254738Sbryanv device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 284254738Sbryanv return (BUS_PROBE_DEFAULT); 285254738Sbryanv } 286254738Sbryanv 287254738Sbryanv return (ENXIO); 288254738Sbryanv} 289254738Sbryanv 290254738Sbryanvstatic int 291254738Sbryanvvmxnet3_attach(device_t dev) 292254738Sbryanv{ 293254738Sbryanv struct vmxnet3_softc *sc; 294254738Sbryanv int error; 295254738Sbryanv 296254738Sbryanv sc = device_get_softc(dev); 297254738Sbryanv sc->vmx_dev = dev; 298254738Sbryanv 299254738Sbryanv pci_enable_busmaster(dev); 300254738Sbryanv 301254738Sbryanv VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 302254738Sbryanv callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 303254738Sbryanv 304254738Sbryanv vmxnet3_initial_config(sc); 305254738Sbryanv 306254738Sbryanv error = vmxnet3_alloc_resources(sc); 307254738Sbryanv if (error) 308254738Sbryanv goto fail; 309254738Sbryanv 310254738Sbryanv error = vmxnet3_check_version(sc); 311254738Sbryanv if (error) 312254738Sbryanv goto fail; 313254738Sbryanv 314254738Sbryanv error = vmxnet3_alloc_rxtx_queues(sc); 315254738Sbryanv if (error) 316254738Sbryanv goto fail; 317254738Sbryanv 318264866Sbryanv#ifndef VMXNET3_LEGACY_TX 319264866Sbryanv error = vmxnet3_alloc_taskqueue(sc); 320264866Sbryanv if (error) 321264866Sbryanv goto fail; 322264866Sbryanv#endif 323264866Sbryanv 324254738Sbryanv error = vmxnet3_alloc_interrupts(sc); 325254738Sbryanv if (error) 326254738Sbryanv goto fail; 327254738Sbryanv 328264866Sbryanv vmxnet3_check_multiqueue(sc); 329264866Sbryanv 330254738Sbryanv error = vmxnet3_alloc_data(sc); 331254738Sbryanv if (error) 332254738Sbryanv goto fail; 333254738Sbryanv 334254738Sbryanv error = vmxnet3_setup_interface(sc); 335254738Sbryanv if (error) 336254738Sbryanv goto fail; 337254738Sbryanv 338254738Sbryanv error = vmxnet3_setup_interrupts(sc); 339254738Sbryanv if (error) { 340254738Sbryanv ether_ifdetach(sc->vmx_ifp); 341254738Sbryanv device_printf(dev, "could not set up interrupt\n"); 342254738Sbryanv goto fail; 343254738Sbryanv } 344254738Sbryanv 345254738Sbryanv vmxnet3_setup_sysctl(sc); 346264866Sbryanv#ifndef VMXNET3_LEGACY_TX 347264866Sbryanv vmxnet3_start_taskqueue(sc); 348264866Sbryanv#endif 349254738Sbryanv 350254738Sbryanvfail: 351254738Sbryanv if (error) 352254738Sbryanv vmxnet3_detach(dev); 353254738Sbryanv 354254738Sbryanv return (error); 355254738Sbryanv} 356254738Sbryanv 357254738Sbryanvstatic int 358254738Sbryanvvmxnet3_detach(device_t dev) 359254738Sbryanv{ 360254738Sbryanv struct vmxnet3_softc *sc; 361254738Sbryanv struct ifnet *ifp; 362254738Sbryanv 363254738Sbryanv sc = device_get_softc(dev); 364254738Sbryanv ifp = sc->vmx_ifp; 365254738Sbryanv 366254738Sbryanv if (device_is_attached(dev)) { 367254738Sbryanv VMXNET3_CORE_LOCK(sc); 368254738Sbryanv vmxnet3_stop(sc); 369254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 370264866Sbryanv 371254738Sbryanv callout_drain(&sc->vmx_tick); 372264866Sbryanv#ifndef VMXNET3_LEGACY_TX 373264866Sbryanv vmxnet3_drain_taskqueue(sc); 374264866Sbryanv#endif 375264866Sbryanv 376264866Sbryanv ether_ifdetach(ifp); 377254738Sbryanv } 378254738Sbryanv 379254738Sbryanv if (sc->vmx_vlan_attach != NULL) { 380254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 381254738Sbryanv sc->vmx_vlan_attach = NULL; 382254738Sbryanv } 383254738Sbryanv if (sc->vmx_vlan_detach != NULL) { 384254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 385254738Sbryanv sc->vmx_vlan_detach = NULL; 386254738Sbryanv } 387254738Sbryanv 388264866Sbryanv#ifndef VMXNET3_LEGACY_TX 389264866Sbryanv vmxnet3_free_taskqueue(sc); 390264866Sbryanv#endif 391254738Sbryanv vmxnet3_free_interrupts(sc); 392254738Sbryanv 393254738Sbryanv if (ifp != NULL) { 394254738Sbryanv if_free(ifp); 395254738Sbryanv sc->vmx_ifp = NULL; 396254738Sbryanv } 397254738Sbryanv 398254738Sbryanv ifmedia_removeall(&sc->vmx_media); 399254738Sbryanv 400254738Sbryanv vmxnet3_free_data(sc); 401254738Sbryanv vmxnet3_free_resources(sc); 402254738Sbryanv vmxnet3_free_rxtx_queues(sc); 403254738Sbryanv 404254738Sbryanv VMXNET3_CORE_LOCK_DESTROY(sc); 405254738Sbryanv 406254738Sbryanv return (0); 407254738Sbryanv} 408254738Sbryanv 409254738Sbryanvstatic int 410254738Sbryanvvmxnet3_shutdown(device_t dev) 411254738Sbryanv{ 412254738Sbryanv 413254738Sbryanv return (0); 414254738Sbryanv} 415254738Sbryanv 416254738Sbryanvstatic int 417254738Sbryanvvmxnet3_alloc_resources(struct vmxnet3_softc *sc) 418254738Sbryanv{ 419254738Sbryanv device_t dev; 420254738Sbryanv int rid; 421254738Sbryanv 422254738Sbryanv dev = sc->vmx_dev; 423254738Sbryanv 424254738Sbryanv rid = PCIR_BAR(0); 425254738Sbryanv sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 426254738Sbryanv RF_ACTIVE); 427254738Sbryanv if (sc->vmx_res0 == NULL) { 428254738Sbryanv device_printf(dev, 429254738Sbryanv "could not map BAR0 memory\n"); 430254738Sbryanv return (ENXIO); 431254738Sbryanv } 432254738Sbryanv 433254738Sbryanv sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 434254738Sbryanv sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 435254738Sbryanv 436254738Sbryanv rid = PCIR_BAR(1); 437254738Sbryanv sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 438254738Sbryanv RF_ACTIVE); 439254738Sbryanv if (sc->vmx_res1 == NULL) { 440254738Sbryanv device_printf(dev, 441254738Sbryanv "could not map BAR1 memory\n"); 442254738Sbryanv return (ENXIO); 443254738Sbryanv } 444254738Sbryanv 445254738Sbryanv sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 446254738Sbryanv sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 447254738Sbryanv 448254738Sbryanv if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 449254738Sbryanv rid = PCIR_BAR(2); 450254738Sbryanv sc->vmx_msix_res = bus_alloc_resource_any(dev, 451254738Sbryanv SYS_RES_MEMORY, &rid, RF_ACTIVE); 452254738Sbryanv } 453254738Sbryanv 454254738Sbryanv if (sc->vmx_msix_res == NULL) 455254738Sbryanv sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 456254738Sbryanv 457254738Sbryanv return (0); 458254738Sbryanv} 459254738Sbryanv 460254738Sbryanvstatic void 461254738Sbryanvvmxnet3_free_resources(struct vmxnet3_softc *sc) 462254738Sbryanv{ 463254738Sbryanv device_t dev; 464254738Sbryanv int rid; 465254738Sbryanv 466254738Sbryanv dev = sc->vmx_dev; 467254738Sbryanv 468254738Sbryanv if (sc->vmx_res0 != NULL) { 469254738Sbryanv rid = PCIR_BAR(0); 470254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 471254738Sbryanv sc->vmx_res0 = NULL; 472254738Sbryanv } 473254738Sbryanv 474254738Sbryanv if (sc->vmx_res1 != NULL) { 475254738Sbryanv rid = PCIR_BAR(1); 476254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 477254738Sbryanv sc->vmx_res1 = NULL; 478254738Sbryanv } 479254738Sbryanv 480254738Sbryanv if (sc->vmx_msix_res != NULL) { 481254738Sbryanv rid = PCIR_BAR(2); 482254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, 483254738Sbryanv sc->vmx_msix_res); 484254738Sbryanv sc->vmx_msix_res = NULL; 485254738Sbryanv } 486254738Sbryanv} 487254738Sbryanv 488254738Sbryanvstatic int 489254738Sbryanvvmxnet3_check_version(struct vmxnet3_softc *sc) 490254738Sbryanv{ 491254738Sbryanv device_t dev; 492254738Sbryanv uint32_t version; 493254738Sbryanv 494254738Sbryanv dev = sc->vmx_dev; 495254738Sbryanv 496254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 497254738Sbryanv if ((version & 0x01) == 0) { 498254738Sbryanv device_printf(dev, "unsupported hardware version %#x\n", 499254738Sbryanv version); 500254738Sbryanv return (ENOTSUP); 501254950Sbryanv } 502254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 503254738Sbryanv 504254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 505254738Sbryanv if ((version & 0x01) == 0) { 506254738Sbryanv device_printf(dev, "unsupported UPT version %#x\n", version); 507254738Sbryanv return (ENOTSUP); 508254950Sbryanv } 509254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 510254738Sbryanv 511254738Sbryanv return (0); 512254738Sbryanv} 513254738Sbryanv 514292568Sjhbstatic int 515292568Sjhbtrunc_powerof2(int val) 516292568Sjhb{ 517292568Sjhb 518292568Sjhb return (1U << (fls(val) - 1)); 519292568Sjhb} 520292568Sjhb 521254738Sbryanvstatic void 522254738Sbryanvvmxnet3_initial_config(struct vmxnet3_softc *sc) 523254738Sbryanv{ 524264866Sbryanv int nqueue, ndesc; 525254738Sbryanv 526264866Sbryanv nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 527264866Sbryanv if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 528264866Sbryanv nqueue = VMXNET3_DEF_TX_QUEUES; 529264866Sbryanv if (nqueue > mp_ncpus) 530264866Sbryanv nqueue = mp_ncpus; 531292568Sjhb sc->vmx_max_ntxqueues = trunc_powerof2(nqueue); 532255055Sbryanv 533264866Sbryanv nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 534264866Sbryanv if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 535264866Sbryanv nqueue = VMXNET3_DEF_RX_QUEUES; 536264866Sbryanv if (nqueue > mp_ncpus) 537264866Sbryanv nqueue = mp_ncpus; 538292568Sjhb sc->vmx_max_nrxqueues = trunc_powerof2(nqueue); 539264866Sbryanv 540264866Sbryanv if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 541264866Sbryanv sc->vmx_max_nrxqueues = 1; 542264866Sbryanv sc->vmx_max_ntxqueues = 1; 543264866Sbryanv } 544264866Sbryanv 545255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 546255055Sbryanv if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 547255055Sbryanv ndesc = VMXNET3_DEF_TX_NDESC; 548255055Sbryanv if (ndesc & VMXNET3_MASK_TX_NDESC) 549255055Sbryanv ndesc &= ~VMXNET3_MASK_TX_NDESC; 550255055Sbryanv sc->vmx_ntxdescs = ndesc; 551255055Sbryanv 552255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 553255055Sbryanv if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 554255055Sbryanv ndesc = VMXNET3_DEF_RX_NDESC; 555255055Sbryanv if (ndesc & VMXNET3_MASK_RX_NDESC) 556255055Sbryanv ndesc &= ~VMXNET3_MASK_RX_NDESC; 557255055Sbryanv sc->vmx_nrxdescs = ndesc; 558254738Sbryanv sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 559254738Sbryanv} 560254738Sbryanv 561264866Sbryanvstatic void 562264866Sbryanvvmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 563264866Sbryanv{ 564264866Sbryanv 565264866Sbryanv if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 566264866Sbryanv goto out; 567264866Sbryanv 568264866Sbryanv /* BMV: Just use the maximum configured for now. */ 569264866Sbryanv sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 570264866Sbryanv sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 571264866Sbryanv 572264866Sbryanv if (sc->vmx_nrxqueues > 1) 573264866Sbryanv sc->vmx_flags |= VMXNET3_FLAG_RSS; 574264866Sbryanv 575264866Sbryanv return; 576264866Sbryanv 577264866Sbryanvout: 578264866Sbryanv sc->vmx_ntxqueues = 1; 579264866Sbryanv sc->vmx_nrxqueues = 1; 580264866Sbryanv} 581264866Sbryanv 582254738Sbryanvstatic int 583254738Sbryanvvmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 584254738Sbryanv{ 585254738Sbryanv device_t dev; 586254738Sbryanv int nmsix, cnt, required; 587254738Sbryanv 588254738Sbryanv dev = sc->vmx_dev; 589254738Sbryanv 590254738Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 591254738Sbryanv return (1); 592254738Sbryanv 593254738Sbryanv /* Allocate an additional vector for the events interrupt. */ 594264866Sbryanv required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 595254738Sbryanv 596254738Sbryanv nmsix = pci_msix_count(dev); 597254738Sbryanv if (nmsix < required) 598254738Sbryanv return (1); 599254738Sbryanv 600254738Sbryanv cnt = required; 601254738Sbryanv if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 602254738Sbryanv sc->vmx_nintrs = required; 603254738Sbryanv return (0); 604254738Sbryanv } else 605254738Sbryanv pci_release_msi(dev); 606254738Sbryanv 607264866Sbryanv /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 608264866Sbryanv 609254738Sbryanv return (1); 610254738Sbryanv} 611254738Sbryanv 612254738Sbryanvstatic int 613254738Sbryanvvmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 614254738Sbryanv{ 615254738Sbryanv device_t dev; 616254738Sbryanv int nmsi, cnt, required; 617254738Sbryanv 618254738Sbryanv dev = sc->vmx_dev; 619254738Sbryanv required = 1; 620254738Sbryanv 621254738Sbryanv nmsi = pci_msi_count(dev); 622254738Sbryanv if (nmsi < required) 623254738Sbryanv return (1); 624254738Sbryanv 625254738Sbryanv cnt = required; 626254738Sbryanv if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 627254738Sbryanv sc->vmx_nintrs = 1; 628254738Sbryanv return (0); 629254738Sbryanv } else 630254738Sbryanv pci_release_msi(dev); 631254738Sbryanv 632254738Sbryanv return (1); 633254738Sbryanv} 634254738Sbryanv 635254738Sbryanvstatic int 636254738Sbryanvvmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 637254738Sbryanv{ 638254738Sbryanv 639254738Sbryanv sc->vmx_nintrs = 1; 640254738Sbryanv return (0); 641254738Sbryanv} 642254738Sbryanv 643254738Sbryanvstatic int 644254738Sbryanvvmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 645254738Sbryanv struct vmxnet3_interrupt *intr) 646254738Sbryanv{ 647254738Sbryanv struct resource *irq; 648254738Sbryanv 649254738Sbryanv irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 650254738Sbryanv if (irq == NULL) 651254738Sbryanv return (ENXIO); 652254738Sbryanv 653254738Sbryanv intr->vmxi_irq = irq; 654254738Sbryanv intr->vmxi_rid = rid; 655254738Sbryanv 656254738Sbryanv return (0); 657254738Sbryanv} 658254738Sbryanv 659254738Sbryanvstatic int 660254738Sbryanvvmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 661254738Sbryanv{ 662254738Sbryanv int i, rid, flags, error; 663254738Sbryanv 664254738Sbryanv rid = 0; 665254738Sbryanv flags = RF_ACTIVE; 666254738Sbryanv 667254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 668254738Sbryanv flags |= RF_SHAREABLE; 669254738Sbryanv else 670254738Sbryanv rid = 1; 671254738Sbryanv 672254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 673254738Sbryanv error = vmxnet3_alloc_interrupt(sc, rid, flags, 674254738Sbryanv &sc->vmx_intrs[i]); 675254738Sbryanv if (error) 676254738Sbryanv return (error); 677254738Sbryanv } 678254738Sbryanv 679254738Sbryanv return (0); 680254738Sbryanv} 681254738Sbryanv 682254738Sbryanvstatic int 683254738Sbryanvvmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 684254738Sbryanv{ 685254738Sbryanv device_t dev; 686254738Sbryanv struct vmxnet3_txqueue *txq; 687254738Sbryanv struct vmxnet3_rxqueue *rxq; 688254738Sbryanv struct vmxnet3_interrupt *intr; 689254738Sbryanv enum intr_type type; 690254738Sbryanv int i, error; 691254738Sbryanv 692254738Sbryanv dev = sc->vmx_dev; 693254738Sbryanv intr = &sc->vmx_intrs[0]; 694254738Sbryanv type = INTR_TYPE_NET | INTR_MPSAFE; 695254738Sbryanv 696254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 697254738Sbryanv txq = &sc->vmx_txq[i]; 698254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 699254738Sbryanv vmxnet3_txq_intr, txq, &intr->vmxi_handler); 700254738Sbryanv if (error) 701254738Sbryanv return (error); 702268281Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 703268281Sbryanv "tq%d", i); 704254738Sbryanv txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 705254738Sbryanv } 706254738Sbryanv 707254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 708254738Sbryanv rxq = &sc->vmx_rxq[i]; 709254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 710254738Sbryanv vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 711254738Sbryanv if (error) 712254738Sbryanv return (error); 713268281Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 714268281Sbryanv "rq%d", i); 715254738Sbryanv rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 716254738Sbryanv } 717254738Sbryanv 718254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 719254738Sbryanv vmxnet3_event_intr, sc, &intr->vmxi_handler); 720254738Sbryanv if (error) 721254738Sbryanv return (error); 722268281Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 723254738Sbryanv sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 724254738Sbryanv 725254738Sbryanv return (0); 726254738Sbryanv} 727254738Sbryanv 728254738Sbryanvstatic int 729254738Sbryanvvmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 730254738Sbryanv{ 731254738Sbryanv struct vmxnet3_interrupt *intr; 732254738Sbryanv int i, error; 733254738Sbryanv 734254738Sbryanv intr = &sc->vmx_intrs[0]; 735254738Sbryanv error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 736254738Sbryanv INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 737254738Sbryanv &intr->vmxi_handler); 738254738Sbryanv 739254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 740254738Sbryanv sc->vmx_txq[i].vxtxq_intr_idx = 0; 741254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 742254738Sbryanv sc->vmx_rxq[i].vxrxq_intr_idx = 0; 743254738Sbryanv sc->vmx_event_intr_idx = 0; 744254738Sbryanv 745254738Sbryanv return (error); 746254738Sbryanv} 747254738Sbryanv 748254738Sbryanvstatic void 749254738Sbryanvvmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 750254738Sbryanv{ 751254738Sbryanv struct vmxnet3_txqueue *txq; 752254738Sbryanv struct vmxnet3_txq_shared *txs; 753254738Sbryanv struct vmxnet3_rxqueue *rxq; 754254738Sbryanv struct vmxnet3_rxq_shared *rxs; 755254738Sbryanv int i; 756254738Sbryanv 757254738Sbryanv sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 758254738Sbryanv 759254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 760254738Sbryanv txq = &sc->vmx_txq[i]; 761254738Sbryanv txs = txq->vxtxq_ts; 762254738Sbryanv txs->intr_idx = txq->vxtxq_intr_idx; 763254738Sbryanv } 764254738Sbryanv 765254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 766254738Sbryanv rxq = &sc->vmx_rxq[i]; 767254738Sbryanv rxs = rxq->vxrxq_rs; 768254738Sbryanv rxs->intr_idx = rxq->vxrxq_intr_idx; 769254738Sbryanv } 770254738Sbryanv} 771254738Sbryanv 772254738Sbryanvstatic int 773254738Sbryanvvmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 774254738Sbryanv{ 775254738Sbryanv int error; 776254738Sbryanv 777254738Sbryanv error = vmxnet3_alloc_intr_resources(sc); 778254738Sbryanv if (error) 779254738Sbryanv return (error); 780254738Sbryanv 781254738Sbryanv switch (sc->vmx_intr_type) { 782254738Sbryanv case VMXNET3_IT_MSIX: 783254738Sbryanv error = vmxnet3_setup_msix_interrupts(sc); 784254738Sbryanv break; 785254738Sbryanv case VMXNET3_IT_MSI: 786254738Sbryanv case VMXNET3_IT_LEGACY: 787254738Sbryanv error = vmxnet3_setup_legacy_interrupt(sc); 788254738Sbryanv break; 789254738Sbryanv default: 790254738Sbryanv panic("%s: invalid interrupt type %d", __func__, 791254738Sbryanv sc->vmx_intr_type); 792254738Sbryanv } 793254738Sbryanv 794254738Sbryanv if (error == 0) 795254738Sbryanv vmxnet3_set_interrupt_idx(sc); 796254738Sbryanv 797254738Sbryanv return (error); 798254738Sbryanv} 799254738Sbryanv 800254738Sbryanvstatic int 801254738Sbryanvvmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 802254738Sbryanv{ 803254738Sbryanv device_t dev; 804254738Sbryanv uint32_t config; 805254738Sbryanv int error; 806254738Sbryanv 807254738Sbryanv dev = sc->vmx_dev; 808254738Sbryanv config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 809254738Sbryanv 810254738Sbryanv sc->vmx_intr_type = config & 0x03; 811254738Sbryanv sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 812254738Sbryanv 813254738Sbryanv switch (sc->vmx_intr_type) { 814254738Sbryanv case VMXNET3_IT_AUTO: 815254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSIX; 816254738Sbryanv /* FALLTHROUGH */ 817254738Sbryanv case VMXNET3_IT_MSIX: 818254738Sbryanv error = vmxnet3_alloc_msix_interrupts(sc); 819254738Sbryanv if (error == 0) 820254738Sbryanv break; 821254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSI; 822254738Sbryanv /* FALLTHROUGH */ 823254738Sbryanv case VMXNET3_IT_MSI: 824254738Sbryanv error = vmxnet3_alloc_msi_interrupts(sc); 825254738Sbryanv if (error == 0) 826254738Sbryanv break; 827254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_LEGACY; 828254738Sbryanv /* FALLTHROUGH */ 829254738Sbryanv case VMXNET3_IT_LEGACY: 830254738Sbryanv error = vmxnet3_alloc_legacy_interrupts(sc); 831254738Sbryanv if (error == 0) 832254738Sbryanv break; 833254738Sbryanv /* FALLTHROUGH */ 834254738Sbryanv default: 835254738Sbryanv sc->vmx_intr_type = -1; 836254738Sbryanv device_printf(dev, "cannot allocate any interrupt resources\n"); 837254738Sbryanv return (ENXIO); 838254738Sbryanv } 839254738Sbryanv 840254738Sbryanv return (error); 841254738Sbryanv} 842254738Sbryanv 843254738Sbryanvstatic void 844254738Sbryanvvmxnet3_free_interrupt(struct vmxnet3_softc *sc, 845254738Sbryanv struct vmxnet3_interrupt *intr) 846254738Sbryanv{ 847254738Sbryanv device_t dev; 848254738Sbryanv 849254738Sbryanv dev = sc->vmx_dev; 850254738Sbryanv 851254738Sbryanv if (intr->vmxi_handler != NULL) { 852254738Sbryanv bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 853254738Sbryanv intr->vmxi_handler = NULL; 854254738Sbryanv } 855254738Sbryanv 856254738Sbryanv if (intr->vmxi_irq != NULL) { 857254738Sbryanv bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 858254738Sbryanv intr->vmxi_irq); 859254738Sbryanv intr->vmxi_irq = NULL; 860254738Sbryanv intr->vmxi_rid = -1; 861254738Sbryanv } 862254738Sbryanv} 863254738Sbryanv 864254738Sbryanvstatic void 865254738Sbryanvvmxnet3_free_interrupts(struct vmxnet3_softc *sc) 866254738Sbryanv{ 867254738Sbryanv int i; 868254738Sbryanv 869254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 870254738Sbryanv vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 871254738Sbryanv 872254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_MSI || 873254738Sbryanv sc->vmx_intr_type == VMXNET3_IT_MSIX) 874254738Sbryanv pci_release_msi(sc->vmx_dev); 875254738Sbryanv} 876254738Sbryanv 877264866Sbryanv#ifndef VMXNET3_LEGACY_TX 878254738Sbryanvstatic int 879264866Sbryanvvmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 880264866Sbryanv{ 881264866Sbryanv device_t dev; 882264866Sbryanv 883264866Sbryanv dev = sc->vmx_dev; 884264866Sbryanv 885264866Sbryanv sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 886264866Sbryanv taskqueue_thread_enqueue, &sc->vmx_tq); 887264866Sbryanv if (sc->vmx_tq == NULL) 888264866Sbryanv return (ENOMEM); 889264866Sbryanv 890264866Sbryanv return (0); 891264866Sbryanv} 892264866Sbryanv 893264866Sbryanvstatic void 894264866Sbryanvvmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 895264866Sbryanv{ 896264866Sbryanv device_t dev; 897264866Sbryanv int nthreads, error; 898264866Sbryanv 899264866Sbryanv dev = sc->vmx_dev; 900264866Sbryanv 901264866Sbryanv /* 902264866Sbryanv * The taskqueue is typically not frequently used, so a dedicated 903264866Sbryanv * thread for each queue is unnecessary. 904264866Sbryanv */ 905264866Sbryanv nthreads = MAX(1, sc->vmx_ntxqueues / 2); 906264866Sbryanv 907264866Sbryanv /* 908264866Sbryanv * Most drivers just ignore the return value - it only fails 909264866Sbryanv * with ENOMEM so an error is not likely. It is hard for us 910264866Sbryanv * to recover from an error here. 911264866Sbryanv */ 912264866Sbryanv error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 913264866Sbryanv "%s taskq", device_get_nameunit(dev)); 914264866Sbryanv if (error) 915264866Sbryanv device_printf(dev, "failed to start taskqueue: %d", error); 916264866Sbryanv} 917264866Sbryanv 918264866Sbryanvstatic void 919264866Sbryanvvmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 920264866Sbryanv{ 921264866Sbryanv struct vmxnet3_txqueue *txq; 922264866Sbryanv int i; 923264866Sbryanv 924264866Sbryanv if (sc->vmx_tq != NULL) { 925264866Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 926264866Sbryanv txq = &sc->vmx_txq[i]; 927264866Sbryanv taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 928264866Sbryanv } 929264866Sbryanv } 930264866Sbryanv} 931264866Sbryanv 932264866Sbryanvstatic void 933264866Sbryanvvmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 934264866Sbryanv{ 935264866Sbryanv if (sc->vmx_tq != NULL) { 936264866Sbryanv taskqueue_free(sc->vmx_tq); 937264866Sbryanv sc->vmx_tq = NULL; 938264866Sbryanv } 939264866Sbryanv} 940264866Sbryanv#endif 941264866Sbryanv 942264866Sbryanvstatic int 943254738Sbryanvvmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 944254738Sbryanv{ 945254738Sbryanv struct vmxnet3_rxqueue *rxq; 946254738Sbryanv struct vmxnet3_rxring *rxr; 947254738Sbryanv int i; 948254738Sbryanv 949254738Sbryanv rxq = &sc->vmx_rxq[q]; 950254738Sbryanv 951254738Sbryanv snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 952254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 953254738Sbryanv mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 954254738Sbryanv 955254738Sbryanv rxq->vxrxq_sc = sc; 956254738Sbryanv rxq->vxrxq_id = q; 957254738Sbryanv 958254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 959254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 960254738Sbryanv rxr->vxrxr_rid = i; 961254738Sbryanv rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 962254738Sbryanv rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 963254738Sbryanv sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 964254738Sbryanv if (rxr->vxrxr_rxbuf == NULL) 965254738Sbryanv return (ENOMEM); 966254950Sbryanv 967254950Sbryanv rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 968254738Sbryanv } 969254738Sbryanv 970254738Sbryanv return (0); 971254738Sbryanv} 972254738Sbryanv 973254738Sbryanvstatic int 974254738Sbryanvvmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 975254738Sbryanv{ 976254738Sbryanv struct vmxnet3_txqueue *txq; 977254738Sbryanv struct vmxnet3_txring *txr; 978254738Sbryanv 979254738Sbryanv txq = &sc->vmx_txq[q]; 980254738Sbryanv txr = &txq->vxtxq_cmd_ring; 981254738Sbryanv 982254738Sbryanv snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 983254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 984254738Sbryanv mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 985254738Sbryanv 986254738Sbryanv txq->vxtxq_sc = sc; 987254738Sbryanv txq->vxtxq_id = q; 988254738Sbryanv 989254738Sbryanv txr->vxtxr_ndesc = sc->vmx_ntxdescs; 990254738Sbryanv txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 991254738Sbryanv sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 992254738Sbryanv if (txr->vxtxr_txbuf == NULL) 993254738Sbryanv return (ENOMEM); 994254738Sbryanv 995254738Sbryanv txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 996254738Sbryanv 997264866Sbryanv#ifndef VMXNET3_LEGACY_TX 998264866Sbryanv TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 999264866Sbryanv 1000264866Sbryanv txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 1001264866Sbryanv M_NOWAIT, &txq->vxtxq_mtx); 1002264866Sbryanv if (txq->vxtxq_br == NULL) 1003264866Sbryanv return (ENOMEM); 1004264866Sbryanv#endif 1005264866Sbryanv 1006254738Sbryanv return (0); 1007254738Sbryanv} 1008254738Sbryanv 1009254738Sbryanvstatic int 1010254738Sbryanvvmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1011254738Sbryanv{ 1012254738Sbryanv int i, error; 1013254738Sbryanv 1014264866Sbryanv /* 1015264866Sbryanv * Only attempt to create multiple queues if MSIX is available. MSIX is 1016264866Sbryanv * disabled by default because its apparently broken for devices passed 1017264866Sbryanv * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1018264866Sbryanv * must be set to zero for MSIX. This check prevents us from allocating 1019264866Sbryanv * queue structures that we will not use. 1020264866Sbryanv */ 1021264866Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1022264866Sbryanv sc->vmx_max_nrxqueues = 1; 1023264866Sbryanv sc->vmx_max_ntxqueues = 1; 1024264866Sbryanv } 1025264866Sbryanv 1026254738Sbryanv sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1027264866Sbryanv sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1028254738Sbryanv sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1029264866Sbryanv sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1030254738Sbryanv if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1031254738Sbryanv return (ENOMEM); 1032254738Sbryanv 1033264866Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1034254738Sbryanv error = vmxnet3_init_rxq(sc, i); 1035254738Sbryanv if (error) 1036254738Sbryanv return (error); 1037254738Sbryanv } 1038254738Sbryanv 1039264866Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1040254738Sbryanv error = vmxnet3_init_txq(sc, i); 1041254738Sbryanv if (error) 1042254738Sbryanv return (error); 1043254738Sbryanv } 1044254738Sbryanv 1045254738Sbryanv return (0); 1046254738Sbryanv} 1047254738Sbryanv 1048254738Sbryanvstatic void 1049254738Sbryanvvmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1050254738Sbryanv{ 1051254738Sbryanv struct vmxnet3_rxring *rxr; 1052254738Sbryanv int i; 1053254738Sbryanv 1054254738Sbryanv rxq->vxrxq_sc = NULL; 1055254738Sbryanv rxq->vxrxq_id = -1; 1056254738Sbryanv 1057254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1058254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1059254738Sbryanv 1060254738Sbryanv if (rxr->vxrxr_rxbuf != NULL) { 1061254738Sbryanv free(rxr->vxrxr_rxbuf, M_DEVBUF); 1062254738Sbryanv rxr->vxrxr_rxbuf = NULL; 1063254738Sbryanv } 1064254738Sbryanv } 1065254738Sbryanv 1066254738Sbryanv if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1067254738Sbryanv mtx_destroy(&rxq->vxrxq_mtx); 1068254738Sbryanv} 1069254738Sbryanv 1070254738Sbryanvstatic void 1071254738Sbryanvvmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1072254738Sbryanv{ 1073254738Sbryanv struct vmxnet3_txring *txr; 1074254738Sbryanv 1075254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1076254738Sbryanv 1077254738Sbryanv txq->vxtxq_sc = NULL; 1078254738Sbryanv txq->vxtxq_id = -1; 1079254738Sbryanv 1080264866Sbryanv#ifndef VMXNET3_LEGACY_TX 1081264866Sbryanv if (txq->vxtxq_br != NULL) { 1082264866Sbryanv buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1083264866Sbryanv txq->vxtxq_br = NULL; 1084264866Sbryanv } 1085264866Sbryanv#endif 1086264866Sbryanv 1087254738Sbryanv if (txr->vxtxr_txbuf != NULL) { 1088254738Sbryanv free(txr->vxtxr_txbuf, M_DEVBUF); 1089254738Sbryanv txr->vxtxr_txbuf = NULL; 1090254738Sbryanv } 1091254738Sbryanv 1092254738Sbryanv if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1093254738Sbryanv mtx_destroy(&txq->vxtxq_mtx); 1094254738Sbryanv} 1095254738Sbryanv 1096254738Sbryanvstatic void 1097254738Sbryanvvmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1098254738Sbryanv{ 1099254738Sbryanv int i; 1100254738Sbryanv 1101254738Sbryanv if (sc->vmx_rxq != NULL) { 1102264866Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1103254738Sbryanv vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1104254738Sbryanv free(sc->vmx_rxq, M_DEVBUF); 1105254738Sbryanv sc->vmx_rxq = NULL; 1106254738Sbryanv } 1107254738Sbryanv 1108254738Sbryanv if (sc->vmx_txq != NULL) { 1109264866Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1110254738Sbryanv vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1111254738Sbryanv free(sc->vmx_txq, M_DEVBUF); 1112254738Sbryanv sc->vmx_txq = NULL; 1113254738Sbryanv } 1114254738Sbryanv} 1115254738Sbryanv 1116254738Sbryanvstatic int 1117254738Sbryanvvmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1118254738Sbryanv{ 1119254738Sbryanv device_t dev; 1120254738Sbryanv uint8_t *kva; 1121254738Sbryanv size_t size; 1122254738Sbryanv int i, error; 1123254738Sbryanv 1124254738Sbryanv dev = sc->vmx_dev; 1125254738Sbryanv 1126254738Sbryanv size = sizeof(struct vmxnet3_driver_shared); 1127254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1128254738Sbryanv if (error) { 1129254738Sbryanv device_printf(dev, "cannot alloc shared memory\n"); 1130254738Sbryanv return (error); 1131254738Sbryanv } 1132254738Sbryanv sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1133254738Sbryanv 1134254738Sbryanv size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1135254738Sbryanv sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1136254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1137254738Sbryanv if (error) { 1138254738Sbryanv device_printf(dev, "cannot alloc queue shared memory\n"); 1139254738Sbryanv return (error); 1140254738Sbryanv } 1141254738Sbryanv sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1142254738Sbryanv kva = sc->vmx_qs; 1143254738Sbryanv 1144254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1145254738Sbryanv sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1146254738Sbryanv kva += sizeof(struct vmxnet3_txq_shared); 1147254738Sbryanv } 1148254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1149254738Sbryanv sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1150254738Sbryanv kva += sizeof(struct vmxnet3_rxq_shared); 1151254738Sbryanv } 1152254738Sbryanv 1153264866Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1154264866Sbryanv size = sizeof(struct vmxnet3_rss_shared); 1155264866Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1156264866Sbryanv if (error) { 1157264866Sbryanv device_printf(dev, "cannot alloc rss shared memory\n"); 1158264866Sbryanv return (error); 1159264866Sbryanv } 1160264866Sbryanv sc->vmx_rss = 1161264866Sbryanv (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1162264866Sbryanv } 1163264866Sbryanv 1164254738Sbryanv return (0); 1165254738Sbryanv} 1166254738Sbryanv 1167254738Sbryanvstatic void 1168254738Sbryanvvmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1169254738Sbryanv{ 1170254738Sbryanv 1171264866Sbryanv if (sc->vmx_rss != NULL) { 1172264866Sbryanv vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1173264866Sbryanv sc->vmx_rss = NULL; 1174264866Sbryanv } 1175264866Sbryanv 1176254738Sbryanv if (sc->vmx_qs != NULL) { 1177254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1178254738Sbryanv sc->vmx_qs = NULL; 1179254738Sbryanv } 1180254738Sbryanv 1181254738Sbryanv if (sc->vmx_ds != NULL) { 1182254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1183254738Sbryanv sc->vmx_ds = NULL; 1184254738Sbryanv } 1185254738Sbryanv} 1186254738Sbryanv 1187254738Sbryanvstatic int 1188254738Sbryanvvmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1189254738Sbryanv{ 1190254738Sbryanv device_t dev; 1191254738Sbryanv struct vmxnet3_txqueue *txq; 1192254738Sbryanv struct vmxnet3_txring *txr; 1193254738Sbryanv struct vmxnet3_comp_ring *txc; 1194254738Sbryanv size_t descsz, compsz; 1195254738Sbryanv int i, q, error; 1196254738Sbryanv 1197254738Sbryanv dev = sc->vmx_dev; 1198254738Sbryanv 1199254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1200254738Sbryanv txq = &sc->vmx_txq[q]; 1201254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1202254738Sbryanv txc = &txq->vxtxq_comp_ring; 1203254738Sbryanv 1204254738Sbryanv descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1205254738Sbryanv compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1206254738Sbryanv 1207254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1208254738Sbryanv 1, 0, /* alignment, boundary */ 1209254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1210254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1211254738Sbryanv NULL, NULL, /* filter, filterarg */ 1212264866Sbryanv VMXNET3_TX_MAXSIZE, /* maxsize */ 1213254738Sbryanv VMXNET3_TX_MAXSEGS, /* nsegments */ 1214254738Sbryanv VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1215254738Sbryanv 0, /* flags */ 1216254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1217254738Sbryanv &txr->vxtxr_txtag); 1218254738Sbryanv if (error) { 1219254738Sbryanv device_printf(dev, 1220254738Sbryanv "unable to create Tx buffer tag for queue %d\n", q); 1221254738Sbryanv return (error); 1222254738Sbryanv } 1223254738Sbryanv 1224254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1225254738Sbryanv if (error) { 1226254738Sbryanv device_printf(dev, "cannot alloc Tx descriptors for " 1227254738Sbryanv "queue %d error %d\n", q, error); 1228254738Sbryanv return (error); 1229254738Sbryanv } 1230254738Sbryanv txr->vxtxr_txd = 1231254738Sbryanv (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1232254738Sbryanv 1233254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1234254738Sbryanv if (error) { 1235254738Sbryanv device_printf(dev, "cannot alloc Tx comp descriptors " 1236254738Sbryanv "for queue %d error %d\n", q, error); 1237254738Sbryanv return (error); 1238254738Sbryanv } 1239254738Sbryanv txc->vxcr_u.txcd = 1240254738Sbryanv (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1241254738Sbryanv 1242254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1243254738Sbryanv error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1244254738Sbryanv &txr->vxtxr_txbuf[i].vtxb_dmamap); 1245254738Sbryanv if (error) { 1246254738Sbryanv device_printf(dev, "unable to create Tx buf " 1247254738Sbryanv "dmamap for queue %d idx %d\n", q, i); 1248254738Sbryanv return (error); 1249254738Sbryanv } 1250254738Sbryanv } 1251254738Sbryanv } 1252254738Sbryanv 1253254738Sbryanv return (0); 1254254738Sbryanv} 1255254738Sbryanv 1256254738Sbryanvstatic void 1257254738Sbryanvvmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1258254738Sbryanv{ 1259254738Sbryanv device_t dev; 1260254738Sbryanv struct vmxnet3_txqueue *txq; 1261254738Sbryanv struct vmxnet3_txring *txr; 1262254738Sbryanv struct vmxnet3_comp_ring *txc; 1263254738Sbryanv struct vmxnet3_txbuf *txb; 1264254738Sbryanv int i, q; 1265254738Sbryanv 1266254738Sbryanv dev = sc->vmx_dev; 1267254738Sbryanv 1268254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1269254738Sbryanv txq = &sc->vmx_txq[q]; 1270254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1271254738Sbryanv txc = &txq->vxtxq_comp_ring; 1272254738Sbryanv 1273254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1274254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 1275254738Sbryanv if (txb->vtxb_dmamap != NULL) { 1276254738Sbryanv bus_dmamap_destroy(txr->vxtxr_txtag, 1277254738Sbryanv txb->vtxb_dmamap); 1278254738Sbryanv txb->vtxb_dmamap = NULL; 1279254738Sbryanv } 1280254738Sbryanv } 1281254738Sbryanv 1282254738Sbryanv if (txc->vxcr_u.txcd != NULL) { 1283254738Sbryanv vmxnet3_dma_free(sc, &txc->vxcr_dma); 1284254738Sbryanv txc->vxcr_u.txcd = NULL; 1285254738Sbryanv } 1286254738Sbryanv 1287254738Sbryanv if (txr->vxtxr_txd != NULL) { 1288254738Sbryanv vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1289254738Sbryanv txr->vxtxr_txd = NULL; 1290254738Sbryanv } 1291254738Sbryanv 1292254738Sbryanv if (txr->vxtxr_txtag != NULL) { 1293254738Sbryanv bus_dma_tag_destroy(txr->vxtxr_txtag); 1294254738Sbryanv txr->vxtxr_txtag = NULL; 1295254738Sbryanv } 1296254738Sbryanv } 1297254738Sbryanv} 1298254738Sbryanv 1299254738Sbryanvstatic int 1300254738Sbryanvvmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1301254738Sbryanv{ 1302254738Sbryanv device_t dev; 1303254738Sbryanv struct vmxnet3_rxqueue *rxq; 1304254738Sbryanv struct vmxnet3_rxring *rxr; 1305254738Sbryanv struct vmxnet3_comp_ring *rxc; 1306254738Sbryanv int descsz, compsz; 1307254738Sbryanv int i, j, q, error; 1308254738Sbryanv 1309254738Sbryanv dev = sc->vmx_dev; 1310254738Sbryanv 1311254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1312254738Sbryanv rxq = &sc->vmx_rxq[q]; 1313254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1314254738Sbryanv compsz = 0; 1315254738Sbryanv 1316254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1317254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1318254738Sbryanv 1319254738Sbryanv descsz = rxr->vxrxr_ndesc * 1320254738Sbryanv sizeof(struct vmxnet3_rxdesc); 1321254738Sbryanv compsz += rxr->vxrxr_ndesc * 1322254738Sbryanv sizeof(struct vmxnet3_rxcompdesc); 1323254738Sbryanv 1324254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1325254738Sbryanv 1, 0, /* alignment, boundary */ 1326254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1327254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1328254738Sbryanv NULL, NULL, /* filter, filterarg */ 1329254738Sbryanv MJUMPAGESIZE, /* maxsize */ 1330254738Sbryanv 1, /* nsegments */ 1331254738Sbryanv MJUMPAGESIZE, /* maxsegsize */ 1332254738Sbryanv 0, /* flags */ 1333254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1334254738Sbryanv &rxr->vxrxr_rxtag); 1335254738Sbryanv if (error) { 1336254738Sbryanv device_printf(dev, 1337254738Sbryanv "unable to create Rx buffer tag for " 1338254738Sbryanv "queue %d\n", q); 1339254738Sbryanv return (error); 1340254738Sbryanv } 1341254738Sbryanv 1342254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, 1343254738Sbryanv &rxr->vxrxr_dma); 1344254738Sbryanv if (error) { 1345254738Sbryanv device_printf(dev, "cannot allocate Rx " 1346254738Sbryanv "descriptors for queue %d/%d error %d\n", 1347254738Sbryanv i, q, error); 1348254738Sbryanv return (error); 1349254738Sbryanv } 1350254738Sbryanv rxr->vxrxr_rxd = 1351254738Sbryanv (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1352254738Sbryanv } 1353254738Sbryanv 1354254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1355254738Sbryanv if (error) { 1356254738Sbryanv device_printf(dev, "cannot alloc Rx comp descriptors " 1357254738Sbryanv "for queue %d error %d\n", q, error); 1358254738Sbryanv return (error); 1359254738Sbryanv } 1360254738Sbryanv rxc->vxcr_u.rxcd = 1361254738Sbryanv (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1362254738Sbryanv 1363254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1364254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1365254738Sbryanv 1366254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1367254738Sbryanv &rxr->vxrxr_spare_dmap); 1368254738Sbryanv if (error) { 1369254738Sbryanv device_printf(dev, "unable to create spare " 1370254738Sbryanv "dmamap for queue %d/%d error %d\n", 1371254738Sbryanv q, i, error); 1372254738Sbryanv return (error); 1373254738Sbryanv } 1374254738Sbryanv 1375254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1376254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1377254738Sbryanv &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1378254738Sbryanv if (error) { 1379254738Sbryanv device_printf(dev, "unable to create " 1380254738Sbryanv "dmamap for queue %d/%d slot %d " 1381254738Sbryanv "error %d\n", 1382254738Sbryanv q, i, j, error); 1383254738Sbryanv return (error); 1384254738Sbryanv } 1385254738Sbryanv } 1386254738Sbryanv } 1387254738Sbryanv } 1388254738Sbryanv 1389254738Sbryanv return (0); 1390254738Sbryanv} 1391254738Sbryanv 1392254738Sbryanvstatic void 1393254738Sbryanvvmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1394254738Sbryanv{ 1395254738Sbryanv device_t dev; 1396254738Sbryanv struct vmxnet3_rxqueue *rxq; 1397254738Sbryanv struct vmxnet3_rxring *rxr; 1398254738Sbryanv struct vmxnet3_comp_ring *rxc; 1399254738Sbryanv struct vmxnet3_rxbuf *rxb; 1400254738Sbryanv int i, j, q; 1401254738Sbryanv 1402254738Sbryanv dev = sc->vmx_dev; 1403254738Sbryanv 1404254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1405254738Sbryanv rxq = &sc->vmx_rxq[q]; 1406254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1407254738Sbryanv 1408254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1409254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1410254738Sbryanv 1411254738Sbryanv if (rxr->vxrxr_spare_dmap != NULL) { 1412254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1413254738Sbryanv rxr->vxrxr_spare_dmap); 1414254738Sbryanv rxr->vxrxr_spare_dmap = NULL; 1415254738Sbryanv } 1416254738Sbryanv 1417254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1418254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 1419254738Sbryanv if (rxb->vrxb_dmamap != NULL) { 1420254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1421254738Sbryanv rxb->vrxb_dmamap); 1422254738Sbryanv rxb->vrxb_dmamap = NULL; 1423254738Sbryanv } 1424254738Sbryanv } 1425254738Sbryanv } 1426254738Sbryanv 1427254738Sbryanv if (rxc->vxcr_u.rxcd != NULL) { 1428254738Sbryanv vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1429254738Sbryanv rxc->vxcr_u.rxcd = NULL; 1430254738Sbryanv } 1431254738Sbryanv 1432254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1433254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1434254738Sbryanv 1435254738Sbryanv if (rxr->vxrxr_rxd != NULL) { 1436254738Sbryanv vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1437254738Sbryanv rxr->vxrxr_rxd = NULL; 1438254738Sbryanv } 1439254738Sbryanv 1440254738Sbryanv if (rxr->vxrxr_rxtag != NULL) { 1441254738Sbryanv bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1442254738Sbryanv rxr->vxrxr_rxtag = NULL; 1443254738Sbryanv } 1444254738Sbryanv } 1445254738Sbryanv } 1446254738Sbryanv} 1447254738Sbryanv 1448254738Sbryanvstatic int 1449254738Sbryanvvmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1450254738Sbryanv{ 1451254738Sbryanv int error; 1452254738Sbryanv 1453254738Sbryanv error = vmxnet3_alloc_txq_data(sc); 1454254738Sbryanv if (error) 1455254738Sbryanv return (error); 1456254738Sbryanv 1457254738Sbryanv error = vmxnet3_alloc_rxq_data(sc); 1458254738Sbryanv if (error) 1459254738Sbryanv return (error); 1460254738Sbryanv 1461254738Sbryanv return (0); 1462254738Sbryanv} 1463254738Sbryanv 1464254738Sbryanvstatic void 1465254738Sbryanvvmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1466254738Sbryanv{ 1467254738Sbryanv 1468254950Sbryanv if (sc->vmx_rxq != NULL) 1469254950Sbryanv vmxnet3_free_rxq_data(sc); 1470254950Sbryanv 1471254950Sbryanv if (sc->vmx_txq != NULL) 1472254950Sbryanv vmxnet3_free_txq_data(sc); 1473254738Sbryanv} 1474254738Sbryanv 1475254738Sbryanvstatic int 1476254738Sbryanvvmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1477254738Sbryanv{ 1478254738Sbryanv int error; 1479254738Sbryanv 1480254738Sbryanv error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1481254738Sbryanv 32, &sc->vmx_mcast_dma); 1482254738Sbryanv if (error) 1483254738Sbryanv device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1484254738Sbryanv else 1485254738Sbryanv sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1486254738Sbryanv 1487254738Sbryanv return (error); 1488254738Sbryanv} 1489254738Sbryanv 1490254738Sbryanvstatic void 1491254738Sbryanvvmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1492254738Sbryanv{ 1493254738Sbryanv 1494254738Sbryanv if (sc->vmx_mcast != NULL) { 1495254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1496254738Sbryanv sc->vmx_mcast = NULL; 1497254738Sbryanv } 1498254738Sbryanv} 1499254738Sbryanv 1500254738Sbryanvstatic void 1501254738Sbryanvvmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1502254738Sbryanv{ 1503254738Sbryanv struct vmxnet3_driver_shared *ds; 1504254738Sbryanv struct vmxnet3_txqueue *txq; 1505254738Sbryanv struct vmxnet3_txq_shared *txs; 1506254738Sbryanv struct vmxnet3_rxqueue *rxq; 1507254738Sbryanv struct vmxnet3_rxq_shared *rxs; 1508254738Sbryanv int i; 1509254738Sbryanv 1510254738Sbryanv ds = sc->vmx_ds; 1511254738Sbryanv 1512254738Sbryanv /* 1513254738Sbryanv * Initialize fields of the shared data that remains the same across 1514254738Sbryanv * reinits. Note the shared data is zero'd when allocated. 1515254738Sbryanv */ 1516254738Sbryanv 1517254738Sbryanv ds->magic = VMXNET3_REV1_MAGIC; 1518254738Sbryanv 1519254738Sbryanv /* DriverInfo */ 1520254738Sbryanv ds->version = VMXNET3_DRIVER_VERSION; 1521256309Sbryanv ds->guest = VMXNET3_GOS_FREEBSD | 1522254738Sbryanv#ifdef __LP64__ 1523254738Sbryanv VMXNET3_GOS_64BIT; 1524254738Sbryanv#else 1525254738Sbryanv VMXNET3_GOS_32BIT; 1526254738Sbryanv#endif 1527254738Sbryanv ds->vmxnet3_revision = 1; 1528254738Sbryanv ds->upt_version = 1; 1529254738Sbryanv 1530254738Sbryanv /* Misc. conf */ 1531254738Sbryanv ds->driver_data = vtophys(sc); 1532254738Sbryanv ds->driver_data_len = sizeof(struct vmxnet3_softc); 1533254738Sbryanv ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1534254738Sbryanv ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1535254738Sbryanv ds->nrxsg_max = sc->vmx_max_rxsegs; 1536254738Sbryanv 1537264866Sbryanv /* RSS conf */ 1538264866Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1539264866Sbryanv ds->rss.version = 1; 1540264866Sbryanv ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1541264866Sbryanv ds->rss.len = sc->vmx_rss_dma.dma_size; 1542264866Sbryanv } 1543264866Sbryanv 1544254738Sbryanv /* Interrupt control. */ 1545254738Sbryanv ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1546254738Sbryanv ds->nintr = sc->vmx_nintrs; 1547254738Sbryanv ds->evintr = sc->vmx_event_intr_idx; 1548254738Sbryanv ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1549254738Sbryanv 1550254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 1551254738Sbryanv ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1552254738Sbryanv 1553254738Sbryanv /* Receive filter. */ 1554254738Sbryanv ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1555254738Sbryanv ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1556254738Sbryanv 1557254738Sbryanv /* Tx queues */ 1558254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1559254738Sbryanv txq = &sc->vmx_txq[i]; 1560254738Sbryanv txs = txq->vxtxq_ts; 1561254738Sbryanv 1562254738Sbryanv txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1563254950Sbryanv txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1564254738Sbryanv txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1565254950Sbryanv txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1566254738Sbryanv txs->driver_data = vtophys(txq); 1567254738Sbryanv txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1568254738Sbryanv } 1569254738Sbryanv 1570254738Sbryanv /* Rx queues */ 1571254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1572254738Sbryanv rxq = &sc->vmx_rxq[i]; 1573254738Sbryanv rxs = rxq->vxrxq_rs; 1574254738Sbryanv 1575254738Sbryanv rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1576254738Sbryanv rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1577254738Sbryanv rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1578254738Sbryanv rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1579254738Sbryanv rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1580254950Sbryanv rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1581254738Sbryanv rxs->driver_data = vtophys(rxq); 1582254738Sbryanv rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1583254738Sbryanv } 1584254738Sbryanv} 1585254738Sbryanv 1586254738Sbryanvstatic void 1587254738Sbryanvvmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1588254738Sbryanv{ 1589254738Sbryanv struct ifnet *ifp; 1590254738Sbryanv 1591254738Sbryanv ifp = sc->vmx_ifp; 1592254738Sbryanv 1593254738Sbryanv /* Use the current MAC address. */ 1594254738Sbryanv bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1595254738Sbryanv vmxnet3_set_lladdr(sc); 1596254738Sbryanv 1597254738Sbryanv ifp->if_hwassist = 0; 1598254738Sbryanv if (ifp->if_capenable & IFCAP_TXCSUM) 1599254738Sbryanv ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1600254738Sbryanv if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1601254738Sbryanv ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1602254738Sbryanv if (ifp->if_capenable & IFCAP_TSO4) 1603264866Sbryanv ifp->if_hwassist |= CSUM_IP_TSO; 1604254738Sbryanv if (ifp->if_capenable & IFCAP_TSO6) 1605264866Sbryanv ifp->if_hwassist |= CSUM_IP6_TSO; 1606254738Sbryanv} 1607254738Sbryanv 1608254738Sbryanvstatic void 1609264866Sbryanvvmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1610264866Sbryanv{ 1611264866Sbryanv /* 1612264866Sbryanv * Use the same key as the Linux driver until FreeBSD can do 1613264866Sbryanv * RSS (presumably Toeplitz) in software. 1614264866Sbryanv */ 1615264866Sbryanv static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1616264866Sbryanv 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1617264866Sbryanv 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1618264866Sbryanv 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1619264866Sbryanv 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1620264866Sbryanv 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1621264866Sbryanv }; 1622264866Sbryanv 1623264866Sbryanv struct vmxnet3_driver_shared *ds; 1624264866Sbryanv struct vmxnet3_rss_shared *rss; 1625264866Sbryanv int i; 1626264866Sbryanv 1627264866Sbryanv ds = sc->vmx_ds; 1628264866Sbryanv rss = sc->vmx_rss; 1629264866Sbryanv 1630264866Sbryanv rss->hash_type = 1631264866Sbryanv UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1632264866Sbryanv UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1633264866Sbryanv rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1634264866Sbryanv rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1635264866Sbryanv rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1636264866Sbryanv memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1637264866Sbryanv 1638264866Sbryanv for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1639264866Sbryanv rss->ind_table[i] = i % sc->vmx_nrxqueues; 1640264866Sbryanv} 1641264866Sbryanv 1642264866Sbryanvstatic void 1643254738Sbryanvvmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1644254738Sbryanv{ 1645254738Sbryanv struct ifnet *ifp; 1646254738Sbryanv struct vmxnet3_driver_shared *ds; 1647254738Sbryanv 1648254738Sbryanv ifp = sc->vmx_ifp; 1649254738Sbryanv ds = sc->vmx_ds; 1650254738Sbryanv 1651264866Sbryanv ds->mtu = ifp->if_mtu; 1652264866Sbryanv ds->ntxqueue = sc->vmx_ntxqueues; 1653264866Sbryanv ds->nrxqueue = sc->vmx_nrxqueues; 1654264866Sbryanv 1655254738Sbryanv ds->upt_features = 0; 1656255055Sbryanv if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1657255055Sbryanv ds->upt_features |= UPT1_F_CSUM; 1658254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1659254738Sbryanv ds->upt_features |= UPT1_F_VLAN; 1660254738Sbryanv if (ifp->if_capenable & IFCAP_LRO) 1661254738Sbryanv ds->upt_features |= UPT1_F_LRO; 1662254738Sbryanv 1663264866Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1664264866Sbryanv ds->upt_features |= UPT1_F_RSS; 1665264866Sbryanv vmxnet3_reinit_rss_shared_data(sc); 1666264866Sbryanv } 1667254738Sbryanv 1668254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1669254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1670254738Sbryanv (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1671254738Sbryanv} 1672254738Sbryanv 1673254738Sbryanvstatic int 1674254738Sbryanvvmxnet3_alloc_data(struct vmxnet3_softc *sc) 1675254738Sbryanv{ 1676254738Sbryanv int error; 1677254738Sbryanv 1678254738Sbryanv error = vmxnet3_alloc_shared_data(sc); 1679254738Sbryanv if (error) 1680254738Sbryanv return (error); 1681254738Sbryanv 1682254738Sbryanv error = vmxnet3_alloc_queue_data(sc); 1683254738Sbryanv if (error) 1684254738Sbryanv return (error); 1685254738Sbryanv 1686254738Sbryanv error = vmxnet3_alloc_mcast_table(sc); 1687254738Sbryanv if (error) 1688254738Sbryanv return (error); 1689254738Sbryanv 1690254738Sbryanv vmxnet3_init_shared_data(sc); 1691254738Sbryanv 1692254738Sbryanv return (0); 1693254738Sbryanv} 1694254738Sbryanv 1695254738Sbryanvstatic void 1696254738Sbryanvvmxnet3_free_data(struct vmxnet3_softc *sc) 1697254738Sbryanv{ 1698254738Sbryanv 1699254738Sbryanv vmxnet3_free_mcast_table(sc); 1700254738Sbryanv vmxnet3_free_queue_data(sc); 1701254738Sbryanv vmxnet3_free_shared_data(sc); 1702254738Sbryanv} 1703254738Sbryanv 1704254738Sbryanvstatic int 1705254738Sbryanvvmxnet3_setup_interface(struct vmxnet3_softc *sc) 1706254738Sbryanv{ 1707254738Sbryanv device_t dev; 1708254738Sbryanv struct ifnet *ifp; 1709254738Sbryanv 1710254738Sbryanv dev = sc->vmx_dev; 1711254738Sbryanv 1712254738Sbryanv ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1713254738Sbryanv if (ifp == NULL) { 1714254738Sbryanv device_printf(dev, "cannot allocate ifnet structure\n"); 1715254738Sbryanv return (ENOSPC); 1716254738Sbryanv } 1717254738Sbryanv 1718254738Sbryanv if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1719254738Sbryanv#if __FreeBSD_version < 1000025 1720254738Sbryanv ifp->if_baudrate = 1000000000; 1721264866Sbryanv#elif __FreeBSD_version < 1100011 1722264866Sbryanv if_initbaudrate(ifp, IF_Gbps(10)); 1723254738Sbryanv#else 1724264866Sbryanv ifp->if_baudrate = IF_Gbps(10); 1725254738Sbryanv#endif 1726254738Sbryanv ifp->if_softc = sc; 1727254738Sbryanv ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1728254738Sbryanv ifp->if_init = vmxnet3_init; 1729254738Sbryanv ifp->if_ioctl = vmxnet3_ioctl; 1730274043Shselasky ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1731274043Shselasky ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1732274043Shselasky ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1733264866Sbryanv 1734264866Sbryanv#ifdef VMXNET3_LEGACY_TX 1735254738Sbryanv ifp->if_start = vmxnet3_start; 1736254738Sbryanv ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1737254738Sbryanv IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1738254738Sbryanv IFQ_SET_READY(&ifp->if_snd); 1739264866Sbryanv#else 1740264866Sbryanv ifp->if_transmit = vmxnet3_txq_mq_start; 1741264866Sbryanv ifp->if_qflush = vmxnet3_qflush; 1742264866Sbryanv#endif 1743254738Sbryanv 1744254738Sbryanv vmxnet3_get_lladdr(sc); 1745254738Sbryanv ether_ifattach(ifp, sc->vmx_lladdr); 1746254738Sbryanv 1747254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1748254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1749254738Sbryanv ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1750255055Sbryanv ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1751255055Sbryanv IFCAP_VLAN_HWCSUM; 1752254738Sbryanv ifp->if_capenable = ifp->if_capabilities; 1753254738Sbryanv 1754255055Sbryanv /* These capabilities are not enabled by default. */ 1755255055Sbryanv ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1756254738Sbryanv 1757254738Sbryanv sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1758254738Sbryanv vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1759254738Sbryanv sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1760254738Sbryanv vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1761254738Sbryanv 1762254738Sbryanv ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1763254738Sbryanv vmxnet3_media_status); 1764254738Sbryanv ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1765254738Sbryanv ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1766254738Sbryanv 1767254738Sbryanv return (0); 1768254738Sbryanv} 1769254738Sbryanv 1770254738Sbryanvstatic void 1771254738Sbryanvvmxnet3_evintr(struct vmxnet3_softc *sc) 1772254738Sbryanv{ 1773254738Sbryanv device_t dev; 1774254738Sbryanv struct ifnet *ifp; 1775254738Sbryanv struct vmxnet3_txq_shared *ts; 1776254738Sbryanv struct vmxnet3_rxq_shared *rs; 1777254738Sbryanv uint32_t event; 1778254738Sbryanv int reset; 1779254738Sbryanv 1780254738Sbryanv dev = sc->vmx_dev; 1781254738Sbryanv ifp = sc->vmx_ifp; 1782254738Sbryanv reset = 0; 1783254738Sbryanv 1784254738Sbryanv VMXNET3_CORE_LOCK(sc); 1785254738Sbryanv 1786254738Sbryanv /* Clear events. */ 1787254738Sbryanv event = sc->vmx_ds->event; 1788254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1789254738Sbryanv 1790264866Sbryanv if (event & VMXNET3_EVENT_LINK) { 1791254738Sbryanv vmxnet3_link_status(sc); 1792264866Sbryanv if (sc->vmx_link_active != 0) 1793264866Sbryanv vmxnet3_tx_start_all(sc); 1794264866Sbryanv } 1795254738Sbryanv 1796254738Sbryanv if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1797254738Sbryanv reset = 1; 1798254738Sbryanv vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1799254738Sbryanv ts = sc->vmx_txq[0].vxtxq_ts; 1800254738Sbryanv if (ts->stopped != 0) 1801254738Sbryanv device_printf(dev, "Tx queue error %#x\n", ts->error); 1802254738Sbryanv rs = sc->vmx_rxq[0].vxrxq_rs; 1803254738Sbryanv if (rs->stopped != 0) 1804254738Sbryanv device_printf(dev, "Rx queue error %#x\n", rs->error); 1805254738Sbryanv device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1806254738Sbryanv } 1807254738Sbryanv 1808254738Sbryanv if (event & VMXNET3_EVENT_DIC) 1809254738Sbryanv device_printf(dev, "device implementation change event\n"); 1810254738Sbryanv if (event & VMXNET3_EVENT_DEBUG) 1811254738Sbryanv device_printf(dev, "debug event\n"); 1812254738Sbryanv 1813254738Sbryanv if (reset != 0) { 1814254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1815254738Sbryanv vmxnet3_init_locked(sc); 1816254738Sbryanv } 1817254738Sbryanv 1818254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 1819254738Sbryanv} 1820254738Sbryanv 1821254738Sbryanvstatic void 1822254738Sbryanvvmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1823254738Sbryanv{ 1824254738Sbryanv struct vmxnet3_softc *sc; 1825254738Sbryanv struct ifnet *ifp; 1826254738Sbryanv struct vmxnet3_txring *txr; 1827254738Sbryanv struct vmxnet3_comp_ring *txc; 1828254738Sbryanv struct vmxnet3_txcompdesc *txcd; 1829254738Sbryanv struct vmxnet3_txbuf *txb; 1830264866Sbryanv struct mbuf *m; 1831254738Sbryanv u_int sop; 1832254738Sbryanv 1833254738Sbryanv sc = txq->vxtxq_sc; 1834254738Sbryanv ifp = sc->vmx_ifp; 1835254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1836254738Sbryanv txc = &txq->vxtxq_comp_ring; 1837254738Sbryanv 1838254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 1839254738Sbryanv 1840254738Sbryanv for (;;) { 1841254738Sbryanv txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1842254738Sbryanv if (txcd->gen != txc->vxcr_gen) 1843254738Sbryanv break; 1844254950Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1845254738Sbryanv 1846254738Sbryanv if (++txc->vxcr_next == txc->vxcr_ndesc) { 1847254738Sbryanv txc->vxcr_next = 0; 1848254738Sbryanv txc->vxcr_gen ^= 1; 1849254738Sbryanv } 1850254738Sbryanv 1851254738Sbryanv sop = txr->vxtxr_next; 1852254738Sbryanv txb = &txr->vxtxr_txbuf[sop]; 1853254738Sbryanv 1854264866Sbryanv if ((m = txb->vtxb_m) != NULL) { 1855254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1856254738Sbryanv BUS_DMASYNC_POSTWRITE); 1857254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1858254738Sbryanv 1859264866Sbryanv txq->vxtxq_stats.vmtxs_opackets++; 1860264866Sbryanv txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1861264866Sbryanv if (m->m_flags & M_MCAST) 1862264866Sbryanv txq->vxtxq_stats.vmtxs_omcasts++; 1863264866Sbryanv 1864264866Sbryanv m_freem(m); 1865254738Sbryanv txb->vtxb_m = NULL; 1866254738Sbryanv } 1867254738Sbryanv 1868254738Sbryanv txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1869254738Sbryanv } 1870254738Sbryanv 1871254738Sbryanv if (txr->vxtxr_head == txr->vxtxr_next) 1872254738Sbryanv txq->vxtxq_watchdog = 0; 1873254738Sbryanv} 1874254738Sbryanv 1875254738Sbryanvstatic int 1876254738Sbryanvvmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1877254738Sbryanv{ 1878254738Sbryanv struct ifnet *ifp; 1879254738Sbryanv struct mbuf *m; 1880254738Sbryanv struct vmxnet3_rxdesc *rxd; 1881254738Sbryanv struct vmxnet3_rxbuf *rxb; 1882254738Sbryanv bus_dma_tag_t tag; 1883254738Sbryanv bus_dmamap_t dmap; 1884254738Sbryanv bus_dma_segment_t segs[1]; 1885254738Sbryanv int idx, clsize, btype, flags, nsegs, error; 1886254738Sbryanv 1887254738Sbryanv ifp = sc->vmx_ifp; 1888254738Sbryanv tag = rxr->vxrxr_rxtag; 1889254738Sbryanv dmap = rxr->vxrxr_spare_dmap; 1890254738Sbryanv idx = rxr->vxrxr_fill; 1891254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 1892254738Sbryanv rxb = &rxr->vxrxr_rxbuf[idx]; 1893254738Sbryanv 1894254738Sbryanv#ifdef VMXNET3_FAILPOINTS 1895254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1896254738Sbryanv if (rxr->vxrxr_rid != 0) 1897254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1898254738Sbryanv#endif 1899254738Sbryanv 1900254738Sbryanv if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1901254738Sbryanv flags = M_PKTHDR; 1902254738Sbryanv clsize = MCLBYTES; 1903254738Sbryanv btype = VMXNET3_BTYPE_HEAD; 1904254738Sbryanv } else { 1905254738Sbryanv#if __FreeBSD_version < 902001 1906254738Sbryanv /* 1907254738Sbryanv * These mbufs will never be used for the start of a frame. 1908254738Sbryanv * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1909254738Sbryanv * required the mbuf to always be a packet header. Avoid 1910254738Sbryanv * unnecessary mbuf initialization in newer versions where 1911254738Sbryanv * that is not the case. 1912254738Sbryanv */ 1913254738Sbryanv flags = M_PKTHDR; 1914254738Sbryanv#else 1915254738Sbryanv flags = 0; 1916254738Sbryanv#endif 1917254738Sbryanv clsize = MJUMPAGESIZE; 1918254738Sbryanv btype = VMXNET3_BTYPE_BODY; 1919254738Sbryanv } 1920254738Sbryanv 1921254738Sbryanv m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1922254738Sbryanv if (m == NULL) { 1923254738Sbryanv sc->vmx_stats.vmst_mgetcl_failed++; 1924254738Sbryanv return (ENOBUFS); 1925254738Sbryanv } 1926254738Sbryanv 1927254738Sbryanv if (btype == VMXNET3_BTYPE_HEAD) { 1928254738Sbryanv m->m_len = m->m_pkthdr.len = clsize; 1929254738Sbryanv m_adj(m, ETHER_ALIGN); 1930254738Sbryanv } else 1931254738Sbryanv m->m_len = clsize; 1932254738Sbryanv 1933254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1934254738Sbryanv BUS_DMA_NOWAIT); 1935254738Sbryanv if (error) { 1936254738Sbryanv m_freem(m); 1937254950Sbryanv sc->vmx_stats.vmst_mbuf_load_failed++; 1938254738Sbryanv return (error); 1939254738Sbryanv } 1940254738Sbryanv KASSERT(nsegs == 1, 1941254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1942254738Sbryanv#if __FreeBSD_version < 902001 1943254738Sbryanv if (btype == VMXNET3_BTYPE_BODY) 1944254738Sbryanv m->m_flags &= ~M_PKTHDR; 1945254738Sbryanv#endif 1946254738Sbryanv 1947254738Sbryanv if (rxb->vrxb_m != NULL) { 1948254738Sbryanv bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1949254738Sbryanv bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1950254738Sbryanv } 1951254738Sbryanv 1952254738Sbryanv rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1953254738Sbryanv rxb->vrxb_dmamap = dmap; 1954254738Sbryanv rxb->vrxb_m = m; 1955254738Sbryanv 1956254738Sbryanv rxd->addr = segs[0].ds_addr; 1957254738Sbryanv rxd->len = segs[0].ds_len; 1958254738Sbryanv rxd->btype = btype; 1959254738Sbryanv rxd->gen = rxr->vxrxr_gen; 1960254738Sbryanv 1961254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 1962254738Sbryanv return (0); 1963254738Sbryanv} 1964254738Sbryanv 1965254738Sbryanvstatic void 1966254738Sbryanvvmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1967254738Sbryanv struct vmxnet3_rxring *rxr, int idx) 1968254738Sbryanv{ 1969254738Sbryanv struct vmxnet3_rxdesc *rxd; 1970254738Sbryanv 1971254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 1972254738Sbryanv rxd->gen = rxr->vxrxr_gen; 1973254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 1974254738Sbryanv} 1975254738Sbryanv 1976254738Sbryanvstatic void 1977254738Sbryanvvmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1978254738Sbryanv{ 1979254738Sbryanv struct vmxnet3_softc *sc; 1980254738Sbryanv struct vmxnet3_rxring *rxr; 1981254738Sbryanv struct vmxnet3_comp_ring *rxc; 1982254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 1983254738Sbryanv int idx, eof; 1984254738Sbryanv 1985254738Sbryanv sc = rxq->vxrxq_sc; 1986254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1987254738Sbryanv 1988254738Sbryanv do { 1989254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1990254738Sbryanv if (rxcd->gen != rxc->vxcr_gen) 1991254738Sbryanv break; /* Not expected. */ 1992254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1993254738Sbryanv 1994254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1995254738Sbryanv rxc->vxcr_next = 0; 1996254738Sbryanv rxc->vxcr_gen ^= 1; 1997254738Sbryanv } 1998254738Sbryanv 1999254738Sbryanv idx = rxcd->rxd_idx; 2000254738Sbryanv eof = rxcd->eop; 2001254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2002254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2003254738Sbryanv else 2004254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2005254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2006254738Sbryanv } while (!eof); 2007254738Sbryanv} 2008254738Sbryanv 2009254738Sbryanvstatic void 2010254738Sbryanvvmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2011254738Sbryanv{ 2012254738Sbryanv 2013254738Sbryanv if (rxcd->ipv4) { 2014254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2015254738Sbryanv if (rxcd->ipcsum_ok) 2016254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2017254738Sbryanv } 2018254738Sbryanv 2019254738Sbryanv if (!rxcd->fragment) { 2020254738Sbryanv if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2021254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2022254738Sbryanv CSUM_PSEUDO_HDR; 2023254738Sbryanv m->m_pkthdr.csum_data = 0xFFFF; 2024254738Sbryanv } 2025254738Sbryanv } 2026254738Sbryanv} 2027254738Sbryanv 2028254738Sbryanvstatic void 2029254738Sbryanvvmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2030254738Sbryanv struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2031254738Sbryanv{ 2032254738Sbryanv struct vmxnet3_softc *sc; 2033254738Sbryanv struct ifnet *ifp; 2034254738Sbryanv 2035254738Sbryanv sc = rxq->vxrxq_sc; 2036254738Sbryanv ifp = sc->vmx_ifp; 2037254738Sbryanv 2038254738Sbryanv if (rxcd->error) { 2039264866Sbryanv rxq->vxrxq_stats.vmrxs_ierrors++; 2040254738Sbryanv m_freem(m); 2041254738Sbryanv return; 2042254738Sbryanv } 2043254738Sbryanv 2044264866Sbryanv#ifdef notyet 2045264866Sbryanv switch (rxcd->rss_type) { 2046264866Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV4: 2047264866Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2048264866Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2049264866Sbryanv break; 2050264866Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2051264866Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2052264866Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2053264866Sbryanv break; 2054264866Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV6: 2055264866Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2056264866Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2057264866Sbryanv break; 2058264866Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2059264866Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2060264866Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2061264866Sbryanv break; 2062264866Sbryanv default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2063264866Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2064264866Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2065264866Sbryanv break; 2066264866Sbryanv } 2067264866Sbryanv#else 2068264866Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2069281955Shiren M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2070264866Sbryanv#endif 2071264866Sbryanv 2072254738Sbryanv if (!rxcd->no_csum) 2073254738Sbryanv vmxnet3_rx_csum(rxcd, m); 2074254738Sbryanv if (rxcd->vlan) { 2075254738Sbryanv m->m_flags |= M_VLANTAG; 2076254738Sbryanv m->m_pkthdr.ether_vtag = rxcd->vtag; 2077254738Sbryanv } 2078254738Sbryanv 2079264866Sbryanv rxq->vxrxq_stats.vmrxs_ipackets++; 2080264866Sbryanv rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2081264866Sbryanv 2082254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2083254738Sbryanv (*ifp->if_input)(ifp, m); 2084254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2085254738Sbryanv} 2086254738Sbryanv 2087254738Sbryanvstatic void 2088254738Sbryanvvmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2089254738Sbryanv{ 2090254738Sbryanv struct vmxnet3_softc *sc; 2091254738Sbryanv struct ifnet *ifp; 2092254738Sbryanv struct vmxnet3_rxring *rxr; 2093254738Sbryanv struct vmxnet3_comp_ring *rxc; 2094254738Sbryanv struct vmxnet3_rxdesc *rxd; 2095254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 2096254738Sbryanv struct mbuf *m, *m_head, *m_tail; 2097254738Sbryanv int idx, length; 2098254738Sbryanv 2099254738Sbryanv sc = rxq->vxrxq_sc; 2100254738Sbryanv ifp = sc->vmx_ifp; 2101254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2102254738Sbryanv 2103254738Sbryanv VMXNET3_RXQ_LOCK_ASSERT(rxq); 2104254738Sbryanv 2105254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2106254738Sbryanv return; 2107254738Sbryanv 2108268011Sbryanv m_head = rxq->vxrxq_mhead; 2109268011Sbryanv rxq->vxrxq_mhead = NULL; 2110268011Sbryanv m_tail = rxq->vxrxq_mtail; 2111268011Sbryanv rxq->vxrxq_mtail = NULL; 2112268011Sbryanv MPASS(m_head == NULL || m_tail != NULL); 2113268011Sbryanv 2114254738Sbryanv for (;;) { 2115254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2116268011Sbryanv if (rxcd->gen != rxc->vxcr_gen) { 2117268011Sbryanv rxq->vxrxq_mhead = m_head; 2118268011Sbryanv rxq->vxrxq_mtail = m_tail; 2119254738Sbryanv break; 2120268011Sbryanv } 2121254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2122254738Sbryanv 2123254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2124254738Sbryanv rxc->vxcr_next = 0; 2125254738Sbryanv rxc->vxcr_gen ^= 1; 2126254738Sbryanv } 2127254738Sbryanv 2128254738Sbryanv idx = rxcd->rxd_idx; 2129254738Sbryanv length = rxcd->len; 2130254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2131254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2132254738Sbryanv else 2133254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2134254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 2135254738Sbryanv 2136254738Sbryanv m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2137254738Sbryanv KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2138254738Sbryanv __func__, rxcd->qid, idx)); 2139254738Sbryanv 2140254738Sbryanv /* 2141254738Sbryanv * The host may skip descriptors. We detect this when this 2142254738Sbryanv * descriptor does not match the previous fill index. Catch 2143254738Sbryanv * up with the host now. 2144254738Sbryanv */ 2145254738Sbryanv if (__predict_false(rxr->vxrxr_fill != idx)) { 2146254738Sbryanv while (rxr->vxrxr_fill != idx) { 2147254738Sbryanv rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2148254738Sbryanv rxr->vxrxr_gen; 2149254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 2150254738Sbryanv } 2151254738Sbryanv } 2152254738Sbryanv 2153254738Sbryanv if (rxcd->sop) { 2154254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2155254738Sbryanv ("%s: start of frame w/o head buffer", __func__)); 2156254738Sbryanv KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2157254738Sbryanv ("%s: start of frame not in ring 0", __func__)); 2158254738Sbryanv KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2159254738Sbryanv ("%s: start of frame at unexcepted index %d (%d)", 2160254738Sbryanv __func__, idx, sc->vmx_rx_max_chain)); 2161254738Sbryanv KASSERT(m_head == NULL, 2162254738Sbryanv ("%s: duplicate start of frame?", __func__)); 2163254738Sbryanv 2164254738Sbryanv if (length == 0) { 2165254738Sbryanv /* Just ignore this descriptor. */ 2166254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2167254738Sbryanv goto nextp; 2168254738Sbryanv } 2169254738Sbryanv 2170254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2171264866Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2172254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2173254738Sbryanv if (!rxcd->eop) 2174254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2175254738Sbryanv goto nextp; 2176254738Sbryanv } 2177254738Sbryanv 2178254738Sbryanv m->m_pkthdr.rcvif = ifp; 2179254738Sbryanv m->m_pkthdr.len = m->m_len = length; 2180254738Sbryanv m->m_pkthdr.csum_flags = 0; 2181254738Sbryanv m_head = m_tail = m; 2182254738Sbryanv 2183254738Sbryanv } else { 2184254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2185254738Sbryanv ("%s: non start of frame w/o body buffer", __func__)); 2186254738Sbryanv KASSERT(m_head != NULL, 2187254738Sbryanv ("%s: frame not started?", __func__)); 2188254738Sbryanv 2189254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2190264866Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2191254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2192254738Sbryanv if (!rxcd->eop) 2193254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2194254738Sbryanv m_freem(m_head); 2195254738Sbryanv m_head = m_tail = NULL; 2196254738Sbryanv goto nextp; 2197254738Sbryanv } 2198254738Sbryanv 2199254738Sbryanv m->m_len = length; 2200254738Sbryanv m_head->m_pkthdr.len += length; 2201254738Sbryanv m_tail->m_next = m; 2202254738Sbryanv m_tail = m; 2203254738Sbryanv } 2204254738Sbryanv 2205254738Sbryanv if (rxcd->eop) { 2206254738Sbryanv vmxnet3_rxq_input(rxq, rxcd, m_head); 2207254738Sbryanv m_head = m_tail = NULL; 2208254738Sbryanv 2209254738Sbryanv /* Must recheck after dropping the Rx lock. */ 2210254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2211254738Sbryanv break; 2212254738Sbryanv } 2213254738Sbryanv 2214254738Sbryanvnextp: 2215254738Sbryanv if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2216254738Sbryanv int qid = rxcd->qid; 2217254738Sbryanv bus_size_t r; 2218254738Sbryanv 2219254738Sbryanv idx = (idx + 1) % rxr->vxrxr_ndesc; 2220254738Sbryanv if (qid >= sc->vmx_nrxqueues) { 2221254738Sbryanv qid -= sc->vmx_nrxqueues; 2222254738Sbryanv r = VMXNET3_BAR0_RXH2(qid); 2223254738Sbryanv } else 2224254738Sbryanv r = VMXNET3_BAR0_RXH1(qid); 2225254738Sbryanv vmxnet3_write_bar0(sc, r, idx); 2226254738Sbryanv } 2227254738Sbryanv } 2228254738Sbryanv} 2229254738Sbryanv 2230254738Sbryanvstatic void 2231254738Sbryanvvmxnet3_legacy_intr(void *xsc) 2232254738Sbryanv{ 2233254738Sbryanv struct vmxnet3_softc *sc; 2234254738Sbryanv struct vmxnet3_rxqueue *rxq; 2235254738Sbryanv struct vmxnet3_txqueue *txq; 2236254738Sbryanv 2237254738Sbryanv sc = xsc; 2238254738Sbryanv rxq = &sc->vmx_rxq[0]; 2239254738Sbryanv txq = &sc->vmx_txq[0]; 2240254738Sbryanv 2241254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2242254738Sbryanv if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2243254738Sbryanv return; 2244254738Sbryanv } 2245254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2246254738Sbryanv vmxnet3_disable_all_intrs(sc); 2247254738Sbryanv 2248254738Sbryanv if (sc->vmx_ds->event != 0) 2249254738Sbryanv vmxnet3_evintr(sc); 2250254738Sbryanv 2251254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2252254738Sbryanv vmxnet3_rxq_eof(rxq); 2253254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2254254738Sbryanv 2255254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2256254738Sbryanv vmxnet3_txq_eof(txq); 2257264866Sbryanv vmxnet3_txq_start(txq); 2258254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2259254738Sbryanv 2260254738Sbryanv vmxnet3_enable_all_intrs(sc); 2261254738Sbryanv} 2262254738Sbryanv 2263254738Sbryanvstatic void 2264254738Sbryanvvmxnet3_txq_intr(void *xtxq) 2265254738Sbryanv{ 2266254738Sbryanv struct vmxnet3_softc *sc; 2267254738Sbryanv struct vmxnet3_txqueue *txq; 2268254738Sbryanv 2269254738Sbryanv txq = xtxq; 2270254738Sbryanv sc = txq->vxtxq_sc; 2271254738Sbryanv 2272254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2273254738Sbryanv vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2274254738Sbryanv 2275254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2276254738Sbryanv vmxnet3_txq_eof(txq); 2277264866Sbryanv vmxnet3_txq_start(txq); 2278254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2279254738Sbryanv 2280254738Sbryanv vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2281254738Sbryanv} 2282254738Sbryanv 2283254738Sbryanvstatic void 2284254738Sbryanvvmxnet3_rxq_intr(void *xrxq) 2285254738Sbryanv{ 2286254738Sbryanv struct vmxnet3_softc *sc; 2287254738Sbryanv struct vmxnet3_rxqueue *rxq; 2288254738Sbryanv 2289254738Sbryanv rxq = xrxq; 2290254738Sbryanv sc = rxq->vxrxq_sc; 2291254738Sbryanv 2292254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2293254738Sbryanv vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2294254738Sbryanv 2295254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2296254738Sbryanv vmxnet3_rxq_eof(rxq); 2297254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2298254738Sbryanv 2299254738Sbryanv vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2300254738Sbryanv} 2301254738Sbryanv 2302254738Sbryanvstatic void 2303254738Sbryanvvmxnet3_event_intr(void *xsc) 2304254738Sbryanv{ 2305254738Sbryanv struct vmxnet3_softc *sc; 2306254738Sbryanv 2307254738Sbryanv sc = xsc; 2308254738Sbryanv 2309254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2310254738Sbryanv vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2311254738Sbryanv 2312254738Sbryanv if (sc->vmx_ds->event != 0) 2313254738Sbryanv vmxnet3_evintr(sc); 2314254738Sbryanv 2315254738Sbryanv vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2316254738Sbryanv} 2317254738Sbryanv 2318254738Sbryanvstatic void 2319254738Sbryanvvmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2320254738Sbryanv{ 2321254738Sbryanv struct vmxnet3_txring *txr; 2322254738Sbryanv struct vmxnet3_txbuf *txb; 2323254738Sbryanv int i; 2324254738Sbryanv 2325254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2326254738Sbryanv 2327254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 2328254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 2329254738Sbryanv 2330254738Sbryanv if (txb->vtxb_m == NULL) 2331254738Sbryanv continue; 2332254738Sbryanv 2333254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2334254738Sbryanv BUS_DMASYNC_POSTWRITE); 2335254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2336254738Sbryanv m_freem(txb->vtxb_m); 2337254738Sbryanv txb->vtxb_m = NULL; 2338254738Sbryanv } 2339254738Sbryanv} 2340254738Sbryanv 2341254738Sbryanvstatic void 2342254738Sbryanvvmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2343254738Sbryanv{ 2344254738Sbryanv struct vmxnet3_rxring *rxr; 2345254738Sbryanv struct vmxnet3_rxbuf *rxb; 2346254738Sbryanv int i, j; 2347254738Sbryanv 2348268011Sbryanv if (rxq->vxrxq_mhead != NULL) { 2349268011Sbryanv m_freem(rxq->vxrxq_mhead); 2350268011Sbryanv rxq->vxrxq_mhead = NULL; 2351268011Sbryanv rxq->vxrxq_mtail = NULL; 2352268011Sbryanv } 2353268011Sbryanv 2354254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2355254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2356254738Sbryanv 2357254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2358254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 2359254738Sbryanv 2360254738Sbryanv if (rxb->vrxb_m == NULL) 2361254738Sbryanv continue; 2362264866Sbryanv 2363254738Sbryanv bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2364254738Sbryanv BUS_DMASYNC_POSTREAD); 2365254738Sbryanv bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2366254738Sbryanv m_freem(rxb->vrxb_m); 2367254738Sbryanv rxb->vrxb_m = NULL; 2368254738Sbryanv } 2369254738Sbryanv } 2370254738Sbryanv} 2371254738Sbryanv 2372254738Sbryanvstatic void 2373254738Sbryanvvmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2374254738Sbryanv{ 2375254738Sbryanv struct vmxnet3_rxqueue *rxq; 2376254738Sbryanv struct vmxnet3_txqueue *txq; 2377254738Sbryanv int i; 2378254738Sbryanv 2379254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 2380254738Sbryanv rxq = &sc->vmx_rxq[i]; 2381254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2382254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2383254738Sbryanv } 2384254738Sbryanv 2385254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 2386254738Sbryanv txq = &sc->vmx_txq[i]; 2387254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2388254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2389254738Sbryanv } 2390254738Sbryanv} 2391254738Sbryanv 2392254738Sbryanvstatic void 2393254738Sbryanvvmxnet3_stop(struct vmxnet3_softc *sc) 2394254738Sbryanv{ 2395254738Sbryanv struct ifnet *ifp; 2396254738Sbryanv int q; 2397254738Sbryanv 2398254738Sbryanv ifp = sc->vmx_ifp; 2399254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 2400254738Sbryanv 2401254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2402254738Sbryanv sc->vmx_link_active = 0; 2403254738Sbryanv callout_stop(&sc->vmx_tick); 2404254738Sbryanv 2405254738Sbryanv /* Disable interrupts. */ 2406254738Sbryanv vmxnet3_disable_all_intrs(sc); 2407254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2408254738Sbryanv 2409254738Sbryanv vmxnet3_stop_rendezvous(sc); 2410254738Sbryanv 2411254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2412254738Sbryanv vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2413254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) 2414254738Sbryanv vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2415254738Sbryanv 2416254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2417254738Sbryanv} 2418254738Sbryanv 2419254738Sbryanvstatic void 2420254738Sbryanvvmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2421254738Sbryanv{ 2422254738Sbryanv struct vmxnet3_txring *txr; 2423254738Sbryanv struct vmxnet3_comp_ring *txc; 2424254738Sbryanv 2425254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2426254738Sbryanv txr->vxtxr_head = 0; 2427254738Sbryanv txr->vxtxr_next = 0; 2428254738Sbryanv txr->vxtxr_gen = VMXNET3_INIT_GEN; 2429254738Sbryanv bzero(txr->vxtxr_txd, 2430254738Sbryanv txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2431254738Sbryanv 2432254738Sbryanv txc = &txq->vxtxq_comp_ring; 2433254738Sbryanv txc->vxcr_next = 0; 2434254738Sbryanv txc->vxcr_gen = VMXNET3_INIT_GEN; 2435254738Sbryanv bzero(txc->vxcr_u.txcd, 2436254738Sbryanv txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2437254738Sbryanv} 2438254738Sbryanv 2439254738Sbryanvstatic int 2440254738Sbryanvvmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2441254738Sbryanv{ 2442254738Sbryanv struct ifnet *ifp; 2443254738Sbryanv struct vmxnet3_rxring *rxr; 2444254738Sbryanv struct vmxnet3_comp_ring *rxc; 2445254738Sbryanv int i, populate, idx, frame_size, error; 2446254738Sbryanv 2447254738Sbryanv ifp = sc->vmx_ifp; 2448254950Sbryanv frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2449254950Sbryanv ifp->if_mtu; 2450254738Sbryanv 2451254738Sbryanv /* 2452254950Sbryanv * If the MTU causes us to exceed what a regular sized cluster can 2453254950Sbryanv * handle, we allocate a second MJUMPAGESIZE cluster after it in 2454254950Sbryanv * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2455254738Sbryanv * 2456254950Sbryanv * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2457254950Sbryanv * our life easier. We do not support changing the ring size after 2458254950Sbryanv * the attach. 2459254738Sbryanv */ 2460254950Sbryanv if (frame_size <= MCLBYTES) 2461254738Sbryanv sc->vmx_rx_max_chain = 1; 2462254738Sbryanv else 2463254738Sbryanv sc->vmx_rx_max_chain = 2; 2464254738Sbryanv 2465254738Sbryanv /* 2466254738Sbryanv * Only populate ring 1 if the configuration will take advantage 2467254738Sbryanv * of it. That is either when LRO is enabled or the frame size 2468254738Sbryanv * exceeds what ring 0 can contain. 2469254738Sbryanv */ 2470254738Sbryanv if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2471254738Sbryanv frame_size <= MCLBYTES + MJUMPAGESIZE) 2472254738Sbryanv populate = 1; 2473254738Sbryanv else 2474254738Sbryanv populate = VMXNET3_RXRINGS_PERQ; 2475254738Sbryanv 2476254738Sbryanv for (i = 0; i < populate; i++) { 2477254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2478254738Sbryanv rxr->vxrxr_fill = 0; 2479254738Sbryanv rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2480254738Sbryanv bzero(rxr->vxrxr_rxd, 2481254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2482254738Sbryanv 2483254738Sbryanv for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2484254738Sbryanv error = vmxnet3_newbuf(sc, rxr); 2485254738Sbryanv if (error) 2486254738Sbryanv return (error); 2487254738Sbryanv } 2488254738Sbryanv } 2489254738Sbryanv 2490254738Sbryanv for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2491254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2492254738Sbryanv rxr->vxrxr_fill = 0; 2493254738Sbryanv rxr->vxrxr_gen = 0; 2494254738Sbryanv bzero(rxr->vxrxr_rxd, 2495254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2496254738Sbryanv } 2497254738Sbryanv 2498254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2499254738Sbryanv rxc->vxcr_next = 0; 2500254738Sbryanv rxc->vxcr_gen = VMXNET3_INIT_GEN; 2501254738Sbryanv bzero(rxc->vxcr_u.rxcd, 2502254738Sbryanv rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2503254738Sbryanv 2504254738Sbryanv return (0); 2505254738Sbryanv} 2506254738Sbryanv 2507254738Sbryanvstatic int 2508254738Sbryanvvmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2509254738Sbryanv{ 2510254738Sbryanv device_t dev; 2511254738Sbryanv int q, error; 2512254738Sbryanv 2513254738Sbryanv dev = sc->vmx_dev; 2514254738Sbryanv 2515254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2516254738Sbryanv vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2517254738Sbryanv 2518254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2519254738Sbryanv error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2520254738Sbryanv if (error) { 2521254738Sbryanv device_printf(dev, "cannot populate Rx queue %d\n", q); 2522254738Sbryanv return (error); 2523254738Sbryanv } 2524254738Sbryanv } 2525254738Sbryanv 2526254738Sbryanv return (0); 2527254738Sbryanv} 2528254738Sbryanv 2529254738Sbryanvstatic int 2530254738Sbryanvvmxnet3_enable_device(struct vmxnet3_softc *sc) 2531254738Sbryanv{ 2532254738Sbryanv int q; 2533254738Sbryanv 2534254738Sbryanv if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2535254738Sbryanv device_printf(sc->vmx_dev, "device enable command failed!\n"); 2536254738Sbryanv return (1); 2537254738Sbryanv } 2538254738Sbryanv 2539254738Sbryanv /* Reset the Rx queue heads. */ 2540254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2541254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2542254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2543254738Sbryanv } 2544254738Sbryanv 2545254738Sbryanv return (0); 2546254738Sbryanv} 2547254738Sbryanv 2548254738Sbryanvstatic void 2549254738Sbryanvvmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2550254738Sbryanv{ 2551254738Sbryanv struct ifnet *ifp; 2552254738Sbryanv 2553254738Sbryanv ifp = sc->vmx_ifp; 2554254738Sbryanv 2555254738Sbryanv vmxnet3_set_rxfilter(sc); 2556254738Sbryanv 2557254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2558254738Sbryanv bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2559254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2560254738Sbryanv else 2561254738Sbryanv bzero(sc->vmx_ds->vlan_filter, 2562254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2563254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2564254738Sbryanv} 2565254738Sbryanv 2566254738Sbryanvstatic int 2567254738Sbryanvvmxnet3_reinit(struct vmxnet3_softc *sc) 2568254738Sbryanv{ 2569254738Sbryanv 2570254738Sbryanv vmxnet3_reinit_interface(sc); 2571254738Sbryanv vmxnet3_reinit_shared_data(sc); 2572254738Sbryanv 2573254738Sbryanv if (vmxnet3_reinit_queues(sc) != 0) 2574254738Sbryanv return (ENXIO); 2575254738Sbryanv 2576254738Sbryanv if (vmxnet3_enable_device(sc) != 0) 2577254738Sbryanv return (ENXIO); 2578254738Sbryanv 2579254738Sbryanv vmxnet3_reinit_rxfilters(sc); 2580254738Sbryanv 2581254738Sbryanv return (0); 2582254738Sbryanv} 2583254738Sbryanv 2584254738Sbryanvstatic void 2585254738Sbryanvvmxnet3_init_locked(struct vmxnet3_softc *sc) 2586254738Sbryanv{ 2587254738Sbryanv struct ifnet *ifp; 2588254738Sbryanv 2589254738Sbryanv ifp = sc->vmx_ifp; 2590254738Sbryanv 2591254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2592254738Sbryanv return; 2593254738Sbryanv 2594254738Sbryanv vmxnet3_stop(sc); 2595254738Sbryanv 2596254738Sbryanv if (vmxnet3_reinit(sc) != 0) { 2597254738Sbryanv vmxnet3_stop(sc); 2598254738Sbryanv return; 2599254738Sbryanv } 2600254738Sbryanv 2601254738Sbryanv ifp->if_drv_flags |= IFF_DRV_RUNNING; 2602254738Sbryanv vmxnet3_link_status(sc); 2603254738Sbryanv 2604254738Sbryanv vmxnet3_enable_all_intrs(sc); 2605254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2606254738Sbryanv} 2607254738Sbryanv 2608254738Sbryanvstatic void 2609254738Sbryanvvmxnet3_init(void *xsc) 2610254738Sbryanv{ 2611254738Sbryanv struct vmxnet3_softc *sc; 2612254738Sbryanv 2613254738Sbryanv sc = xsc; 2614254738Sbryanv 2615254738Sbryanv VMXNET3_CORE_LOCK(sc); 2616254738Sbryanv vmxnet3_init_locked(sc); 2617254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 2618254738Sbryanv} 2619254738Sbryanv 2620254738Sbryanv/* 2621254738Sbryanv * BMV: Much of this can go away once we finally have offsets in 2622254738Sbryanv * the mbuf packet header. Bug andre@. 2623254738Sbryanv */ 2624254738Sbryanvstatic int 2625264866Sbryanvvmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2626264866Sbryanv int *etype, int *proto, int *start) 2627254738Sbryanv{ 2628254738Sbryanv struct ether_vlan_header *evh; 2629254738Sbryanv int offset; 2630267382Sbryanv#if defined(INET) 2631270739Sbryanv struct ip *ip = NULL; 2632270739Sbryanv struct ip iphdr; 2633267382Sbryanv#endif 2634267382Sbryanv#if defined(INET6) 2635270739Sbryanv struct ip6_hdr *ip6 = NULL; 2636270739Sbryanv struct ip6_hdr ip6hdr; 2637267382Sbryanv#endif 2638254738Sbryanv 2639254738Sbryanv evh = mtod(m, struct ether_vlan_header *); 2640254738Sbryanv if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2641254738Sbryanv /* BMV: We should handle nested VLAN tags too. */ 2642254738Sbryanv *etype = ntohs(evh->evl_proto); 2643254738Sbryanv offset = sizeof(struct ether_vlan_header); 2644254738Sbryanv } else { 2645254738Sbryanv *etype = ntohs(evh->evl_encap_proto); 2646254738Sbryanv offset = sizeof(struct ether_header); 2647254738Sbryanv } 2648254738Sbryanv 2649254738Sbryanv switch (*etype) { 2650254738Sbryanv#if defined(INET) 2651267382Sbryanv case ETHERTYPE_IP: 2652254738Sbryanv if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2653254738Sbryanv m_copydata(m, offset, sizeof(struct ip), 2654254738Sbryanv (caddr_t) &iphdr); 2655254738Sbryanv ip = &iphdr; 2656254738Sbryanv } else 2657264866Sbryanv ip = mtodo(m, offset); 2658254738Sbryanv *proto = ip->ip_p; 2659254738Sbryanv *start = offset + (ip->ip_hl << 2); 2660254738Sbryanv break; 2661254738Sbryanv#endif 2662254738Sbryanv#if defined(INET6) 2663254738Sbryanv case ETHERTYPE_IPV6: 2664267382Sbryanv if (__predict_false(m->m_len < 2665267382Sbryanv offset + sizeof(struct ip6_hdr))) { 2666267382Sbryanv m_copydata(m, offset, sizeof(struct ip6_hdr), 2667267382Sbryanv (caddr_t) &ip6hdr); 2668267382Sbryanv ip6 = &ip6hdr; 2669267382Sbryanv } else 2670267382Sbryanv ip6 = mtodo(m, offset); 2671254738Sbryanv *proto = -1; 2672254738Sbryanv *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2673254738Sbryanv /* Assert the network stack sent us a valid packet. */ 2674254738Sbryanv KASSERT(*start > offset, 2675254738Sbryanv ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2676254738Sbryanv *start, offset, *proto)); 2677254738Sbryanv break; 2678254738Sbryanv#endif 2679254738Sbryanv default: 2680254738Sbryanv return (EINVAL); 2681254738Sbryanv } 2682254738Sbryanv 2683254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2684254738Sbryanv struct tcphdr *tcp, tcphdr; 2685267382Sbryanv uint16_t sum; 2686254738Sbryanv 2687254738Sbryanv if (__predict_false(*proto != IPPROTO_TCP)) { 2688254738Sbryanv /* Likely failed to correctly parse the mbuf. */ 2689254738Sbryanv return (EINVAL); 2690254738Sbryanv } 2691254738Sbryanv 2692264866Sbryanv txq->vxtxq_stats.vmtxs_tso++; 2693264866Sbryanv 2694267382Sbryanv switch (*etype) { 2695267382Sbryanv#if defined(INET) 2696267382Sbryanv case ETHERTYPE_IP: 2697267382Sbryanv sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2698267382Sbryanv htons(IPPROTO_TCP)); 2699267382Sbryanv break; 2700267382Sbryanv#endif 2701267382Sbryanv#if defined(INET6) 2702267382Sbryanv case ETHERTYPE_IPV6: 2703267382Sbryanv sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2704267382Sbryanv break; 2705267382Sbryanv#endif 2706267382Sbryanv default: 2707267382Sbryanv sum = 0; 2708267382Sbryanv break; 2709267382Sbryanv } 2710267382Sbryanv 2711267382Sbryanv if (m->m_len < *start + sizeof(struct tcphdr)) { 2712267382Sbryanv m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2713267382Sbryanv sizeof(uint16_t), (caddr_t) &sum); 2714267382Sbryanv m_copydata(m, *start, sizeof(struct tcphdr), 2715267382Sbryanv (caddr_t) &tcphdr); 2716267382Sbryanv tcp = &tcphdr; 2717267382Sbryanv } else { 2718267382Sbryanv tcp = mtodo(m, *start); 2719267382Sbryanv tcp->th_sum = sum; 2720267382Sbryanv } 2721267382Sbryanv 2722264866Sbryanv /* 2723264866Sbryanv * For TSO, the size of the protocol header is also 2724264866Sbryanv * included in the descriptor header size. 2725264866Sbryanv */ 2726254738Sbryanv *start += (tcp->th_off << 2); 2727264866Sbryanv } else 2728264866Sbryanv txq->vxtxq_stats.vmtxs_csum++; 2729254738Sbryanv 2730254738Sbryanv return (0); 2731254738Sbryanv} 2732254738Sbryanv 2733254738Sbryanvstatic int 2734254738Sbryanvvmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2735254738Sbryanv bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2736254738Sbryanv{ 2737254738Sbryanv struct vmxnet3_txring *txr; 2738254738Sbryanv struct mbuf *m; 2739254738Sbryanv bus_dma_tag_t tag; 2740264866Sbryanv int error; 2741254738Sbryanv 2742254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2743254738Sbryanv m = *m0; 2744254738Sbryanv tag = txr->vxtxr_txtag; 2745254738Sbryanv 2746254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2747254738Sbryanv if (error == 0 || error != EFBIG) 2748254738Sbryanv return (error); 2749254738Sbryanv 2750264866Sbryanv m = m_defrag(m, M_NOWAIT); 2751254738Sbryanv if (m != NULL) { 2752254738Sbryanv *m0 = m; 2753254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2754254738Sbryanv } else 2755254738Sbryanv error = ENOBUFS; 2756254738Sbryanv 2757254738Sbryanv if (error) { 2758254738Sbryanv m_freem(*m0); 2759254738Sbryanv *m0 = NULL; 2760264866Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2761254738Sbryanv } else 2762264866Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2763254738Sbryanv 2764254738Sbryanv return (error); 2765254738Sbryanv} 2766254738Sbryanv 2767254738Sbryanvstatic void 2768254738Sbryanvvmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2769254738Sbryanv{ 2770254738Sbryanv struct vmxnet3_txring *txr; 2771254738Sbryanv 2772254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2773254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2774254738Sbryanv} 2775254738Sbryanv 2776254738Sbryanvstatic int 2777254738Sbryanvvmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2778254738Sbryanv{ 2779254738Sbryanv struct vmxnet3_softc *sc; 2780254738Sbryanv struct vmxnet3_txring *txr; 2781254738Sbryanv struct vmxnet3_txdesc *txd, *sop; 2782254738Sbryanv struct mbuf *m; 2783254738Sbryanv bus_dmamap_t dmap; 2784254738Sbryanv bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2785254738Sbryanv int i, gen, nsegs, etype, proto, start, error; 2786254738Sbryanv 2787254738Sbryanv sc = txq->vxtxq_sc; 2788254738Sbryanv start = 0; 2789254738Sbryanv txd = NULL; 2790254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2791254738Sbryanv dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2792254738Sbryanv 2793254738Sbryanv error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2794254738Sbryanv if (error) 2795254738Sbryanv return (error); 2796254738Sbryanv 2797254738Sbryanv m = *m0; 2798254738Sbryanv M_ASSERTPKTHDR(m); 2799254738Sbryanv KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2800254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2801254738Sbryanv 2802254738Sbryanv if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2803264866Sbryanv txq->vxtxq_stats.vmtxs_full++; 2804254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2805254738Sbryanv return (ENOSPC); 2806254738Sbryanv } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2807264866Sbryanv error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2808254738Sbryanv if (error) { 2809264866Sbryanv txq->vxtxq_stats.vmtxs_offload_failed++; 2810254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2811254738Sbryanv m_freem(m); 2812254738Sbryanv *m0 = NULL; 2813254738Sbryanv return (error); 2814254738Sbryanv } 2815254738Sbryanv } 2816254738Sbryanv 2817267382Sbryanv txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2818254738Sbryanv sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2819254738Sbryanv gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2820254738Sbryanv 2821254738Sbryanv for (i = 0; i < nsegs; i++) { 2822254738Sbryanv txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2823254738Sbryanv 2824254738Sbryanv txd->addr = segs[i].ds_addr; 2825254738Sbryanv txd->len = segs[i].ds_len; 2826254738Sbryanv txd->gen = gen; 2827254738Sbryanv txd->dtype = 0; 2828254738Sbryanv txd->offload_mode = VMXNET3_OM_NONE; 2829254738Sbryanv txd->offload_pos = 0; 2830254738Sbryanv txd->hlen = 0; 2831254738Sbryanv txd->eop = 0; 2832254738Sbryanv txd->compreq = 0; 2833254738Sbryanv txd->vtag_mode = 0; 2834254738Sbryanv txd->vtag = 0; 2835254738Sbryanv 2836254738Sbryanv if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2837254738Sbryanv txr->vxtxr_head = 0; 2838254738Sbryanv txr->vxtxr_gen ^= 1; 2839254738Sbryanv } 2840254738Sbryanv gen = txr->vxtxr_gen; 2841254738Sbryanv } 2842254738Sbryanv txd->eop = 1; 2843254738Sbryanv txd->compreq = 1; 2844254738Sbryanv 2845254738Sbryanv if (m->m_flags & M_VLANTAG) { 2846254738Sbryanv sop->vtag_mode = 1; 2847254738Sbryanv sop->vtag = m->m_pkthdr.ether_vtag; 2848254738Sbryanv } 2849254738Sbryanv 2850254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2851254738Sbryanv sop->offload_mode = VMXNET3_OM_TSO; 2852254738Sbryanv sop->hlen = start; 2853254738Sbryanv sop->offload_pos = m->m_pkthdr.tso_segsz; 2854254738Sbryanv } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2855254738Sbryanv VMXNET3_CSUM_OFFLOAD_IPV6)) { 2856254738Sbryanv sop->offload_mode = VMXNET3_OM_CSUM; 2857254738Sbryanv sop->hlen = start; 2858254738Sbryanv sop->offload_pos = start + m->m_pkthdr.csum_data; 2859254738Sbryanv } 2860254738Sbryanv 2861254738Sbryanv /* Finally, change the ownership. */ 2862254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2863254738Sbryanv sop->gen ^= 1; 2864254738Sbryanv 2865268011Sbryanv txq->vxtxq_ts->npending += nsegs; 2866268011Sbryanv if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2867254738Sbryanv txq->vxtxq_ts->npending = 0; 2868254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2869254738Sbryanv txr->vxtxr_head); 2870254738Sbryanv } 2871254738Sbryanv 2872254738Sbryanv return (0); 2873254738Sbryanv} 2874254738Sbryanv 2875264866Sbryanv#ifdef VMXNET3_LEGACY_TX 2876264866Sbryanv 2877264866Sbryanvstatic void 2878254738Sbryanvvmxnet3_start_locked(struct ifnet *ifp) 2879254738Sbryanv{ 2880254738Sbryanv struct vmxnet3_softc *sc; 2881254738Sbryanv struct vmxnet3_txqueue *txq; 2882254738Sbryanv struct vmxnet3_txring *txr; 2883254738Sbryanv struct mbuf *m_head; 2884255055Sbryanv int tx, avail; 2885254738Sbryanv 2886254738Sbryanv sc = ifp->if_softc; 2887254738Sbryanv txq = &sc->vmx_txq[0]; 2888254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2889254738Sbryanv tx = 0; 2890254738Sbryanv 2891254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 2892254738Sbryanv 2893254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2894254738Sbryanv sc->vmx_link_active == 0) 2895254738Sbryanv return; 2896254738Sbryanv 2897255055Sbryanv while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2898255055Sbryanv if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2899255055Sbryanv break; 2900255055Sbryanv 2901254738Sbryanv IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2902254738Sbryanv if (m_head == NULL) 2903254738Sbryanv break; 2904254738Sbryanv 2905255055Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 2906255055Sbryanv if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2907255055Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2908255055Sbryanv break; 2909255055Sbryanv } 2910255055Sbryanv 2911254738Sbryanv if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2912254738Sbryanv if (m_head != NULL) 2913254738Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2914254738Sbryanv break; 2915254738Sbryanv } 2916254738Sbryanv 2917254738Sbryanv tx++; 2918254738Sbryanv ETHER_BPF_MTAP(ifp, m_head); 2919254738Sbryanv } 2920254738Sbryanv 2921268011Sbryanv if (tx > 0) 2922254738Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2923254738Sbryanv} 2924254738Sbryanv 2925254738Sbryanvstatic void 2926254738Sbryanvvmxnet3_start(struct ifnet *ifp) 2927254738Sbryanv{ 2928254738Sbryanv struct vmxnet3_softc *sc; 2929254738Sbryanv struct vmxnet3_txqueue *txq; 2930254738Sbryanv 2931254738Sbryanv sc = ifp->if_softc; 2932254738Sbryanv txq = &sc->vmx_txq[0]; 2933254738Sbryanv 2934254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2935254738Sbryanv vmxnet3_start_locked(ifp); 2936254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2937254738Sbryanv} 2938254738Sbryanv 2939264866Sbryanv#else /* !VMXNET3_LEGACY_TX */ 2940264866Sbryanv 2941264866Sbryanvstatic int 2942264866Sbryanvvmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2943264866Sbryanv{ 2944264866Sbryanv struct vmxnet3_softc *sc; 2945264866Sbryanv struct vmxnet3_txring *txr; 2946264866Sbryanv struct buf_ring *br; 2947264866Sbryanv struct ifnet *ifp; 2948264866Sbryanv int tx, avail, error; 2949264866Sbryanv 2950264866Sbryanv sc = txq->vxtxq_sc; 2951264866Sbryanv br = txq->vxtxq_br; 2952264866Sbryanv ifp = sc->vmx_ifp; 2953264866Sbryanv txr = &txq->vxtxq_cmd_ring; 2954264866Sbryanv tx = 0; 2955264866Sbryanv error = 0; 2956264866Sbryanv 2957264866Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 2958264866Sbryanv 2959264866Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2960264866Sbryanv sc->vmx_link_active == 0) { 2961264866Sbryanv if (m != NULL) 2962264866Sbryanv error = drbr_enqueue(ifp, br, m); 2963264866Sbryanv return (error); 2964264866Sbryanv } 2965264866Sbryanv 2966264866Sbryanv if (m != NULL) { 2967264866Sbryanv error = drbr_enqueue(ifp, br, m); 2968264866Sbryanv if (error) 2969264866Sbryanv return (error); 2970264866Sbryanv } 2971264866Sbryanv 2972264866Sbryanv while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2973264866Sbryanv m = drbr_peek(ifp, br); 2974264866Sbryanv if (m == NULL) 2975264866Sbryanv break; 2976264866Sbryanv 2977264866Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 2978264866Sbryanv if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2979264866Sbryanv drbr_putback(ifp, br, m); 2980264866Sbryanv break; 2981264866Sbryanv } 2982264866Sbryanv 2983268011Sbryanv if (vmxnet3_txq_encap(txq, &m) != 0) { 2984264866Sbryanv if (m != NULL) 2985264866Sbryanv drbr_putback(ifp, br, m); 2986264866Sbryanv else 2987264866Sbryanv drbr_advance(ifp, br); 2988264866Sbryanv break; 2989264866Sbryanv } 2990264866Sbryanv drbr_advance(ifp, br); 2991264866Sbryanv 2992264866Sbryanv tx++; 2993264866Sbryanv ETHER_BPF_MTAP(ifp, m); 2994264866Sbryanv } 2995264866Sbryanv 2996268011Sbryanv if (tx > 0) 2997264866Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2998264866Sbryanv 2999267279Sluigi return (0); 3000264866Sbryanv} 3001264866Sbryanv 3002264866Sbryanvstatic int 3003264866Sbryanvvmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 3004264866Sbryanv{ 3005264866Sbryanv struct vmxnet3_softc *sc; 3006264866Sbryanv struct vmxnet3_txqueue *txq; 3007264866Sbryanv int i, ntxq, error; 3008264866Sbryanv 3009264866Sbryanv sc = ifp->if_softc; 3010264866Sbryanv ntxq = sc->vmx_ntxqueues; 3011264866Sbryanv 3012281955Shiren /* check if flowid is set */ 3013281955Shiren if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3014264866Sbryanv i = m->m_pkthdr.flowid % ntxq; 3015264866Sbryanv else 3016264866Sbryanv i = curcpu % ntxq; 3017264866Sbryanv 3018264866Sbryanv txq = &sc->vmx_txq[i]; 3019264866Sbryanv 3020264866Sbryanv if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3021264866Sbryanv error = vmxnet3_txq_mq_start_locked(txq, m); 3022264866Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3023264866Sbryanv } else { 3024264866Sbryanv error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3025264866Sbryanv taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3026264866Sbryanv } 3027264866Sbryanv 3028264866Sbryanv return (error); 3029264866Sbryanv} 3030264866Sbryanv 3031254738Sbryanvstatic void 3032264866Sbryanvvmxnet3_txq_tq_deferred(void *xtxq, int pending) 3033264866Sbryanv{ 3034264866Sbryanv struct vmxnet3_softc *sc; 3035264866Sbryanv struct vmxnet3_txqueue *txq; 3036264866Sbryanv 3037264866Sbryanv txq = xtxq; 3038264866Sbryanv sc = txq->vxtxq_sc; 3039264866Sbryanv 3040264866Sbryanv VMXNET3_TXQ_LOCK(txq); 3041264866Sbryanv if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3042264866Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3043264866Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3044264866Sbryanv} 3045264866Sbryanv 3046264866Sbryanv#endif /* VMXNET3_LEGACY_TX */ 3047264866Sbryanv 3048264866Sbryanvstatic void 3049264866Sbryanvvmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3050264866Sbryanv{ 3051264866Sbryanv struct vmxnet3_softc *sc; 3052264866Sbryanv struct ifnet *ifp; 3053264866Sbryanv 3054264866Sbryanv sc = txq->vxtxq_sc; 3055264866Sbryanv ifp = sc->vmx_ifp; 3056264866Sbryanv 3057264866Sbryanv#ifdef VMXNET3_LEGACY_TX 3058264866Sbryanv if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3059264866Sbryanv vmxnet3_start_locked(ifp); 3060264866Sbryanv#else 3061264866Sbryanv if (!drbr_empty(ifp, txq->vxtxq_br)) 3062264866Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3063264866Sbryanv#endif 3064264866Sbryanv} 3065264866Sbryanv 3066264866Sbryanvstatic void 3067264866Sbryanvvmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3068264866Sbryanv{ 3069264866Sbryanv struct vmxnet3_txqueue *txq; 3070264866Sbryanv int i; 3071264866Sbryanv 3072264866Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3073264866Sbryanv 3074264866Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3075264866Sbryanv txq = &sc->vmx_txq[i]; 3076264866Sbryanv 3077264866Sbryanv VMXNET3_TXQ_LOCK(txq); 3078264866Sbryanv vmxnet3_txq_start(txq); 3079264866Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3080264866Sbryanv } 3081264866Sbryanv} 3082264866Sbryanv 3083264866Sbryanvstatic void 3084254738Sbryanvvmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3085254738Sbryanv{ 3086254738Sbryanv struct ifnet *ifp; 3087254738Sbryanv int idx, bit; 3088254738Sbryanv 3089254738Sbryanv ifp = sc->vmx_ifp; 3090254738Sbryanv idx = (tag >> 5) & 0x7F; 3091254738Sbryanv bit = tag & 0x1F; 3092254738Sbryanv 3093254738Sbryanv if (tag == 0 || tag > 4095) 3094254738Sbryanv return; 3095254738Sbryanv 3096254738Sbryanv VMXNET3_CORE_LOCK(sc); 3097254738Sbryanv 3098254738Sbryanv /* Update our private VLAN bitvector. */ 3099254738Sbryanv if (add) 3100254738Sbryanv sc->vmx_vlan_filter[idx] |= (1 << bit); 3101254738Sbryanv else 3102254738Sbryanv sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3103254738Sbryanv 3104254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3105254738Sbryanv if (add) 3106254738Sbryanv sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3107254738Sbryanv else 3108254738Sbryanv sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3109254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3110254738Sbryanv } 3111254738Sbryanv 3112254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3113254738Sbryanv} 3114254738Sbryanv 3115254738Sbryanvstatic void 3116254738Sbryanvvmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3117254738Sbryanv{ 3118254738Sbryanv 3119254738Sbryanv if (ifp->if_softc == arg) 3120254738Sbryanv vmxnet3_update_vlan_filter(arg, 1, tag); 3121254738Sbryanv} 3122254738Sbryanv 3123254738Sbryanvstatic void 3124254738Sbryanvvmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3125254738Sbryanv{ 3126254738Sbryanv 3127254738Sbryanv if (ifp->if_softc == arg) 3128254738Sbryanv vmxnet3_update_vlan_filter(arg, 0, tag); 3129254738Sbryanv} 3130254738Sbryanv 3131254738Sbryanvstatic void 3132254738Sbryanvvmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3133254738Sbryanv{ 3134254738Sbryanv struct ifnet *ifp; 3135254738Sbryanv struct vmxnet3_driver_shared *ds; 3136254738Sbryanv struct ifmultiaddr *ifma; 3137254738Sbryanv u_int mode; 3138254738Sbryanv 3139254738Sbryanv ifp = sc->vmx_ifp; 3140254738Sbryanv ds = sc->vmx_ds; 3141254738Sbryanv 3142264866Sbryanv mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3143254738Sbryanv if (ifp->if_flags & IFF_PROMISC) 3144254738Sbryanv mode |= VMXNET3_RXMODE_PROMISC; 3145254738Sbryanv if (ifp->if_flags & IFF_ALLMULTI) 3146254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3147254738Sbryanv else { 3148254738Sbryanv int cnt = 0, overflow = 0; 3149254738Sbryanv 3150254738Sbryanv if_maddr_rlock(ifp); 3151254738Sbryanv TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3152254738Sbryanv if (ifma->ifma_addr->sa_family != AF_LINK) 3153254738Sbryanv continue; 3154254738Sbryanv else if (cnt == VMXNET3_MULTICAST_MAX) { 3155254738Sbryanv overflow = 1; 3156254738Sbryanv break; 3157254738Sbryanv } 3158254738Sbryanv 3159254738Sbryanv bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3160254738Sbryanv &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3161254738Sbryanv cnt++; 3162254738Sbryanv } 3163254738Sbryanv if_maddr_runlock(ifp); 3164254738Sbryanv 3165254738Sbryanv if (overflow != 0) { 3166254738Sbryanv cnt = 0; 3167254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3168254738Sbryanv } else if (cnt > 0) 3169254738Sbryanv mode |= VMXNET3_RXMODE_MCAST; 3170254738Sbryanv ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3171254738Sbryanv } 3172254738Sbryanv 3173254738Sbryanv ds->rxmode = mode; 3174254738Sbryanv 3175254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3176254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3177254738Sbryanv} 3178254738Sbryanv 3179254738Sbryanvstatic int 3180254738Sbryanvvmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3181254738Sbryanv{ 3182254738Sbryanv struct ifnet *ifp; 3183254738Sbryanv 3184254738Sbryanv ifp = sc->vmx_ifp; 3185254738Sbryanv 3186254738Sbryanv if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3187254738Sbryanv return (EINVAL); 3188254738Sbryanv 3189254738Sbryanv ifp->if_mtu = mtu; 3190254738Sbryanv 3191254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3192254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3193254738Sbryanv vmxnet3_init_locked(sc); 3194254738Sbryanv } 3195254738Sbryanv 3196254738Sbryanv return (0); 3197254738Sbryanv} 3198254738Sbryanv 3199254738Sbryanvstatic int 3200254738Sbryanvvmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3201254738Sbryanv{ 3202254738Sbryanv struct vmxnet3_softc *sc; 3203254738Sbryanv struct ifreq *ifr; 3204254738Sbryanv int reinit, mask, error; 3205254738Sbryanv 3206254738Sbryanv sc = ifp->if_softc; 3207254738Sbryanv ifr = (struct ifreq *) data; 3208254738Sbryanv error = 0; 3209254738Sbryanv 3210254738Sbryanv switch (cmd) { 3211254738Sbryanv case SIOCSIFMTU: 3212254738Sbryanv if (ifp->if_mtu != ifr->ifr_mtu) { 3213254738Sbryanv VMXNET3_CORE_LOCK(sc); 3214254738Sbryanv error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3215254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3216254738Sbryanv } 3217254738Sbryanv break; 3218254738Sbryanv 3219254738Sbryanv case SIOCSIFFLAGS: 3220254738Sbryanv VMXNET3_CORE_LOCK(sc); 3221254738Sbryanv if (ifp->if_flags & IFF_UP) { 3222254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3223254738Sbryanv if ((ifp->if_flags ^ sc->vmx_if_flags) & 3224254738Sbryanv (IFF_PROMISC | IFF_ALLMULTI)) { 3225254738Sbryanv vmxnet3_set_rxfilter(sc); 3226254738Sbryanv } 3227254738Sbryanv } else 3228254738Sbryanv vmxnet3_init_locked(sc); 3229254738Sbryanv } else { 3230254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3231254738Sbryanv vmxnet3_stop(sc); 3232254738Sbryanv } 3233254738Sbryanv sc->vmx_if_flags = ifp->if_flags; 3234254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3235254738Sbryanv break; 3236254738Sbryanv 3237254738Sbryanv case SIOCADDMULTI: 3238254738Sbryanv case SIOCDELMULTI: 3239254738Sbryanv VMXNET3_CORE_LOCK(sc); 3240254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3241254738Sbryanv vmxnet3_set_rxfilter(sc); 3242254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3243254738Sbryanv break; 3244254738Sbryanv 3245254738Sbryanv case SIOCSIFMEDIA: 3246254738Sbryanv case SIOCGIFMEDIA: 3247254738Sbryanv error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3248254738Sbryanv break; 3249254738Sbryanv 3250254738Sbryanv case SIOCSIFCAP: 3251254738Sbryanv VMXNET3_CORE_LOCK(sc); 3252254738Sbryanv mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3253254738Sbryanv 3254254738Sbryanv if (mask & IFCAP_TXCSUM) 3255254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM; 3256254738Sbryanv if (mask & IFCAP_TXCSUM_IPV6) 3257254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3258254738Sbryanv if (mask & IFCAP_TSO4) 3259254738Sbryanv ifp->if_capenable ^= IFCAP_TSO4; 3260254738Sbryanv if (mask & IFCAP_TSO6) 3261254738Sbryanv ifp->if_capenable ^= IFCAP_TSO6; 3262254738Sbryanv 3263254738Sbryanv if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3264255055Sbryanv IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3265255055Sbryanv /* Changing these features requires us to reinit. */ 3266254738Sbryanv reinit = 1; 3267254738Sbryanv 3268254738Sbryanv if (mask & IFCAP_RXCSUM) 3269254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM; 3270254738Sbryanv if (mask & IFCAP_RXCSUM_IPV6) 3271254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3272254738Sbryanv if (mask & IFCAP_LRO) 3273254738Sbryanv ifp->if_capenable ^= IFCAP_LRO; 3274255055Sbryanv if (mask & IFCAP_VLAN_HWTAGGING) 3275255055Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3276254738Sbryanv if (mask & IFCAP_VLAN_HWFILTER) 3277254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3278254738Sbryanv } else 3279254738Sbryanv reinit = 0; 3280254738Sbryanv 3281254738Sbryanv if (mask & IFCAP_VLAN_HWTSO) 3282254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3283254738Sbryanv 3284254738Sbryanv if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3285254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3286254738Sbryanv vmxnet3_init_locked(sc); 3287254738Sbryanv } 3288254738Sbryanv 3289254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3290254738Sbryanv VLAN_CAPABILITIES(ifp); 3291254738Sbryanv break; 3292254738Sbryanv 3293254738Sbryanv default: 3294254738Sbryanv error = ether_ioctl(ifp, cmd, data); 3295254738Sbryanv break; 3296254738Sbryanv } 3297254738Sbryanv 3298254738Sbryanv VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3299254738Sbryanv 3300254738Sbryanv return (error); 3301254738Sbryanv} 3302254738Sbryanv 3303264866Sbryanv#ifndef VMXNET3_LEGACY_TX 3304264866Sbryanvstatic void 3305264866Sbryanvvmxnet3_qflush(struct ifnet *ifp) 3306264866Sbryanv{ 3307264866Sbryanv struct vmxnet3_softc *sc; 3308264866Sbryanv struct vmxnet3_txqueue *txq; 3309264866Sbryanv struct mbuf *m; 3310264866Sbryanv int i; 3311264866Sbryanv 3312264866Sbryanv sc = ifp->if_softc; 3313264866Sbryanv 3314264866Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3315264866Sbryanv txq = &sc->vmx_txq[i]; 3316264866Sbryanv 3317264866Sbryanv VMXNET3_TXQ_LOCK(txq); 3318264866Sbryanv while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3319264866Sbryanv m_freem(m); 3320264866Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3321264866Sbryanv } 3322264866Sbryanv 3323264866Sbryanv if_qflush(ifp); 3324264866Sbryanv} 3325264866Sbryanv#endif 3326264866Sbryanv 3327254738Sbryanvstatic int 3328254738Sbryanvvmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3329254738Sbryanv{ 3330254738Sbryanv struct vmxnet3_softc *sc; 3331254738Sbryanv 3332254738Sbryanv sc = txq->vxtxq_sc; 3333254738Sbryanv 3334254738Sbryanv VMXNET3_TXQ_LOCK(txq); 3335254738Sbryanv if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3336254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3337254738Sbryanv return (0); 3338254738Sbryanv } 3339254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3340254738Sbryanv 3341254738Sbryanv if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3342254738Sbryanv txq->vxtxq_id); 3343254738Sbryanv return (1); 3344254738Sbryanv} 3345254738Sbryanv 3346254738Sbryanvstatic void 3347264866Sbryanvvmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3348254738Sbryanv{ 3349254738Sbryanv 3350254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3351254738Sbryanv} 3352254738Sbryanv 3353254738Sbryanvstatic void 3354264866Sbryanvvmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq, 3355264866Sbryanv struct vmxnet3_txq_stats *accum) 3356264866Sbryanv{ 3357264866Sbryanv struct vmxnet3_txq_stats *st; 3358264866Sbryanv 3359264866Sbryanv st = &txq->vxtxq_stats; 3360264866Sbryanv 3361264866Sbryanv accum->vmtxs_opackets += st->vmtxs_opackets; 3362264866Sbryanv accum->vmtxs_obytes += st->vmtxs_obytes; 3363264866Sbryanv accum->vmtxs_omcasts += st->vmtxs_omcasts; 3364264866Sbryanv accum->vmtxs_csum += st->vmtxs_csum; 3365264866Sbryanv accum->vmtxs_tso += st->vmtxs_tso; 3366264866Sbryanv accum->vmtxs_full += st->vmtxs_full; 3367264866Sbryanv accum->vmtxs_offload_failed += st->vmtxs_offload_failed; 3368264866Sbryanv} 3369264866Sbryanv 3370264866Sbryanvstatic void 3371264866Sbryanvvmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq, 3372264866Sbryanv struct vmxnet3_rxq_stats *accum) 3373264866Sbryanv{ 3374264866Sbryanv struct vmxnet3_rxq_stats *st; 3375264866Sbryanv 3376264866Sbryanv st = &rxq->vxrxq_stats; 3377264866Sbryanv 3378264866Sbryanv accum->vmrxs_ipackets += st->vmrxs_ipackets; 3379264866Sbryanv accum->vmrxs_ibytes += st->vmrxs_ibytes; 3380264866Sbryanv accum->vmrxs_iqdrops += st->vmrxs_iqdrops; 3381264866Sbryanv accum->vmrxs_ierrors += st->vmrxs_ierrors; 3382264866Sbryanv} 3383264866Sbryanv 3384264866Sbryanvstatic void 3385264866Sbryanvvmxnet3_accumulate_stats(struct vmxnet3_softc *sc) 3386264866Sbryanv{ 3387264866Sbryanv struct ifnet *ifp; 3388264866Sbryanv struct vmxnet3_statistics *st; 3389264866Sbryanv struct vmxnet3_txq_stats txaccum; 3390264866Sbryanv struct vmxnet3_rxq_stats rxaccum; 3391264866Sbryanv int i; 3392264866Sbryanv 3393264866Sbryanv ifp = sc->vmx_ifp; 3394264866Sbryanv st = &sc->vmx_stats; 3395264866Sbryanv 3396264866Sbryanv bzero(&txaccum, sizeof(struct vmxnet3_txq_stats)); 3397264866Sbryanv bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats)); 3398264866Sbryanv 3399264866Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3400264866Sbryanv vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum); 3401264866Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 3402264866Sbryanv vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum); 3403264866Sbryanv 3404264866Sbryanv /* 3405264866Sbryanv * With the exception of if_ierrors, these ifnet statistics are 3406264866Sbryanv * only updated in the driver, so just set them to our accumulated 3407264866Sbryanv * values. if_ierrors is updated in ether_input() for malformed 3408264866Sbryanv * frames that we should have already discarded. 3409264866Sbryanv */ 3410264866Sbryanv ifp->if_ipackets = rxaccum.vmrxs_ipackets; 3411264866Sbryanv ifp->if_iqdrops = rxaccum.vmrxs_iqdrops; 3412264866Sbryanv ifp->if_ierrors = rxaccum.vmrxs_ierrors; 3413264866Sbryanv ifp->if_opackets = txaccum.vmtxs_opackets; 3414264866Sbryanv#ifndef VMXNET3_LEGACY_TX 3415264866Sbryanv ifp->if_obytes = txaccum.vmtxs_obytes; 3416264866Sbryanv ifp->if_omcasts = txaccum.vmtxs_omcasts; 3417264866Sbryanv#endif 3418264866Sbryanv} 3419264866Sbryanv 3420264866Sbryanvstatic void 3421254738Sbryanvvmxnet3_tick(void *xsc) 3422254738Sbryanv{ 3423254738Sbryanv struct vmxnet3_softc *sc; 3424254738Sbryanv struct ifnet *ifp; 3425254738Sbryanv int i, timedout; 3426254738Sbryanv 3427254738Sbryanv sc = xsc; 3428254738Sbryanv ifp = sc->vmx_ifp; 3429254738Sbryanv timedout = 0; 3430254738Sbryanv 3431254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3432254738Sbryanv 3433264866Sbryanv vmxnet3_accumulate_stats(sc); 3434264866Sbryanv vmxnet3_refresh_host_stats(sc); 3435264866Sbryanv 3436254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3437254738Sbryanv timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3438254738Sbryanv 3439254738Sbryanv if (timedout != 0) { 3440254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3441254738Sbryanv vmxnet3_init_locked(sc); 3442254738Sbryanv } else 3443254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3444254738Sbryanv} 3445254738Sbryanv 3446254738Sbryanvstatic int 3447254738Sbryanvvmxnet3_link_is_up(struct vmxnet3_softc *sc) 3448254738Sbryanv{ 3449254738Sbryanv uint32_t status; 3450254738Sbryanv 3451254738Sbryanv /* Also update the link speed while here. */ 3452254738Sbryanv status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3453254738Sbryanv sc->vmx_link_speed = status >> 16; 3454254738Sbryanv return !!(status & 0x1); 3455254738Sbryanv} 3456254738Sbryanv 3457254738Sbryanvstatic void 3458254738Sbryanvvmxnet3_link_status(struct vmxnet3_softc *sc) 3459254738Sbryanv{ 3460254738Sbryanv struct ifnet *ifp; 3461254738Sbryanv int link; 3462254738Sbryanv 3463254738Sbryanv ifp = sc->vmx_ifp; 3464254738Sbryanv link = vmxnet3_link_is_up(sc); 3465254738Sbryanv 3466254738Sbryanv if (link != 0 && sc->vmx_link_active == 0) { 3467254738Sbryanv sc->vmx_link_active = 1; 3468254738Sbryanv if_link_state_change(ifp, LINK_STATE_UP); 3469254738Sbryanv } else if (link == 0 && sc->vmx_link_active != 0) { 3470254738Sbryanv sc->vmx_link_active = 0; 3471254738Sbryanv if_link_state_change(ifp, LINK_STATE_DOWN); 3472254738Sbryanv } 3473254738Sbryanv} 3474254738Sbryanv 3475254738Sbryanvstatic void 3476254738Sbryanvvmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3477254738Sbryanv{ 3478254738Sbryanv struct vmxnet3_softc *sc; 3479254738Sbryanv 3480254738Sbryanv sc = ifp->if_softc; 3481254738Sbryanv 3482254738Sbryanv ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3483254738Sbryanv ifmr->ifm_status = IFM_AVALID; 3484254738Sbryanv 3485254738Sbryanv VMXNET3_CORE_LOCK(sc); 3486254738Sbryanv if (vmxnet3_link_is_up(sc) != 0) 3487254738Sbryanv ifmr->ifm_status |= IFM_ACTIVE; 3488254738Sbryanv else 3489254738Sbryanv ifmr->ifm_status |= IFM_NONE; 3490254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3491254738Sbryanv} 3492254738Sbryanv 3493254738Sbryanvstatic int 3494254738Sbryanvvmxnet3_media_change(struct ifnet *ifp) 3495254738Sbryanv{ 3496254738Sbryanv 3497254738Sbryanv /* Ignore. */ 3498254738Sbryanv return (0); 3499254738Sbryanv} 3500254738Sbryanv 3501254738Sbryanvstatic void 3502254738Sbryanvvmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3503254738Sbryanv{ 3504254738Sbryanv uint32_t ml, mh; 3505254738Sbryanv 3506254738Sbryanv ml = sc->vmx_lladdr[0]; 3507254738Sbryanv ml |= sc->vmx_lladdr[1] << 8; 3508254738Sbryanv ml |= sc->vmx_lladdr[2] << 16; 3509254738Sbryanv ml |= sc->vmx_lladdr[3] << 24; 3510254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3511254738Sbryanv 3512254738Sbryanv mh = sc->vmx_lladdr[4]; 3513254738Sbryanv mh |= sc->vmx_lladdr[5] << 8; 3514254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3515254738Sbryanv} 3516254738Sbryanv 3517254738Sbryanvstatic void 3518254738Sbryanvvmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3519254738Sbryanv{ 3520254738Sbryanv uint32_t ml, mh; 3521254738Sbryanv 3522254738Sbryanv ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3523254738Sbryanv mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3524254738Sbryanv 3525254738Sbryanv sc->vmx_lladdr[0] = ml; 3526254738Sbryanv sc->vmx_lladdr[1] = ml >> 8; 3527254738Sbryanv sc->vmx_lladdr[2] = ml >> 16; 3528254738Sbryanv sc->vmx_lladdr[3] = ml >> 24; 3529254738Sbryanv sc->vmx_lladdr[4] = mh; 3530254738Sbryanv sc->vmx_lladdr[5] = mh >> 8; 3531254738Sbryanv} 3532254738Sbryanv 3533254738Sbryanvstatic void 3534254738Sbryanvvmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3535254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3536254738Sbryanv{ 3537254738Sbryanv struct sysctl_oid *node, *txsnode; 3538254738Sbryanv struct sysctl_oid_list *list, *txslist; 3539254738Sbryanv struct vmxnet3_txq_stats *stats; 3540254738Sbryanv struct UPT1_TxStats *txstats; 3541254738Sbryanv char namebuf[16]; 3542254738Sbryanv 3543254738Sbryanv stats = &txq->vxtxq_stats; 3544254738Sbryanv txstats = &txq->vxtxq_ts->stats; 3545254738Sbryanv 3546254738Sbryanv snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3547254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3548254738Sbryanv NULL, "Transmit Queue"); 3549254738Sbryanv txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3550254738Sbryanv 3551264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3552264866Sbryanv &stats->vmtxs_opackets, "Transmit packets"); 3553264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3554264866Sbryanv &stats->vmtxs_obytes, "Transmit bytes"); 3555264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3556264866Sbryanv &stats->vmtxs_omcasts, "Transmit multicasts"); 3557264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3558264866Sbryanv &stats->vmtxs_csum, "Transmit checksum offloaded"); 3559264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3560264866Sbryanv &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3561254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3562264866Sbryanv &stats->vmtxs_full, "Transmit ring full"); 3563254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3564264866Sbryanv &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3565254738Sbryanv 3566254738Sbryanv /* 3567254738Sbryanv * Add statistics reported by the host. These are updated once 3568254738Sbryanv * per second. 3569254738Sbryanv */ 3570254738Sbryanv txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3571254738Sbryanv NULL, "Host Statistics"); 3572254738Sbryanv txslist = SYSCTL_CHILDREN(txsnode); 3573254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3574254738Sbryanv &txstats->TSO_packets, "TSO packets"); 3575254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3576254738Sbryanv &txstats->TSO_bytes, "TSO bytes"); 3577254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3578254738Sbryanv &txstats->ucast_packets, "Unicast packets"); 3579254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3580254738Sbryanv &txstats->ucast_bytes, "Unicast bytes"); 3581254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3582254738Sbryanv &txstats->mcast_packets, "Multicast packets"); 3583254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3584254738Sbryanv &txstats->mcast_bytes, "Multicast bytes"); 3585254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3586254738Sbryanv &txstats->error, "Errors"); 3587254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3588254738Sbryanv &txstats->discard, "Discards"); 3589254738Sbryanv} 3590254738Sbryanv 3591254738Sbryanvstatic void 3592254738Sbryanvvmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3593254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3594254738Sbryanv{ 3595254738Sbryanv struct sysctl_oid *node, *rxsnode; 3596254738Sbryanv struct sysctl_oid_list *list, *rxslist; 3597254738Sbryanv struct vmxnet3_rxq_stats *stats; 3598254738Sbryanv struct UPT1_RxStats *rxstats; 3599254738Sbryanv char namebuf[16]; 3600254738Sbryanv 3601254738Sbryanv stats = &rxq->vxrxq_stats; 3602254738Sbryanv rxstats = &rxq->vxrxq_rs->stats; 3603254738Sbryanv 3604254738Sbryanv snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3605254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3606254738Sbryanv NULL, "Receive Queue"); 3607254738Sbryanv rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3608254738Sbryanv 3609264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3610264866Sbryanv &stats->vmrxs_ipackets, "Receive packets"); 3611264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3612264866Sbryanv &stats->vmrxs_ibytes, "Receive bytes"); 3613264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3614264866Sbryanv &stats->vmrxs_iqdrops, "Receive drops"); 3615264866Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3616264866Sbryanv &stats->vmrxs_ierrors, "Receive errors"); 3617264866Sbryanv 3618254738Sbryanv /* 3619254738Sbryanv * Add statistics reported by the host. These are updated once 3620254738Sbryanv * per second. 3621254738Sbryanv */ 3622254738Sbryanv rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3623254738Sbryanv NULL, "Host Statistics"); 3624254738Sbryanv rxslist = SYSCTL_CHILDREN(rxsnode); 3625254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3626254738Sbryanv &rxstats->LRO_packets, "LRO packets"); 3627254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3628254738Sbryanv &rxstats->LRO_bytes, "LRO bytes"); 3629254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3630254738Sbryanv &rxstats->ucast_packets, "Unicast packets"); 3631254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3632254738Sbryanv &rxstats->ucast_bytes, "Unicast bytes"); 3633254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3634254738Sbryanv &rxstats->mcast_packets, "Multicast packets"); 3635254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3636254738Sbryanv &rxstats->mcast_bytes, "Multicast bytes"); 3637254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3638254738Sbryanv &rxstats->bcast_packets, "Broadcast packets"); 3639254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3640254738Sbryanv &rxstats->bcast_bytes, "Broadcast bytes"); 3641254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3642254738Sbryanv &rxstats->nobuffer, "No buffer"); 3643254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3644254738Sbryanv &rxstats->error, "Errors"); 3645254738Sbryanv} 3646254738Sbryanv 3647254738Sbryanvstatic void 3648254738Sbryanvvmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3649254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3650254738Sbryanv{ 3651254738Sbryanv struct sysctl_oid *node; 3652254738Sbryanv struct sysctl_oid_list *list; 3653254738Sbryanv int i; 3654254738Sbryanv 3655254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3656254738Sbryanv struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3657254738Sbryanv 3658254738Sbryanv node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3659254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3660254738Sbryanv list = SYSCTL_CHILDREN(node); 3661254738Sbryanv 3662254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3663254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3664254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3665254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3666254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3667254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3668254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3669254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3670254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3671254738Sbryanv &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3672254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3673254738Sbryanv &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3674254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3675254738Sbryanv &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3676254738Sbryanv } 3677254738Sbryanv 3678254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 3679254738Sbryanv struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3680254738Sbryanv 3681254738Sbryanv node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3682254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3683254738Sbryanv list = SYSCTL_CHILDREN(node); 3684254738Sbryanv 3685254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3686254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3687254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3688254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3689254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3690254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3691254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3692254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3693254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3694254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3695254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3696254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3697254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3698254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3699254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3700254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3701254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3702254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3703254738Sbryanv } 3704254738Sbryanv} 3705254738Sbryanv 3706254738Sbryanvstatic void 3707254738Sbryanvvmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3708254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3709254738Sbryanv{ 3710254738Sbryanv int i; 3711254738Sbryanv 3712254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3713254738Sbryanv vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3714254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 3715254738Sbryanv vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3716254738Sbryanv 3717254738Sbryanv vmxnet3_setup_debug_sysctl(sc, ctx, child); 3718254738Sbryanv} 3719254738Sbryanv 3720254738Sbryanvstatic void 3721254738Sbryanvvmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3722254738Sbryanv{ 3723254738Sbryanv device_t dev; 3724254738Sbryanv struct vmxnet3_statistics *stats; 3725254738Sbryanv struct sysctl_ctx_list *ctx; 3726254738Sbryanv struct sysctl_oid *tree; 3727254738Sbryanv struct sysctl_oid_list *child; 3728254738Sbryanv 3729254738Sbryanv dev = sc->vmx_dev; 3730254738Sbryanv ctx = device_get_sysctl_ctx(dev); 3731254738Sbryanv tree = device_get_sysctl_tree(dev); 3732254738Sbryanv child = SYSCTL_CHILDREN(tree); 3733254738Sbryanv 3734264866Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3735264866Sbryanv &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3736264866Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3737264866Sbryanv &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3738254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3739254738Sbryanv &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3740254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3741254738Sbryanv &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3742254738Sbryanv 3743254738Sbryanv stats = &sc->vmx_stats; 3744264866Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3745264866Sbryanv &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3746264866Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3747264866Sbryanv &stats->vmst_defrag_failed, 0, 3748264866Sbryanv "Tx mbuf dropped because defrag failed"); 3749254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3750254738Sbryanv &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3751254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3752254738Sbryanv &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3753254738Sbryanv 3754254738Sbryanv vmxnet3_setup_queue_sysctl(sc, ctx, child); 3755254738Sbryanv} 3756254738Sbryanv 3757254738Sbryanvstatic void 3758254738Sbryanvvmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3759254738Sbryanv{ 3760254738Sbryanv 3761254738Sbryanv bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3762254738Sbryanv} 3763254738Sbryanv 3764254738Sbryanvstatic uint32_t 3765254738Sbryanvvmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3766254738Sbryanv{ 3767254738Sbryanv 3768254738Sbryanv return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3769254738Sbryanv} 3770254738Sbryanv 3771254738Sbryanvstatic void 3772254738Sbryanvvmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3773254738Sbryanv{ 3774254738Sbryanv 3775254738Sbryanv bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3776254738Sbryanv} 3777254738Sbryanv 3778254738Sbryanvstatic void 3779254738Sbryanvvmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3780254738Sbryanv{ 3781254738Sbryanv 3782254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3783254738Sbryanv} 3784254738Sbryanv 3785254738Sbryanvstatic uint32_t 3786254738Sbryanvvmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3787254738Sbryanv{ 3788254738Sbryanv 3789254738Sbryanv vmxnet3_write_cmd(sc, cmd); 3790254738Sbryanv bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3791254738Sbryanv BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3792254738Sbryanv return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3793254738Sbryanv} 3794254738Sbryanv 3795254738Sbryanvstatic void 3796254738Sbryanvvmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3797254738Sbryanv{ 3798254738Sbryanv 3799254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3800254738Sbryanv} 3801254738Sbryanv 3802254738Sbryanvstatic void 3803254738Sbryanvvmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3804254738Sbryanv{ 3805254738Sbryanv 3806254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3807254738Sbryanv} 3808254738Sbryanv 3809254738Sbryanvstatic void 3810254738Sbryanvvmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3811254738Sbryanv{ 3812254738Sbryanv int i; 3813254738Sbryanv 3814254738Sbryanv sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3815254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3816254738Sbryanv vmxnet3_enable_intr(sc, i); 3817254738Sbryanv} 3818254738Sbryanv 3819254738Sbryanvstatic void 3820254738Sbryanvvmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3821254738Sbryanv{ 3822254738Sbryanv int i; 3823254738Sbryanv 3824254738Sbryanv sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3825254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3826254738Sbryanv vmxnet3_disable_intr(sc, i); 3827254738Sbryanv} 3828254738Sbryanv 3829254738Sbryanvstatic void 3830254738Sbryanvvmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3831254738Sbryanv{ 3832254738Sbryanv bus_addr_t *baddr = arg; 3833254738Sbryanv 3834254738Sbryanv if (error == 0) 3835254738Sbryanv *baddr = segs->ds_addr; 3836254738Sbryanv} 3837254738Sbryanv 3838254738Sbryanvstatic int 3839254738Sbryanvvmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3840254738Sbryanv struct vmxnet3_dma_alloc *dma) 3841254738Sbryanv{ 3842254738Sbryanv device_t dev; 3843254738Sbryanv int error; 3844254738Sbryanv 3845254738Sbryanv dev = sc->vmx_dev; 3846254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3847254738Sbryanv 3848254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 3849254738Sbryanv align, 0, /* alignment, bounds */ 3850254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 3851254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 3852254738Sbryanv NULL, NULL, /* filter, filterarg */ 3853254738Sbryanv size, /* maxsize */ 3854254738Sbryanv 1, /* nsegments */ 3855254738Sbryanv size, /* maxsegsize */ 3856254738Sbryanv BUS_DMA_ALLOCNOW, /* flags */ 3857254738Sbryanv NULL, /* lockfunc */ 3858254738Sbryanv NULL, /* lockfuncarg */ 3859254738Sbryanv &dma->dma_tag); 3860254738Sbryanv if (error) { 3861254738Sbryanv device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3862254738Sbryanv goto fail; 3863254738Sbryanv } 3864254738Sbryanv 3865254738Sbryanv error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3866254738Sbryanv BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3867254738Sbryanv if (error) { 3868254738Sbryanv device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3869254738Sbryanv goto fail; 3870254738Sbryanv } 3871254738Sbryanv 3872254738Sbryanv error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3873254738Sbryanv size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3874254738Sbryanv if (error) { 3875254738Sbryanv device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3876254738Sbryanv goto fail; 3877254738Sbryanv } 3878254738Sbryanv 3879254738Sbryanv dma->dma_size = size; 3880254738Sbryanv 3881254738Sbryanvfail: 3882254738Sbryanv if (error) 3883254738Sbryanv vmxnet3_dma_free(sc, dma); 3884254738Sbryanv 3885254738Sbryanv return (error); 3886254738Sbryanv} 3887254738Sbryanv 3888254738Sbryanvstatic void 3889254738Sbryanvvmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3890254738Sbryanv{ 3891254738Sbryanv 3892254738Sbryanv if (dma->dma_tag != NULL) { 3893254738Sbryanv if (dma->dma_map != NULL) { 3894254738Sbryanv bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3895254738Sbryanv BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3896254738Sbryanv bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3897254738Sbryanv } 3898254738Sbryanv 3899254738Sbryanv if (dma->dma_vaddr != NULL) { 3900254738Sbryanv bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3901254738Sbryanv dma->dma_map); 3902254738Sbryanv } 3903254738Sbryanv 3904254738Sbryanv bus_dma_tag_destroy(dma->dma_tag); 3905254738Sbryanv } 3906254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3907254738Sbryanv} 3908254738Sbryanv 3909255055Sbryanvstatic int 3910255055Sbryanvvmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3911255055Sbryanv{ 3912255055Sbryanv char path[64]; 3913255055Sbryanv 3914255055Sbryanv snprintf(path, sizeof(path), 3915255055Sbryanv "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3916255055Sbryanv TUNABLE_INT_FETCH(path, &def); 3917255055Sbryanv 3918255055Sbryanv return (def); 3919255055Sbryanv} 3920255055Sbryanv 3921254738Sbryanv/* 3922254738Sbryanv * Since this is a purely paravirtualized device, we do not have 3923254738Sbryanv * to worry about DMA coherency. But at times, we must make sure 3924254738Sbryanv * both the compiler and CPU do not reorder memory operations. 3925254738Sbryanv */ 3926254738Sbryanvstatic inline void 3927254738Sbryanvvmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3928254738Sbryanv{ 3929254738Sbryanv 3930254738Sbryanv switch (type) { 3931254738Sbryanv case VMXNET3_BARRIER_RD: 3932254738Sbryanv rmb(); 3933254738Sbryanv break; 3934254738Sbryanv case VMXNET3_BARRIER_WR: 3935254738Sbryanv wmb(); 3936254738Sbryanv break; 3937254738Sbryanv case VMXNET3_BARRIER_RDWR: 3938254738Sbryanv mb(); 3939254738Sbryanv break; 3940254738Sbryanv default: 3941254738Sbryanv panic("%s: bad barrier type %d", __func__, type); 3942254738Sbryanv } 3943254738Sbryanv} 3944