1254738Sbryanv/*- 2254738Sbryanv * Copyright (c) 2013 Tsubai Masanari 3254738Sbryanv * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4254738Sbryanv * 5254738Sbryanv * Permission to use, copy, modify, and distribute this software for any 6254738Sbryanv * purpose with or without fee is hereby granted, provided that the above 7254738Sbryanv * copyright notice and this permission notice appear in all copies. 8254738Sbryanv * 9254738Sbryanv * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10254738Sbryanv * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11254738Sbryanv * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12254738Sbryanv * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13254738Sbryanv * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14254738Sbryanv * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15254738Sbryanv * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16254738Sbryanv * 17254738Sbryanv * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18254738Sbryanv */ 19254738Sbryanv 20254738Sbryanv/* Driver for VMware vmxnet3 virtual ethernet devices. */ 21254738Sbryanv 22254738Sbryanv#include <sys/cdefs.h> 23254738Sbryanv__FBSDID("$FreeBSD: stable/11/sys/dev/vmware/vmxnet3/if_vmx.c 344272 2019-02-19 10:07:48Z vmaffione $"); 24254738Sbryanv 25254738Sbryanv#include <sys/param.h> 26254738Sbryanv#include <sys/systm.h> 27257241Sglebius#include <sys/eventhandler.h> 28254738Sbryanv#include <sys/kernel.h> 29254738Sbryanv#include <sys/endian.h> 30254738Sbryanv#include <sys/sockio.h> 31254738Sbryanv#include <sys/mbuf.h> 32254738Sbryanv#include <sys/malloc.h> 33254738Sbryanv#include <sys/module.h> 34254738Sbryanv#include <sys/socket.h> 35254738Sbryanv#include <sys/sysctl.h> 36263259Sbryanv#include <sys/smp.h> 37263259Sbryanv#include <sys/taskqueue.h> 38254738Sbryanv#include <vm/vm.h> 39254738Sbryanv#include <vm/pmap.h> 40254738Sbryanv 41254738Sbryanv#include <net/ethernet.h> 42254738Sbryanv#include <net/if.h> 43257176Sglebius#include <net/if_var.h> 44254738Sbryanv#include <net/if_arp.h> 45254738Sbryanv#include <net/if_dl.h> 46254738Sbryanv#include <net/if_types.h> 47254738Sbryanv#include <net/if_media.h> 48254738Sbryanv#include <net/if_vlan_var.h> 49254738Sbryanv 50254738Sbryanv#include <net/bpf.h> 51254738Sbryanv 52254738Sbryanv#include <netinet/in_systm.h> 53254738Sbryanv#include <netinet/in.h> 54254738Sbryanv#include <netinet/ip.h> 55254738Sbryanv#include <netinet/ip6.h> 56254738Sbryanv#include <netinet6/ip6_var.h> 57254738Sbryanv#include <netinet/udp.h> 58254738Sbryanv#include <netinet/tcp.h> 59254738Sbryanv 60267253Sbryanv#include <machine/in_cksum.h> 61267253Sbryanv 62254738Sbryanv#include <machine/bus.h> 63254738Sbryanv#include <machine/resource.h> 64254738Sbryanv#include <sys/bus.h> 65254738Sbryanv#include <sys/rman.h> 66254738Sbryanv 67254738Sbryanv#include <dev/pci/pcireg.h> 68254738Sbryanv#include <dev/pci/pcivar.h> 69254738Sbryanv 70254738Sbryanv#include "if_vmxreg.h" 71254738Sbryanv#include "if_vmxvar.h" 72254738Sbryanv 73254738Sbryanv#include "opt_inet.h" 74254738Sbryanv#include "opt_inet6.h" 75254738Sbryanv 76254738Sbryanv#ifdef VMXNET3_FAILPOINTS 77254738Sbryanv#include <sys/fail.h> 78254738Sbryanvstatic SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 79254738Sbryanv "vmxnet3 fail points"); 80254738Sbryanv#define VMXNET3_FP _debug_fail_point_vmxnet3 81254738Sbryanv#endif 82254738Sbryanv 83254738Sbryanvstatic int vmxnet3_probe(device_t); 84254738Sbryanvstatic int vmxnet3_attach(device_t); 85254738Sbryanvstatic int vmxnet3_detach(device_t); 86254738Sbryanvstatic int vmxnet3_shutdown(device_t); 87254738Sbryanv 88254738Sbryanvstatic int vmxnet3_alloc_resources(struct vmxnet3_softc *); 89254738Sbryanvstatic void vmxnet3_free_resources(struct vmxnet3_softc *); 90254738Sbryanvstatic int vmxnet3_check_version(struct vmxnet3_softc *); 91254738Sbryanvstatic void vmxnet3_initial_config(struct vmxnet3_softc *); 92263259Sbryanvstatic void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 93254738Sbryanv 94254738Sbryanvstatic int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95254738Sbryanvstatic int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96254738Sbryanvstatic int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 97254738Sbryanvstatic int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 98254738Sbryanv struct vmxnet3_interrupt *); 99254738Sbryanvstatic int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 100254738Sbryanvstatic int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 101254738Sbryanvstatic int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 102254738Sbryanvstatic int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 103254738Sbryanvstatic int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 104254738Sbryanv 105254738Sbryanvstatic void vmxnet3_free_interrupt(struct vmxnet3_softc *, 106254738Sbryanv struct vmxnet3_interrupt *); 107254738Sbryanvstatic void vmxnet3_free_interrupts(struct vmxnet3_softc *); 108254738Sbryanv 109263259Sbryanv#ifndef VMXNET3_LEGACY_TX 110263259Sbryanvstatic int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 111263259Sbryanvstatic void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 112263259Sbryanvstatic void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 113263259Sbryanvstatic void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 114263259Sbryanv#endif 115263259Sbryanv 116254738Sbryanvstatic int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 117254738Sbryanvstatic int vmxnet3_init_txq(struct vmxnet3_softc *, int); 118254738Sbryanvstatic int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 119254738Sbryanvstatic void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 120254738Sbryanvstatic void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 121254738Sbryanvstatic void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 122254738Sbryanv 123254738Sbryanvstatic int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 124254738Sbryanvstatic void vmxnet3_free_shared_data(struct vmxnet3_softc *); 125254738Sbryanvstatic int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 126254738Sbryanvstatic void vmxnet3_free_txq_data(struct vmxnet3_softc *); 127254738Sbryanvstatic int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 128254738Sbryanvstatic void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 129254738Sbryanvstatic int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 130254738Sbryanvstatic void vmxnet3_free_queue_data(struct vmxnet3_softc *); 131254738Sbryanvstatic int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 132254738Sbryanvstatic void vmxnet3_init_shared_data(struct vmxnet3_softc *); 133303136Smavstatic void vmxnet3_init_hwassist(struct vmxnet3_softc *); 134254738Sbryanvstatic void vmxnet3_reinit_interface(struct vmxnet3_softc *); 135263259Sbryanvstatic void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 136254738Sbryanvstatic void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 137254738Sbryanvstatic int vmxnet3_alloc_data(struct vmxnet3_softc *); 138254738Sbryanvstatic void vmxnet3_free_data(struct vmxnet3_softc *); 139254738Sbryanvstatic int vmxnet3_setup_interface(struct vmxnet3_softc *); 140254738Sbryanv 141254738Sbryanvstatic void vmxnet3_evintr(struct vmxnet3_softc *); 142254738Sbryanvstatic void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 143254738Sbryanvstatic void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 144254738Sbryanvstatic int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 145254738Sbryanvstatic void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 146254738Sbryanv struct vmxnet3_rxring *, int); 147254738Sbryanvstatic void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 148254738Sbryanvstatic void vmxnet3_legacy_intr(void *); 149254738Sbryanvstatic void vmxnet3_txq_intr(void *); 150254738Sbryanvstatic void vmxnet3_rxq_intr(void *); 151254738Sbryanvstatic void vmxnet3_event_intr(void *); 152254738Sbryanv 153254738Sbryanvstatic void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 154254738Sbryanvstatic void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 155254738Sbryanvstatic void vmxnet3_stop(struct vmxnet3_softc *); 156254738Sbryanv 157254738Sbryanvstatic void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 158254738Sbryanvstatic int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 159254738Sbryanvstatic int vmxnet3_reinit_queues(struct vmxnet3_softc *); 160254738Sbryanvstatic int vmxnet3_enable_device(struct vmxnet3_softc *); 161254738Sbryanvstatic void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 162254738Sbryanvstatic int vmxnet3_reinit(struct vmxnet3_softc *); 163254738Sbryanvstatic void vmxnet3_init_locked(struct vmxnet3_softc *); 164254738Sbryanvstatic void vmxnet3_init(void *); 165254738Sbryanv 166263259Sbryanvstatic int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 167263259Sbryanv int *, int *, int *); 168254738Sbryanvstatic int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 169254738Sbryanv bus_dmamap_t, bus_dma_segment_t [], int *); 170254738Sbryanvstatic void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 171254738Sbryanvstatic int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 172263259Sbryanv#ifdef VMXNET3_LEGACY_TX 173254738Sbryanvstatic void vmxnet3_start_locked(struct ifnet *); 174254738Sbryanvstatic void vmxnet3_start(struct ifnet *); 175263259Sbryanv#else 176263259Sbryanvstatic int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 177263259Sbryanv struct mbuf *); 178263259Sbryanvstatic int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 179263259Sbryanvstatic void vmxnet3_txq_tq_deferred(void *, int); 180263259Sbryanv#endif 181263259Sbryanvstatic void vmxnet3_txq_start(struct vmxnet3_txqueue *); 182263259Sbryanvstatic void vmxnet3_tx_start_all(struct vmxnet3_softc *); 183254738Sbryanv 184254738Sbryanvstatic void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 185254738Sbryanv uint16_t); 186254738Sbryanvstatic void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 187254738Sbryanvstatic void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 188254738Sbryanvstatic void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 189254738Sbryanvstatic int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 190254738Sbryanvstatic int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 191272099Sglebiusstatic uint64_t vmxnet3_get_counter(struct ifnet *, ift_counter); 192254738Sbryanv 193263259Sbryanv#ifndef VMXNET3_LEGACY_TX 194263259Sbryanvstatic void vmxnet3_qflush(struct ifnet *); 195263259Sbryanv#endif 196263259Sbryanv 197254738Sbryanvstatic int vmxnet3_watchdog(struct vmxnet3_txqueue *); 198263259Sbryanvstatic void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 199254738Sbryanvstatic void vmxnet3_tick(void *); 200254738Sbryanvstatic void vmxnet3_link_status(struct vmxnet3_softc *); 201254738Sbryanvstatic void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 202254738Sbryanvstatic int vmxnet3_media_change(struct ifnet *); 203254738Sbryanvstatic void vmxnet3_set_lladdr(struct vmxnet3_softc *); 204254738Sbryanvstatic void vmxnet3_get_lladdr(struct vmxnet3_softc *); 205254738Sbryanv 206254738Sbryanvstatic void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 207254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 208254738Sbryanvstatic void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 209254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 210254738Sbryanvstatic void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 211254738Sbryanv struct sysctl_ctx_list *, struct sysctl_oid_list *); 212254738Sbryanvstatic void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 213254738Sbryanv 214254738Sbryanvstatic void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 215254738Sbryanv uint32_t); 216254738Sbryanvstatic uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 217254738Sbryanvstatic void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 218254738Sbryanv uint32_t); 219254738Sbryanvstatic void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 220254738Sbryanvstatic uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 221254738Sbryanv 222254738Sbryanvstatic void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 223254738Sbryanvstatic void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 224254738Sbryanvstatic void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 225254738Sbryanvstatic void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 226254738Sbryanv 227254738Sbryanvstatic int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 228254738Sbryanv bus_size_t, struct vmxnet3_dma_alloc *); 229254738Sbryanvstatic void vmxnet3_dma_free(struct vmxnet3_softc *, 230254738Sbryanv struct vmxnet3_dma_alloc *); 231255055Sbryanvstatic int vmxnet3_tunable_int(struct vmxnet3_softc *, 232255055Sbryanv const char *, int); 233254738Sbryanv 234254738Sbryanvtypedef enum { 235254738Sbryanv VMXNET3_BARRIER_RD, 236254738Sbryanv VMXNET3_BARRIER_WR, 237254738Sbryanv VMXNET3_BARRIER_RDWR, 238254738Sbryanv} vmxnet3_barrier_t; 239254738Sbryanv 240254738Sbryanvstatic void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 241254738Sbryanv 242344272Svmaffione#ifdef DEV_NETMAP 243344272Svmaffione#include "vmx_netmap.h" 244344272Svmaffione#endif 245344272Svmaffione 246255055Sbryanv/* Tunables. */ 247263259Sbryanvstatic int vmxnet3_mq_disable = 0; 248263259SbryanvTUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 249263259Sbryanvstatic int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 250263259SbryanvTUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 251263259Sbryanvstatic int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 252263259SbryanvTUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 253255055Sbryanvstatic int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 254255055SbryanvTUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 255255055Sbryanvstatic int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 256255055SbryanvTUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 257255055Sbryanv 258254738Sbryanvstatic device_method_t vmxnet3_methods[] = { 259254738Sbryanv /* Device interface. */ 260254738Sbryanv DEVMETHOD(device_probe, vmxnet3_probe), 261254738Sbryanv DEVMETHOD(device_attach, vmxnet3_attach), 262254738Sbryanv DEVMETHOD(device_detach, vmxnet3_detach), 263254738Sbryanv DEVMETHOD(device_shutdown, vmxnet3_shutdown), 264254738Sbryanv 265254738Sbryanv DEVMETHOD_END 266254738Sbryanv}; 267254738Sbryanv 268254738Sbryanvstatic driver_t vmxnet3_driver = { 269254738Sbryanv "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 270254738Sbryanv}; 271254738Sbryanv 272254738Sbryanvstatic devclass_t vmxnet3_devclass; 273254738SbryanvDRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 274254738Sbryanv 275254738SbryanvMODULE_DEPEND(vmx, pci, 1, 1, 1); 276254738SbryanvMODULE_DEPEND(vmx, ether, 1, 1, 1); 277344272Svmaffione#ifdef DEV_NETMAP 278344272SvmaffioneMODULE_DEPEND(vmx, netmap, 1, 1, 1); 279344272Svmaffione#endif 280254738Sbryanv 281254738Sbryanv#define VMXNET3_VMWARE_VENDOR_ID 0x15AD 282254738Sbryanv#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 283254738Sbryanv 284254738Sbryanvstatic int 285254738Sbryanvvmxnet3_probe(device_t dev) 286254738Sbryanv{ 287254738Sbryanv 288254738Sbryanv if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 289254738Sbryanv pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 290254738Sbryanv device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 291254738Sbryanv return (BUS_PROBE_DEFAULT); 292254738Sbryanv } 293254738Sbryanv 294254738Sbryanv return (ENXIO); 295254738Sbryanv} 296254738Sbryanv 297254738Sbryanvstatic int 298254738Sbryanvvmxnet3_attach(device_t dev) 299254738Sbryanv{ 300254738Sbryanv struct vmxnet3_softc *sc; 301254738Sbryanv int error; 302254738Sbryanv 303254738Sbryanv sc = device_get_softc(dev); 304254738Sbryanv sc->vmx_dev = dev; 305254738Sbryanv 306254738Sbryanv pci_enable_busmaster(dev); 307254738Sbryanv 308254738Sbryanv VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 309254738Sbryanv callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 310254738Sbryanv 311254738Sbryanv vmxnet3_initial_config(sc); 312254738Sbryanv 313254738Sbryanv error = vmxnet3_alloc_resources(sc); 314254738Sbryanv if (error) 315254738Sbryanv goto fail; 316254738Sbryanv 317254738Sbryanv error = vmxnet3_check_version(sc); 318254738Sbryanv if (error) 319254738Sbryanv goto fail; 320254738Sbryanv 321254738Sbryanv error = vmxnet3_alloc_rxtx_queues(sc); 322254738Sbryanv if (error) 323254738Sbryanv goto fail; 324254738Sbryanv 325263259Sbryanv#ifndef VMXNET3_LEGACY_TX 326263259Sbryanv error = vmxnet3_alloc_taskqueue(sc); 327263259Sbryanv if (error) 328263259Sbryanv goto fail; 329263259Sbryanv#endif 330263259Sbryanv 331254738Sbryanv error = vmxnet3_alloc_interrupts(sc); 332254738Sbryanv if (error) 333254738Sbryanv goto fail; 334254738Sbryanv 335263259Sbryanv vmxnet3_check_multiqueue(sc); 336263259Sbryanv 337254738Sbryanv error = vmxnet3_alloc_data(sc); 338254738Sbryanv if (error) 339254738Sbryanv goto fail; 340254738Sbryanv 341254738Sbryanv error = vmxnet3_setup_interface(sc); 342254738Sbryanv if (error) 343254738Sbryanv goto fail; 344254738Sbryanv 345254738Sbryanv error = vmxnet3_setup_interrupts(sc); 346254738Sbryanv if (error) { 347254738Sbryanv ether_ifdetach(sc->vmx_ifp); 348254738Sbryanv device_printf(dev, "could not set up interrupt\n"); 349254738Sbryanv goto fail; 350254738Sbryanv } 351254738Sbryanv 352254738Sbryanv vmxnet3_setup_sysctl(sc); 353263259Sbryanv#ifndef VMXNET3_LEGACY_TX 354263259Sbryanv vmxnet3_start_taskqueue(sc); 355263259Sbryanv#endif 356254738Sbryanv 357344272Svmaffione#ifdef DEV_NETMAP 358344272Svmaffione vmxnet3_netmap_attach(sc); 359344272Svmaffione#endif 360344272Svmaffione 361254738Sbryanvfail: 362254738Sbryanv if (error) 363254738Sbryanv vmxnet3_detach(dev); 364254738Sbryanv 365254738Sbryanv return (error); 366254738Sbryanv} 367254738Sbryanv 368254738Sbryanvstatic int 369254738Sbryanvvmxnet3_detach(device_t dev) 370254738Sbryanv{ 371254738Sbryanv struct vmxnet3_softc *sc; 372254738Sbryanv struct ifnet *ifp; 373254738Sbryanv 374254738Sbryanv sc = device_get_softc(dev); 375254738Sbryanv ifp = sc->vmx_ifp; 376254738Sbryanv 377254738Sbryanv if (device_is_attached(dev)) { 378254738Sbryanv VMXNET3_CORE_LOCK(sc); 379254738Sbryanv vmxnet3_stop(sc); 380254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 381263259Sbryanv 382254738Sbryanv callout_drain(&sc->vmx_tick); 383263259Sbryanv#ifndef VMXNET3_LEGACY_TX 384263259Sbryanv vmxnet3_drain_taskqueue(sc); 385263259Sbryanv#endif 386263259Sbryanv 387263259Sbryanv ether_ifdetach(ifp); 388254738Sbryanv } 389254738Sbryanv 390254738Sbryanv if (sc->vmx_vlan_attach != NULL) { 391254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 392254738Sbryanv sc->vmx_vlan_attach = NULL; 393254738Sbryanv } 394254738Sbryanv if (sc->vmx_vlan_detach != NULL) { 395254738Sbryanv EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 396254738Sbryanv sc->vmx_vlan_detach = NULL; 397254738Sbryanv } 398254738Sbryanv 399263259Sbryanv#ifndef VMXNET3_LEGACY_TX 400263259Sbryanv vmxnet3_free_taskqueue(sc); 401263259Sbryanv#endif 402254738Sbryanv vmxnet3_free_interrupts(sc); 403254738Sbryanv 404344272Svmaffione#ifdef DEV_NETMAP 405344272Svmaffione netmap_detach(ifp); 406344272Svmaffione#endif 407344272Svmaffione 408254738Sbryanv if (ifp != NULL) { 409254738Sbryanv if_free(ifp); 410254738Sbryanv sc->vmx_ifp = NULL; 411254738Sbryanv } 412254738Sbryanv 413254738Sbryanv ifmedia_removeall(&sc->vmx_media); 414254738Sbryanv 415254738Sbryanv vmxnet3_free_data(sc); 416254738Sbryanv vmxnet3_free_resources(sc); 417254738Sbryanv vmxnet3_free_rxtx_queues(sc); 418254738Sbryanv 419254738Sbryanv VMXNET3_CORE_LOCK_DESTROY(sc); 420254738Sbryanv 421254738Sbryanv return (0); 422254738Sbryanv} 423254738Sbryanv 424254738Sbryanvstatic int 425254738Sbryanvvmxnet3_shutdown(device_t dev) 426254738Sbryanv{ 427254738Sbryanv 428254738Sbryanv return (0); 429254738Sbryanv} 430254738Sbryanv 431254738Sbryanvstatic int 432254738Sbryanvvmxnet3_alloc_resources(struct vmxnet3_softc *sc) 433254738Sbryanv{ 434254738Sbryanv device_t dev; 435254738Sbryanv int rid; 436254738Sbryanv 437254738Sbryanv dev = sc->vmx_dev; 438254738Sbryanv 439254738Sbryanv rid = PCIR_BAR(0); 440254738Sbryanv sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 441254738Sbryanv RF_ACTIVE); 442254738Sbryanv if (sc->vmx_res0 == NULL) { 443254738Sbryanv device_printf(dev, 444254738Sbryanv "could not map BAR0 memory\n"); 445254738Sbryanv return (ENXIO); 446254738Sbryanv } 447254738Sbryanv 448254738Sbryanv sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 449254738Sbryanv sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 450254738Sbryanv 451254738Sbryanv rid = PCIR_BAR(1); 452254738Sbryanv sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 453254738Sbryanv RF_ACTIVE); 454254738Sbryanv if (sc->vmx_res1 == NULL) { 455254738Sbryanv device_printf(dev, 456254738Sbryanv "could not map BAR1 memory\n"); 457254738Sbryanv return (ENXIO); 458254738Sbryanv } 459254738Sbryanv 460254738Sbryanv sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 461254738Sbryanv sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 462254738Sbryanv 463254738Sbryanv if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 464254738Sbryanv rid = PCIR_BAR(2); 465254738Sbryanv sc->vmx_msix_res = bus_alloc_resource_any(dev, 466254738Sbryanv SYS_RES_MEMORY, &rid, RF_ACTIVE); 467254738Sbryanv } 468254738Sbryanv 469254738Sbryanv if (sc->vmx_msix_res == NULL) 470254738Sbryanv sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 471254738Sbryanv 472254738Sbryanv return (0); 473254738Sbryanv} 474254738Sbryanv 475254738Sbryanvstatic void 476254738Sbryanvvmxnet3_free_resources(struct vmxnet3_softc *sc) 477254738Sbryanv{ 478254738Sbryanv device_t dev; 479254738Sbryanv int rid; 480254738Sbryanv 481254738Sbryanv dev = sc->vmx_dev; 482254738Sbryanv 483254738Sbryanv if (sc->vmx_res0 != NULL) { 484254738Sbryanv rid = PCIR_BAR(0); 485254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 486254738Sbryanv sc->vmx_res0 = NULL; 487254738Sbryanv } 488254738Sbryanv 489254738Sbryanv if (sc->vmx_res1 != NULL) { 490254738Sbryanv rid = PCIR_BAR(1); 491254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 492254738Sbryanv sc->vmx_res1 = NULL; 493254738Sbryanv } 494254738Sbryanv 495254738Sbryanv if (sc->vmx_msix_res != NULL) { 496254738Sbryanv rid = PCIR_BAR(2); 497254738Sbryanv bus_release_resource(dev, SYS_RES_MEMORY, rid, 498254738Sbryanv sc->vmx_msix_res); 499254738Sbryanv sc->vmx_msix_res = NULL; 500254738Sbryanv } 501254738Sbryanv} 502254738Sbryanv 503254738Sbryanvstatic int 504254738Sbryanvvmxnet3_check_version(struct vmxnet3_softc *sc) 505254738Sbryanv{ 506254738Sbryanv device_t dev; 507254738Sbryanv uint32_t version; 508254738Sbryanv 509254738Sbryanv dev = sc->vmx_dev; 510254738Sbryanv 511254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 512254738Sbryanv if ((version & 0x01) == 0) { 513254738Sbryanv device_printf(dev, "unsupported hardware version %#x\n", 514254738Sbryanv version); 515254738Sbryanv return (ENOTSUP); 516254950Sbryanv } 517254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 518254738Sbryanv 519254738Sbryanv version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 520254738Sbryanv if ((version & 0x01) == 0) { 521254738Sbryanv device_printf(dev, "unsupported UPT version %#x\n", version); 522254738Sbryanv return (ENOTSUP); 523254950Sbryanv } 524254950Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 525254738Sbryanv 526254738Sbryanv return (0); 527254738Sbryanv} 528254738Sbryanv 529290948Sjhbstatic int 530290948Sjhbtrunc_powerof2(int val) 531290948Sjhb{ 532290948Sjhb 533290948Sjhb return (1U << (fls(val) - 1)); 534290948Sjhb} 535290948Sjhb 536254738Sbryanvstatic void 537254738Sbryanvvmxnet3_initial_config(struct vmxnet3_softc *sc) 538254738Sbryanv{ 539263259Sbryanv int nqueue, ndesc; 540254738Sbryanv 541263259Sbryanv nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 542263259Sbryanv if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 543263259Sbryanv nqueue = VMXNET3_DEF_TX_QUEUES; 544263259Sbryanv if (nqueue > mp_ncpus) 545263259Sbryanv nqueue = mp_ncpus; 546290948Sjhb sc->vmx_max_ntxqueues = trunc_powerof2(nqueue); 547255055Sbryanv 548263259Sbryanv nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 549263259Sbryanv if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 550263259Sbryanv nqueue = VMXNET3_DEF_RX_QUEUES; 551263259Sbryanv if (nqueue > mp_ncpus) 552263259Sbryanv nqueue = mp_ncpus; 553290948Sjhb sc->vmx_max_nrxqueues = trunc_powerof2(nqueue); 554263259Sbryanv 555263259Sbryanv if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 556263259Sbryanv sc->vmx_max_nrxqueues = 1; 557263259Sbryanv sc->vmx_max_ntxqueues = 1; 558263259Sbryanv } 559263259Sbryanv 560255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 561255055Sbryanv if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 562255055Sbryanv ndesc = VMXNET3_DEF_TX_NDESC; 563255055Sbryanv if (ndesc & VMXNET3_MASK_TX_NDESC) 564255055Sbryanv ndesc &= ~VMXNET3_MASK_TX_NDESC; 565255055Sbryanv sc->vmx_ntxdescs = ndesc; 566255055Sbryanv 567255055Sbryanv ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 568255055Sbryanv if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 569255055Sbryanv ndesc = VMXNET3_DEF_RX_NDESC; 570255055Sbryanv if (ndesc & VMXNET3_MASK_RX_NDESC) 571255055Sbryanv ndesc &= ~VMXNET3_MASK_RX_NDESC; 572255055Sbryanv sc->vmx_nrxdescs = ndesc; 573254738Sbryanv sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 574254738Sbryanv} 575254738Sbryanv 576263259Sbryanvstatic void 577263259Sbryanvvmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 578263259Sbryanv{ 579263259Sbryanv 580263259Sbryanv if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 581263259Sbryanv goto out; 582263259Sbryanv 583263259Sbryanv /* BMV: Just use the maximum configured for now. */ 584263259Sbryanv sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 585263259Sbryanv sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 586263259Sbryanv 587263259Sbryanv if (sc->vmx_nrxqueues > 1) 588263259Sbryanv sc->vmx_flags |= VMXNET3_FLAG_RSS; 589263259Sbryanv 590263259Sbryanv return; 591263259Sbryanv 592263259Sbryanvout: 593263259Sbryanv sc->vmx_ntxqueues = 1; 594263259Sbryanv sc->vmx_nrxqueues = 1; 595263259Sbryanv} 596263259Sbryanv 597254738Sbryanvstatic int 598254738Sbryanvvmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 599254738Sbryanv{ 600254738Sbryanv device_t dev; 601254738Sbryanv int nmsix, cnt, required; 602254738Sbryanv 603254738Sbryanv dev = sc->vmx_dev; 604254738Sbryanv 605254738Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 606254738Sbryanv return (1); 607254738Sbryanv 608254738Sbryanv /* Allocate an additional vector for the events interrupt. */ 609263259Sbryanv required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 610254738Sbryanv 611254738Sbryanv nmsix = pci_msix_count(dev); 612254738Sbryanv if (nmsix < required) 613254738Sbryanv return (1); 614254738Sbryanv 615254738Sbryanv cnt = required; 616254738Sbryanv if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 617254738Sbryanv sc->vmx_nintrs = required; 618254738Sbryanv return (0); 619254738Sbryanv } else 620254738Sbryanv pci_release_msi(dev); 621254738Sbryanv 622263259Sbryanv /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 623263259Sbryanv 624254738Sbryanv return (1); 625254738Sbryanv} 626254738Sbryanv 627254738Sbryanvstatic int 628254738Sbryanvvmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 629254738Sbryanv{ 630254738Sbryanv device_t dev; 631254738Sbryanv int nmsi, cnt, required; 632254738Sbryanv 633254738Sbryanv dev = sc->vmx_dev; 634254738Sbryanv required = 1; 635254738Sbryanv 636254738Sbryanv nmsi = pci_msi_count(dev); 637254738Sbryanv if (nmsi < required) 638254738Sbryanv return (1); 639254738Sbryanv 640254738Sbryanv cnt = required; 641254738Sbryanv if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 642254738Sbryanv sc->vmx_nintrs = 1; 643254738Sbryanv return (0); 644254738Sbryanv } else 645254738Sbryanv pci_release_msi(dev); 646254738Sbryanv 647254738Sbryanv return (1); 648254738Sbryanv} 649254738Sbryanv 650254738Sbryanvstatic int 651254738Sbryanvvmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 652254738Sbryanv{ 653254738Sbryanv 654254738Sbryanv sc->vmx_nintrs = 1; 655254738Sbryanv return (0); 656254738Sbryanv} 657254738Sbryanv 658254738Sbryanvstatic int 659254738Sbryanvvmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 660254738Sbryanv struct vmxnet3_interrupt *intr) 661254738Sbryanv{ 662254738Sbryanv struct resource *irq; 663254738Sbryanv 664254738Sbryanv irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 665254738Sbryanv if (irq == NULL) 666254738Sbryanv return (ENXIO); 667254738Sbryanv 668254738Sbryanv intr->vmxi_irq = irq; 669254738Sbryanv intr->vmxi_rid = rid; 670254738Sbryanv 671254738Sbryanv return (0); 672254738Sbryanv} 673254738Sbryanv 674254738Sbryanvstatic int 675254738Sbryanvvmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 676254738Sbryanv{ 677254738Sbryanv int i, rid, flags, error; 678254738Sbryanv 679254738Sbryanv rid = 0; 680254738Sbryanv flags = RF_ACTIVE; 681254738Sbryanv 682254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 683254738Sbryanv flags |= RF_SHAREABLE; 684254738Sbryanv else 685254738Sbryanv rid = 1; 686254738Sbryanv 687254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 688254738Sbryanv error = vmxnet3_alloc_interrupt(sc, rid, flags, 689254738Sbryanv &sc->vmx_intrs[i]); 690254738Sbryanv if (error) 691254738Sbryanv return (error); 692254738Sbryanv } 693254738Sbryanv 694254738Sbryanv return (0); 695254738Sbryanv} 696254738Sbryanv 697254738Sbryanvstatic int 698254738Sbryanvvmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 699254738Sbryanv{ 700254738Sbryanv device_t dev; 701254738Sbryanv struct vmxnet3_txqueue *txq; 702254738Sbryanv struct vmxnet3_rxqueue *rxq; 703254738Sbryanv struct vmxnet3_interrupt *intr; 704254738Sbryanv enum intr_type type; 705254738Sbryanv int i, error; 706254738Sbryanv 707254738Sbryanv dev = sc->vmx_dev; 708254738Sbryanv intr = &sc->vmx_intrs[0]; 709254738Sbryanv type = INTR_TYPE_NET | INTR_MPSAFE; 710254738Sbryanv 711254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 712254738Sbryanv txq = &sc->vmx_txq[i]; 713254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 714254738Sbryanv vmxnet3_txq_intr, txq, &intr->vmxi_handler); 715254738Sbryanv if (error) 716254738Sbryanv return (error); 717268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 718268012Sbryanv "tq%d", i); 719254738Sbryanv txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 720254738Sbryanv } 721254738Sbryanv 722254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 723254738Sbryanv rxq = &sc->vmx_rxq[i]; 724254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 725254738Sbryanv vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 726254738Sbryanv if (error) 727254738Sbryanv return (error); 728268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 729268012Sbryanv "rq%d", i); 730254738Sbryanv rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 731254738Sbryanv } 732254738Sbryanv 733254738Sbryanv error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 734254738Sbryanv vmxnet3_event_intr, sc, &intr->vmxi_handler); 735254738Sbryanv if (error) 736254738Sbryanv return (error); 737268012Sbryanv bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 738254738Sbryanv sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 739254738Sbryanv 740254738Sbryanv return (0); 741254738Sbryanv} 742254738Sbryanv 743254738Sbryanvstatic int 744254738Sbryanvvmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 745254738Sbryanv{ 746254738Sbryanv struct vmxnet3_interrupt *intr; 747254738Sbryanv int i, error; 748254738Sbryanv 749254738Sbryanv intr = &sc->vmx_intrs[0]; 750254738Sbryanv error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 751254738Sbryanv INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 752254738Sbryanv &intr->vmxi_handler); 753254738Sbryanv 754254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 755254738Sbryanv sc->vmx_txq[i].vxtxq_intr_idx = 0; 756254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 757254738Sbryanv sc->vmx_rxq[i].vxrxq_intr_idx = 0; 758254738Sbryanv sc->vmx_event_intr_idx = 0; 759254738Sbryanv 760254738Sbryanv return (error); 761254738Sbryanv} 762254738Sbryanv 763254738Sbryanvstatic void 764254738Sbryanvvmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 765254738Sbryanv{ 766254738Sbryanv struct vmxnet3_txqueue *txq; 767254738Sbryanv struct vmxnet3_txq_shared *txs; 768254738Sbryanv struct vmxnet3_rxqueue *rxq; 769254738Sbryanv struct vmxnet3_rxq_shared *rxs; 770254738Sbryanv int i; 771254738Sbryanv 772254738Sbryanv sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 773254738Sbryanv 774254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 775254738Sbryanv txq = &sc->vmx_txq[i]; 776254738Sbryanv txs = txq->vxtxq_ts; 777254738Sbryanv txs->intr_idx = txq->vxtxq_intr_idx; 778254738Sbryanv } 779254738Sbryanv 780254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 781254738Sbryanv rxq = &sc->vmx_rxq[i]; 782254738Sbryanv rxs = rxq->vxrxq_rs; 783254738Sbryanv rxs->intr_idx = rxq->vxrxq_intr_idx; 784254738Sbryanv } 785254738Sbryanv} 786254738Sbryanv 787254738Sbryanvstatic int 788254738Sbryanvvmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 789254738Sbryanv{ 790254738Sbryanv int error; 791254738Sbryanv 792254738Sbryanv error = vmxnet3_alloc_intr_resources(sc); 793254738Sbryanv if (error) 794254738Sbryanv return (error); 795254738Sbryanv 796254738Sbryanv switch (sc->vmx_intr_type) { 797254738Sbryanv case VMXNET3_IT_MSIX: 798254738Sbryanv error = vmxnet3_setup_msix_interrupts(sc); 799254738Sbryanv break; 800254738Sbryanv case VMXNET3_IT_MSI: 801254738Sbryanv case VMXNET3_IT_LEGACY: 802254738Sbryanv error = vmxnet3_setup_legacy_interrupt(sc); 803254738Sbryanv break; 804254738Sbryanv default: 805254738Sbryanv panic("%s: invalid interrupt type %d", __func__, 806254738Sbryanv sc->vmx_intr_type); 807254738Sbryanv } 808254738Sbryanv 809254738Sbryanv if (error == 0) 810254738Sbryanv vmxnet3_set_interrupt_idx(sc); 811254738Sbryanv 812254738Sbryanv return (error); 813254738Sbryanv} 814254738Sbryanv 815254738Sbryanvstatic int 816254738Sbryanvvmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 817254738Sbryanv{ 818254738Sbryanv device_t dev; 819254738Sbryanv uint32_t config; 820254738Sbryanv int error; 821254738Sbryanv 822254738Sbryanv dev = sc->vmx_dev; 823254738Sbryanv config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 824254738Sbryanv 825254738Sbryanv sc->vmx_intr_type = config & 0x03; 826254738Sbryanv sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 827254738Sbryanv 828254738Sbryanv switch (sc->vmx_intr_type) { 829254738Sbryanv case VMXNET3_IT_AUTO: 830254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSIX; 831254738Sbryanv /* FALLTHROUGH */ 832254738Sbryanv case VMXNET3_IT_MSIX: 833254738Sbryanv error = vmxnet3_alloc_msix_interrupts(sc); 834254738Sbryanv if (error == 0) 835254738Sbryanv break; 836254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_MSI; 837254738Sbryanv /* FALLTHROUGH */ 838254738Sbryanv case VMXNET3_IT_MSI: 839254738Sbryanv error = vmxnet3_alloc_msi_interrupts(sc); 840254738Sbryanv if (error == 0) 841254738Sbryanv break; 842254738Sbryanv sc->vmx_intr_type = VMXNET3_IT_LEGACY; 843254738Sbryanv /* FALLTHROUGH */ 844254738Sbryanv case VMXNET3_IT_LEGACY: 845254738Sbryanv error = vmxnet3_alloc_legacy_interrupts(sc); 846254738Sbryanv if (error == 0) 847254738Sbryanv break; 848254738Sbryanv /* FALLTHROUGH */ 849254738Sbryanv default: 850254738Sbryanv sc->vmx_intr_type = -1; 851254738Sbryanv device_printf(dev, "cannot allocate any interrupt resources\n"); 852254738Sbryanv return (ENXIO); 853254738Sbryanv } 854254738Sbryanv 855254738Sbryanv return (error); 856254738Sbryanv} 857254738Sbryanv 858254738Sbryanvstatic void 859254738Sbryanvvmxnet3_free_interrupt(struct vmxnet3_softc *sc, 860254738Sbryanv struct vmxnet3_interrupt *intr) 861254738Sbryanv{ 862254738Sbryanv device_t dev; 863254738Sbryanv 864254738Sbryanv dev = sc->vmx_dev; 865254738Sbryanv 866254738Sbryanv if (intr->vmxi_handler != NULL) { 867254738Sbryanv bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 868254738Sbryanv intr->vmxi_handler = NULL; 869254738Sbryanv } 870254738Sbryanv 871254738Sbryanv if (intr->vmxi_irq != NULL) { 872254738Sbryanv bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 873254738Sbryanv intr->vmxi_irq); 874254738Sbryanv intr->vmxi_irq = NULL; 875254738Sbryanv intr->vmxi_rid = -1; 876254738Sbryanv } 877254738Sbryanv} 878254738Sbryanv 879254738Sbryanvstatic void 880254738Sbryanvvmxnet3_free_interrupts(struct vmxnet3_softc *sc) 881254738Sbryanv{ 882254738Sbryanv int i; 883254738Sbryanv 884254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 885254738Sbryanv vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 886254738Sbryanv 887254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_MSI || 888254738Sbryanv sc->vmx_intr_type == VMXNET3_IT_MSIX) 889254738Sbryanv pci_release_msi(sc->vmx_dev); 890254738Sbryanv} 891254738Sbryanv 892263259Sbryanv#ifndef VMXNET3_LEGACY_TX 893254738Sbryanvstatic int 894263259Sbryanvvmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 895263259Sbryanv{ 896263259Sbryanv device_t dev; 897263259Sbryanv 898263259Sbryanv dev = sc->vmx_dev; 899263259Sbryanv 900263259Sbryanv sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 901263259Sbryanv taskqueue_thread_enqueue, &sc->vmx_tq); 902263259Sbryanv if (sc->vmx_tq == NULL) 903263259Sbryanv return (ENOMEM); 904263259Sbryanv 905263259Sbryanv return (0); 906263259Sbryanv} 907263259Sbryanv 908263259Sbryanvstatic void 909263259Sbryanvvmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 910263259Sbryanv{ 911263259Sbryanv device_t dev; 912263259Sbryanv int nthreads, error; 913263259Sbryanv 914263259Sbryanv dev = sc->vmx_dev; 915263259Sbryanv 916263259Sbryanv /* 917263259Sbryanv * The taskqueue is typically not frequently used, so a dedicated 918263259Sbryanv * thread for each queue is unnecessary. 919263259Sbryanv */ 920263259Sbryanv nthreads = MAX(1, sc->vmx_ntxqueues / 2); 921263259Sbryanv 922263259Sbryanv /* 923263259Sbryanv * Most drivers just ignore the return value - it only fails 924263259Sbryanv * with ENOMEM so an error is not likely. It is hard for us 925263259Sbryanv * to recover from an error here. 926263259Sbryanv */ 927263259Sbryanv error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 928263259Sbryanv "%s taskq", device_get_nameunit(dev)); 929263259Sbryanv if (error) 930263259Sbryanv device_printf(dev, "failed to start taskqueue: %d", error); 931263259Sbryanv} 932263259Sbryanv 933263259Sbryanvstatic void 934263259Sbryanvvmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 935263259Sbryanv{ 936263259Sbryanv struct vmxnet3_txqueue *txq; 937263259Sbryanv int i; 938263259Sbryanv 939263259Sbryanv if (sc->vmx_tq != NULL) { 940263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 941263259Sbryanv txq = &sc->vmx_txq[i]; 942263259Sbryanv taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 943263259Sbryanv } 944263259Sbryanv } 945263259Sbryanv} 946263259Sbryanv 947263259Sbryanvstatic void 948263259Sbryanvvmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 949263259Sbryanv{ 950263259Sbryanv if (sc->vmx_tq != NULL) { 951263259Sbryanv taskqueue_free(sc->vmx_tq); 952263259Sbryanv sc->vmx_tq = NULL; 953263259Sbryanv } 954263259Sbryanv} 955263259Sbryanv#endif 956263259Sbryanv 957263259Sbryanvstatic int 958254738Sbryanvvmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 959254738Sbryanv{ 960254738Sbryanv struct vmxnet3_rxqueue *rxq; 961254738Sbryanv struct vmxnet3_rxring *rxr; 962254738Sbryanv int i; 963254738Sbryanv 964254738Sbryanv rxq = &sc->vmx_rxq[q]; 965254738Sbryanv 966254738Sbryanv snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 967254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 968254738Sbryanv mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 969254738Sbryanv 970254738Sbryanv rxq->vxrxq_sc = sc; 971254738Sbryanv rxq->vxrxq_id = q; 972254738Sbryanv 973254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 974254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 975254738Sbryanv rxr->vxrxr_rid = i; 976254738Sbryanv rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 977254738Sbryanv rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 978254738Sbryanv sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 979254738Sbryanv if (rxr->vxrxr_rxbuf == NULL) 980254738Sbryanv return (ENOMEM); 981254950Sbryanv 982254950Sbryanv rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 983254738Sbryanv } 984254738Sbryanv 985254738Sbryanv return (0); 986254738Sbryanv} 987254738Sbryanv 988254738Sbryanvstatic int 989254738Sbryanvvmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 990254738Sbryanv{ 991254738Sbryanv struct vmxnet3_txqueue *txq; 992254738Sbryanv struct vmxnet3_txring *txr; 993254738Sbryanv 994254738Sbryanv txq = &sc->vmx_txq[q]; 995254738Sbryanv txr = &txq->vxtxq_cmd_ring; 996254738Sbryanv 997254738Sbryanv snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 998254738Sbryanv device_get_nameunit(sc->vmx_dev), q); 999254738Sbryanv mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 1000254738Sbryanv 1001254738Sbryanv txq->vxtxq_sc = sc; 1002254738Sbryanv txq->vxtxq_id = q; 1003254738Sbryanv 1004254738Sbryanv txr->vxtxr_ndesc = sc->vmx_ntxdescs; 1005254738Sbryanv txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 1006254738Sbryanv sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 1007254738Sbryanv if (txr->vxtxr_txbuf == NULL) 1008254738Sbryanv return (ENOMEM); 1009254738Sbryanv 1010254738Sbryanv txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 1011254738Sbryanv 1012263259Sbryanv#ifndef VMXNET3_LEGACY_TX 1013263259Sbryanv TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 1014263259Sbryanv 1015263259Sbryanv txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 1016263259Sbryanv M_NOWAIT, &txq->vxtxq_mtx); 1017263259Sbryanv if (txq->vxtxq_br == NULL) 1018263259Sbryanv return (ENOMEM); 1019263259Sbryanv#endif 1020263259Sbryanv 1021254738Sbryanv return (0); 1022254738Sbryanv} 1023254738Sbryanv 1024254738Sbryanvstatic int 1025254738Sbryanvvmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1026254738Sbryanv{ 1027254738Sbryanv int i, error; 1028254738Sbryanv 1029263259Sbryanv /* 1030263259Sbryanv * Only attempt to create multiple queues if MSIX is available. MSIX is 1031263259Sbryanv * disabled by default because its apparently broken for devices passed 1032263259Sbryanv * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1033263259Sbryanv * must be set to zero for MSIX. This check prevents us from allocating 1034263259Sbryanv * queue structures that we will not use. 1035263259Sbryanv */ 1036263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1037263259Sbryanv sc->vmx_max_nrxqueues = 1; 1038263259Sbryanv sc->vmx_max_ntxqueues = 1; 1039263259Sbryanv } 1040263259Sbryanv 1041254738Sbryanv sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1042263259Sbryanv sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1043254738Sbryanv sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1044263259Sbryanv sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1045254738Sbryanv if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1046254738Sbryanv return (ENOMEM); 1047254738Sbryanv 1048263259Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1049254738Sbryanv error = vmxnet3_init_rxq(sc, i); 1050254738Sbryanv if (error) 1051254738Sbryanv return (error); 1052254738Sbryanv } 1053254738Sbryanv 1054263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1055254738Sbryanv error = vmxnet3_init_txq(sc, i); 1056254738Sbryanv if (error) 1057254738Sbryanv return (error); 1058254738Sbryanv } 1059254738Sbryanv 1060254738Sbryanv return (0); 1061254738Sbryanv} 1062254738Sbryanv 1063254738Sbryanvstatic void 1064254738Sbryanvvmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1065254738Sbryanv{ 1066254738Sbryanv struct vmxnet3_rxring *rxr; 1067254738Sbryanv int i; 1068254738Sbryanv 1069254738Sbryanv rxq->vxrxq_sc = NULL; 1070254738Sbryanv rxq->vxrxq_id = -1; 1071254738Sbryanv 1072254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1073254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1074254738Sbryanv 1075254738Sbryanv if (rxr->vxrxr_rxbuf != NULL) { 1076254738Sbryanv free(rxr->vxrxr_rxbuf, M_DEVBUF); 1077254738Sbryanv rxr->vxrxr_rxbuf = NULL; 1078254738Sbryanv } 1079254738Sbryanv } 1080254738Sbryanv 1081254738Sbryanv if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1082254738Sbryanv mtx_destroy(&rxq->vxrxq_mtx); 1083254738Sbryanv} 1084254738Sbryanv 1085254738Sbryanvstatic void 1086254738Sbryanvvmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1087254738Sbryanv{ 1088254738Sbryanv struct vmxnet3_txring *txr; 1089254738Sbryanv 1090254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1091254738Sbryanv 1092254738Sbryanv txq->vxtxq_sc = NULL; 1093254738Sbryanv txq->vxtxq_id = -1; 1094254738Sbryanv 1095263259Sbryanv#ifndef VMXNET3_LEGACY_TX 1096263259Sbryanv if (txq->vxtxq_br != NULL) { 1097263259Sbryanv buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1098263259Sbryanv txq->vxtxq_br = NULL; 1099263259Sbryanv } 1100263259Sbryanv#endif 1101263259Sbryanv 1102254738Sbryanv if (txr->vxtxr_txbuf != NULL) { 1103254738Sbryanv free(txr->vxtxr_txbuf, M_DEVBUF); 1104254738Sbryanv txr->vxtxr_txbuf = NULL; 1105254738Sbryanv } 1106254738Sbryanv 1107254738Sbryanv if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1108254738Sbryanv mtx_destroy(&txq->vxtxq_mtx); 1109254738Sbryanv} 1110254738Sbryanv 1111254738Sbryanvstatic void 1112254738Sbryanvvmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1113254738Sbryanv{ 1114254738Sbryanv int i; 1115254738Sbryanv 1116254738Sbryanv if (sc->vmx_rxq != NULL) { 1117263259Sbryanv for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1118254738Sbryanv vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1119254738Sbryanv free(sc->vmx_rxq, M_DEVBUF); 1120254738Sbryanv sc->vmx_rxq = NULL; 1121254738Sbryanv } 1122254738Sbryanv 1123254738Sbryanv if (sc->vmx_txq != NULL) { 1124263259Sbryanv for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1125254738Sbryanv vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1126254738Sbryanv free(sc->vmx_txq, M_DEVBUF); 1127254738Sbryanv sc->vmx_txq = NULL; 1128254738Sbryanv } 1129254738Sbryanv} 1130254738Sbryanv 1131254738Sbryanvstatic int 1132254738Sbryanvvmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1133254738Sbryanv{ 1134254738Sbryanv device_t dev; 1135254738Sbryanv uint8_t *kva; 1136254738Sbryanv size_t size; 1137254738Sbryanv int i, error; 1138254738Sbryanv 1139254738Sbryanv dev = sc->vmx_dev; 1140254738Sbryanv 1141254738Sbryanv size = sizeof(struct vmxnet3_driver_shared); 1142254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1143254738Sbryanv if (error) { 1144254738Sbryanv device_printf(dev, "cannot alloc shared memory\n"); 1145254738Sbryanv return (error); 1146254738Sbryanv } 1147254738Sbryanv sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1148254738Sbryanv 1149254738Sbryanv size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1150254738Sbryanv sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1151254738Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1152254738Sbryanv if (error) { 1153254738Sbryanv device_printf(dev, "cannot alloc queue shared memory\n"); 1154254738Sbryanv return (error); 1155254738Sbryanv } 1156254738Sbryanv sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1157254738Sbryanv kva = sc->vmx_qs; 1158254738Sbryanv 1159254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1160254738Sbryanv sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1161254738Sbryanv kva += sizeof(struct vmxnet3_txq_shared); 1162254738Sbryanv } 1163254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1164254738Sbryanv sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1165254738Sbryanv kva += sizeof(struct vmxnet3_rxq_shared); 1166254738Sbryanv } 1167254738Sbryanv 1168263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1169263259Sbryanv size = sizeof(struct vmxnet3_rss_shared); 1170263259Sbryanv error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1171263259Sbryanv if (error) { 1172263259Sbryanv device_printf(dev, "cannot alloc rss shared memory\n"); 1173263259Sbryanv return (error); 1174263259Sbryanv } 1175263259Sbryanv sc->vmx_rss = 1176263259Sbryanv (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1177263259Sbryanv } 1178263259Sbryanv 1179254738Sbryanv return (0); 1180254738Sbryanv} 1181254738Sbryanv 1182254738Sbryanvstatic void 1183254738Sbryanvvmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1184254738Sbryanv{ 1185254738Sbryanv 1186263259Sbryanv if (sc->vmx_rss != NULL) { 1187263259Sbryanv vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1188263259Sbryanv sc->vmx_rss = NULL; 1189263259Sbryanv } 1190263259Sbryanv 1191254738Sbryanv if (sc->vmx_qs != NULL) { 1192254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1193254738Sbryanv sc->vmx_qs = NULL; 1194254738Sbryanv } 1195254738Sbryanv 1196254738Sbryanv if (sc->vmx_ds != NULL) { 1197254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1198254738Sbryanv sc->vmx_ds = NULL; 1199254738Sbryanv } 1200254738Sbryanv} 1201254738Sbryanv 1202254738Sbryanvstatic int 1203254738Sbryanvvmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1204254738Sbryanv{ 1205254738Sbryanv device_t dev; 1206254738Sbryanv struct vmxnet3_txqueue *txq; 1207254738Sbryanv struct vmxnet3_txring *txr; 1208254738Sbryanv struct vmxnet3_comp_ring *txc; 1209254738Sbryanv size_t descsz, compsz; 1210254738Sbryanv int i, q, error; 1211254738Sbryanv 1212254738Sbryanv dev = sc->vmx_dev; 1213254738Sbryanv 1214254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1215254738Sbryanv txq = &sc->vmx_txq[q]; 1216254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1217254738Sbryanv txc = &txq->vxtxq_comp_ring; 1218254738Sbryanv 1219254738Sbryanv descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1220254738Sbryanv compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1221254738Sbryanv 1222254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1223254738Sbryanv 1, 0, /* alignment, boundary */ 1224254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1225254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1226254738Sbryanv NULL, NULL, /* filter, filterarg */ 1227263259Sbryanv VMXNET3_TX_MAXSIZE, /* maxsize */ 1228254738Sbryanv VMXNET3_TX_MAXSEGS, /* nsegments */ 1229254738Sbryanv VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1230254738Sbryanv 0, /* flags */ 1231254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1232254738Sbryanv &txr->vxtxr_txtag); 1233254738Sbryanv if (error) { 1234254738Sbryanv device_printf(dev, 1235254738Sbryanv "unable to create Tx buffer tag for queue %d\n", q); 1236254738Sbryanv return (error); 1237254738Sbryanv } 1238254738Sbryanv 1239254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1240254738Sbryanv if (error) { 1241254738Sbryanv device_printf(dev, "cannot alloc Tx descriptors for " 1242254738Sbryanv "queue %d error %d\n", q, error); 1243254738Sbryanv return (error); 1244254738Sbryanv } 1245254738Sbryanv txr->vxtxr_txd = 1246254738Sbryanv (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1247254738Sbryanv 1248254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1249254738Sbryanv if (error) { 1250254738Sbryanv device_printf(dev, "cannot alloc Tx comp descriptors " 1251254738Sbryanv "for queue %d error %d\n", q, error); 1252254738Sbryanv return (error); 1253254738Sbryanv } 1254254738Sbryanv txc->vxcr_u.txcd = 1255254738Sbryanv (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1256254738Sbryanv 1257254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1258254738Sbryanv error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1259254738Sbryanv &txr->vxtxr_txbuf[i].vtxb_dmamap); 1260254738Sbryanv if (error) { 1261254738Sbryanv device_printf(dev, "unable to create Tx buf " 1262254738Sbryanv "dmamap for queue %d idx %d\n", q, i); 1263254738Sbryanv return (error); 1264254738Sbryanv } 1265254738Sbryanv } 1266254738Sbryanv } 1267254738Sbryanv 1268254738Sbryanv return (0); 1269254738Sbryanv} 1270254738Sbryanv 1271254738Sbryanvstatic void 1272254738Sbryanvvmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1273254738Sbryanv{ 1274254738Sbryanv device_t dev; 1275254738Sbryanv struct vmxnet3_txqueue *txq; 1276254738Sbryanv struct vmxnet3_txring *txr; 1277254738Sbryanv struct vmxnet3_comp_ring *txc; 1278254738Sbryanv struct vmxnet3_txbuf *txb; 1279254738Sbryanv int i, q; 1280254738Sbryanv 1281254738Sbryanv dev = sc->vmx_dev; 1282254738Sbryanv 1283254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) { 1284254738Sbryanv txq = &sc->vmx_txq[q]; 1285254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1286254738Sbryanv txc = &txq->vxtxq_comp_ring; 1287254738Sbryanv 1288254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 1289254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 1290254738Sbryanv if (txb->vtxb_dmamap != NULL) { 1291254738Sbryanv bus_dmamap_destroy(txr->vxtxr_txtag, 1292254738Sbryanv txb->vtxb_dmamap); 1293254738Sbryanv txb->vtxb_dmamap = NULL; 1294254738Sbryanv } 1295254738Sbryanv } 1296254738Sbryanv 1297254738Sbryanv if (txc->vxcr_u.txcd != NULL) { 1298254738Sbryanv vmxnet3_dma_free(sc, &txc->vxcr_dma); 1299254738Sbryanv txc->vxcr_u.txcd = NULL; 1300254738Sbryanv } 1301254738Sbryanv 1302254738Sbryanv if (txr->vxtxr_txd != NULL) { 1303254738Sbryanv vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1304254738Sbryanv txr->vxtxr_txd = NULL; 1305254738Sbryanv } 1306254738Sbryanv 1307254738Sbryanv if (txr->vxtxr_txtag != NULL) { 1308254738Sbryanv bus_dma_tag_destroy(txr->vxtxr_txtag); 1309254738Sbryanv txr->vxtxr_txtag = NULL; 1310254738Sbryanv } 1311254738Sbryanv } 1312254738Sbryanv} 1313254738Sbryanv 1314254738Sbryanvstatic int 1315254738Sbryanvvmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1316254738Sbryanv{ 1317254738Sbryanv device_t dev; 1318254738Sbryanv struct vmxnet3_rxqueue *rxq; 1319254738Sbryanv struct vmxnet3_rxring *rxr; 1320254738Sbryanv struct vmxnet3_comp_ring *rxc; 1321254738Sbryanv int descsz, compsz; 1322254738Sbryanv int i, j, q, error; 1323254738Sbryanv 1324254738Sbryanv dev = sc->vmx_dev; 1325254738Sbryanv 1326254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1327254738Sbryanv rxq = &sc->vmx_rxq[q]; 1328254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1329254738Sbryanv compsz = 0; 1330254738Sbryanv 1331254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1332254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1333254738Sbryanv 1334254738Sbryanv descsz = rxr->vxrxr_ndesc * 1335254738Sbryanv sizeof(struct vmxnet3_rxdesc); 1336254738Sbryanv compsz += rxr->vxrxr_ndesc * 1337254738Sbryanv sizeof(struct vmxnet3_rxcompdesc); 1338254738Sbryanv 1339254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 1340254738Sbryanv 1, 0, /* alignment, boundary */ 1341254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 1342254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 1343254738Sbryanv NULL, NULL, /* filter, filterarg */ 1344254738Sbryanv MJUMPAGESIZE, /* maxsize */ 1345254738Sbryanv 1, /* nsegments */ 1346254738Sbryanv MJUMPAGESIZE, /* maxsegsize */ 1347254738Sbryanv 0, /* flags */ 1348254738Sbryanv NULL, NULL, /* lockfunc, lockarg */ 1349254738Sbryanv &rxr->vxrxr_rxtag); 1350254738Sbryanv if (error) { 1351254738Sbryanv device_printf(dev, 1352254738Sbryanv "unable to create Rx buffer tag for " 1353254738Sbryanv "queue %d\n", q); 1354254738Sbryanv return (error); 1355254738Sbryanv } 1356254738Sbryanv 1357254738Sbryanv error = vmxnet3_dma_malloc(sc, descsz, 512, 1358254738Sbryanv &rxr->vxrxr_dma); 1359254738Sbryanv if (error) { 1360254738Sbryanv device_printf(dev, "cannot allocate Rx " 1361254738Sbryanv "descriptors for queue %d/%d error %d\n", 1362254738Sbryanv i, q, error); 1363254738Sbryanv return (error); 1364254738Sbryanv } 1365254738Sbryanv rxr->vxrxr_rxd = 1366254738Sbryanv (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1367254738Sbryanv } 1368254738Sbryanv 1369254738Sbryanv error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1370254738Sbryanv if (error) { 1371254738Sbryanv device_printf(dev, "cannot alloc Rx comp descriptors " 1372254738Sbryanv "for queue %d error %d\n", q, error); 1373254738Sbryanv return (error); 1374254738Sbryanv } 1375254738Sbryanv rxc->vxcr_u.rxcd = 1376254738Sbryanv (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1377254738Sbryanv 1378254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1379254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1380254738Sbryanv 1381254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1382254738Sbryanv &rxr->vxrxr_spare_dmap); 1383254738Sbryanv if (error) { 1384254738Sbryanv device_printf(dev, "unable to create spare " 1385254738Sbryanv "dmamap for queue %d/%d error %d\n", 1386254738Sbryanv q, i, error); 1387254738Sbryanv return (error); 1388254738Sbryanv } 1389254738Sbryanv 1390254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1391254738Sbryanv error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1392254738Sbryanv &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1393254738Sbryanv if (error) { 1394254738Sbryanv device_printf(dev, "unable to create " 1395254738Sbryanv "dmamap for queue %d/%d slot %d " 1396254738Sbryanv "error %d\n", 1397254738Sbryanv q, i, j, error); 1398254738Sbryanv return (error); 1399254738Sbryanv } 1400254738Sbryanv } 1401254738Sbryanv } 1402254738Sbryanv } 1403254738Sbryanv 1404254738Sbryanv return (0); 1405254738Sbryanv} 1406254738Sbryanv 1407254738Sbryanvstatic void 1408254738Sbryanvvmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1409254738Sbryanv{ 1410254738Sbryanv device_t dev; 1411254738Sbryanv struct vmxnet3_rxqueue *rxq; 1412254738Sbryanv struct vmxnet3_rxring *rxr; 1413254738Sbryanv struct vmxnet3_comp_ring *rxc; 1414254738Sbryanv struct vmxnet3_rxbuf *rxb; 1415254738Sbryanv int i, j, q; 1416254738Sbryanv 1417254738Sbryanv dev = sc->vmx_dev; 1418254738Sbryanv 1419254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 1420254738Sbryanv rxq = &sc->vmx_rxq[q]; 1421254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 1422254738Sbryanv 1423254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1424254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1425254738Sbryanv 1426254738Sbryanv if (rxr->vxrxr_spare_dmap != NULL) { 1427254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1428254738Sbryanv rxr->vxrxr_spare_dmap); 1429254738Sbryanv rxr->vxrxr_spare_dmap = NULL; 1430254738Sbryanv } 1431254738Sbryanv 1432254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1433254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 1434254738Sbryanv if (rxb->vrxb_dmamap != NULL) { 1435254738Sbryanv bus_dmamap_destroy(rxr->vxrxr_rxtag, 1436254738Sbryanv rxb->vrxb_dmamap); 1437254738Sbryanv rxb->vrxb_dmamap = NULL; 1438254738Sbryanv } 1439254738Sbryanv } 1440254738Sbryanv } 1441254738Sbryanv 1442254738Sbryanv if (rxc->vxcr_u.rxcd != NULL) { 1443254738Sbryanv vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1444254738Sbryanv rxc->vxcr_u.rxcd = NULL; 1445254738Sbryanv } 1446254738Sbryanv 1447254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1448254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 1449254738Sbryanv 1450254738Sbryanv if (rxr->vxrxr_rxd != NULL) { 1451254738Sbryanv vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1452254738Sbryanv rxr->vxrxr_rxd = NULL; 1453254738Sbryanv } 1454254738Sbryanv 1455254738Sbryanv if (rxr->vxrxr_rxtag != NULL) { 1456254738Sbryanv bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1457254738Sbryanv rxr->vxrxr_rxtag = NULL; 1458254738Sbryanv } 1459254738Sbryanv } 1460254738Sbryanv } 1461254738Sbryanv} 1462254738Sbryanv 1463254738Sbryanvstatic int 1464254738Sbryanvvmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1465254738Sbryanv{ 1466254738Sbryanv int error; 1467254738Sbryanv 1468254738Sbryanv error = vmxnet3_alloc_txq_data(sc); 1469254738Sbryanv if (error) 1470254738Sbryanv return (error); 1471254738Sbryanv 1472254738Sbryanv error = vmxnet3_alloc_rxq_data(sc); 1473254738Sbryanv if (error) 1474254738Sbryanv return (error); 1475254738Sbryanv 1476254738Sbryanv return (0); 1477254738Sbryanv} 1478254738Sbryanv 1479254738Sbryanvstatic void 1480254738Sbryanvvmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1481254738Sbryanv{ 1482254738Sbryanv 1483254950Sbryanv if (sc->vmx_rxq != NULL) 1484254950Sbryanv vmxnet3_free_rxq_data(sc); 1485254950Sbryanv 1486254950Sbryanv if (sc->vmx_txq != NULL) 1487254950Sbryanv vmxnet3_free_txq_data(sc); 1488254738Sbryanv} 1489254738Sbryanv 1490254738Sbryanvstatic int 1491254738Sbryanvvmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1492254738Sbryanv{ 1493254738Sbryanv int error; 1494254738Sbryanv 1495254738Sbryanv error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1496254738Sbryanv 32, &sc->vmx_mcast_dma); 1497254738Sbryanv if (error) 1498254738Sbryanv device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1499254738Sbryanv else 1500254738Sbryanv sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1501254738Sbryanv 1502254738Sbryanv return (error); 1503254738Sbryanv} 1504254738Sbryanv 1505254738Sbryanvstatic void 1506254738Sbryanvvmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1507254738Sbryanv{ 1508254738Sbryanv 1509254738Sbryanv if (sc->vmx_mcast != NULL) { 1510254738Sbryanv vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1511254738Sbryanv sc->vmx_mcast = NULL; 1512254738Sbryanv } 1513254738Sbryanv} 1514254738Sbryanv 1515254738Sbryanvstatic void 1516254738Sbryanvvmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1517254738Sbryanv{ 1518254738Sbryanv struct vmxnet3_driver_shared *ds; 1519254738Sbryanv struct vmxnet3_txqueue *txq; 1520254738Sbryanv struct vmxnet3_txq_shared *txs; 1521254738Sbryanv struct vmxnet3_rxqueue *rxq; 1522254738Sbryanv struct vmxnet3_rxq_shared *rxs; 1523254738Sbryanv int i; 1524254738Sbryanv 1525254738Sbryanv ds = sc->vmx_ds; 1526254738Sbryanv 1527254738Sbryanv /* 1528254738Sbryanv * Initialize fields of the shared data that remains the same across 1529254738Sbryanv * reinits. Note the shared data is zero'd when allocated. 1530254738Sbryanv */ 1531254738Sbryanv 1532254738Sbryanv ds->magic = VMXNET3_REV1_MAGIC; 1533254738Sbryanv 1534254738Sbryanv /* DriverInfo */ 1535254738Sbryanv ds->version = VMXNET3_DRIVER_VERSION; 1536256308Sbryanv ds->guest = VMXNET3_GOS_FREEBSD | 1537254738Sbryanv#ifdef __LP64__ 1538254738Sbryanv VMXNET3_GOS_64BIT; 1539254738Sbryanv#else 1540254738Sbryanv VMXNET3_GOS_32BIT; 1541254738Sbryanv#endif 1542254738Sbryanv ds->vmxnet3_revision = 1; 1543254738Sbryanv ds->upt_version = 1; 1544254738Sbryanv 1545254738Sbryanv /* Misc. conf */ 1546254738Sbryanv ds->driver_data = vtophys(sc); 1547254738Sbryanv ds->driver_data_len = sizeof(struct vmxnet3_softc); 1548254738Sbryanv ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1549254738Sbryanv ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1550254738Sbryanv ds->nrxsg_max = sc->vmx_max_rxsegs; 1551254738Sbryanv 1552263259Sbryanv /* RSS conf */ 1553263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1554263259Sbryanv ds->rss.version = 1; 1555263259Sbryanv ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1556263259Sbryanv ds->rss.len = sc->vmx_rss_dma.dma_size; 1557263259Sbryanv } 1558263259Sbryanv 1559254738Sbryanv /* Interrupt control. */ 1560254738Sbryanv ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1561254738Sbryanv ds->nintr = sc->vmx_nintrs; 1562254738Sbryanv ds->evintr = sc->vmx_event_intr_idx; 1563254738Sbryanv ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1564254738Sbryanv 1565254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 1566254738Sbryanv ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1567254738Sbryanv 1568254738Sbryanv /* Receive filter. */ 1569254738Sbryanv ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1570254738Sbryanv ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1571254738Sbryanv 1572254738Sbryanv /* Tx queues */ 1573254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 1574254738Sbryanv txq = &sc->vmx_txq[i]; 1575254738Sbryanv txs = txq->vxtxq_ts; 1576254738Sbryanv 1577254738Sbryanv txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1578254950Sbryanv txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1579254738Sbryanv txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1580254950Sbryanv txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1581254738Sbryanv txs->driver_data = vtophys(txq); 1582254738Sbryanv txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1583254738Sbryanv } 1584254738Sbryanv 1585254738Sbryanv /* Rx queues */ 1586254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 1587254738Sbryanv rxq = &sc->vmx_rxq[i]; 1588254738Sbryanv rxs = rxq->vxrxq_rs; 1589254738Sbryanv 1590254738Sbryanv rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1591254738Sbryanv rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1592254738Sbryanv rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1593254738Sbryanv rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1594254738Sbryanv rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1595254950Sbryanv rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1596254738Sbryanv rxs->driver_data = vtophys(rxq); 1597254738Sbryanv rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1598254738Sbryanv } 1599254738Sbryanv} 1600254738Sbryanv 1601254738Sbryanvstatic void 1602303136Smavvmxnet3_init_hwassist(struct vmxnet3_softc *sc) 1603303136Smav{ 1604303136Smav struct ifnet *ifp = sc->vmx_ifp; 1605303136Smav uint64_t hwassist; 1606303136Smav 1607303136Smav hwassist = 0; 1608303136Smav if (ifp->if_capenable & IFCAP_TXCSUM) 1609303136Smav hwassist |= VMXNET3_CSUM_OFFLOAD; 1610303136Smav if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1611303136Smav hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1612303136Smav if (ifp->if_capenable & IFCAP_TSO4) 1613303136Smav hwassist |= CSUM_IP_TSO; 1614303136Smav if (ifp->if_capenable & IFCAP_TSO6) 1615303136Smav hwassist |= CSUM_IP6_TSO; 1616303136Smav ifp->if_hwassist = hwassist; 1617303136Smav} 1618303136Smav 1619303136Smavstatic void 1620254738Sbryanvvmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1621254738Sbryanv{ 1622254738Sbryanv struct ifnet *ifp; 1623254738Sbryanv 1624254738Sbryanv ifp = sc->vmx_ifp; 1625254738Sbryanv 1626254738Sbryanv /* Use the current MAC address. */ 1627254738Sbryanv bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1628254738Sbryanv vmxnet3_set_lladdr(sc); 1629254738Sbryanv 1630303136Smav vmxnet3_init_hwassist(sc); 1631254738Sbryanv} 1632254738Sbryanv 1633254738Sbryanvstatic void 1634263259Sbryanvvmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1635263259Sbryanv{ 1636263259Sbryanv /* 1637263259Sbryanv * Use the same key as the Linux driver until FreeBSD can do 1638263259Sbryanv * RSS (presumably Toeplitz) in software. 1639263259Sbryanv */ 1640263259Sbryanv static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1641263259Sbryanv 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1642263259Sbryanv 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1643263259Sbryanv 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1644263259Sbryanv 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1645263259Sbryanv 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1646263259Sbryanv }; 1647263259Sbryanv 1648263259Sbryanv struct vmxnet3_driver_shared *ds; 1649263259Sbryanv struct vmxnet3_rss_shared *rss; 1650263259Sbryanv int i; 1651263259Sbryanv 1652263259Sbryanv ds = sc->vmx_ds; 1653263259Sbryanv rss = sc->vmx_rss; 1654263259Sbryanv 1655263259Sbryanv rss->hash_type = 1656263259Sbryanv UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1657263259Sbryanv UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1658263259Sbryanv rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1659263259Sbryanv rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1660263259Sbryanv rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1661263259Sbryanv memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1662263259Sbryanv 1663263259Sbryanv for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1664263259Sbryanv rss->ind_table[i] = i % sc->vmx_nrxqueues; 1665263259Sbryanv} 1666263259Sbryanv 1667263259Sbryanvstatic void 1668254738Sbryanvvmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1669254738Sbryanv{ 1670254738Sbryanv struct ifnet *ifp; 1671254738Sbryanv struct vmxnet3_driver_shared *ds; 1672254738Sbryanv 1673254738Sbryanv ifp = sc->vmx_ifp; 1674254738Sbryanv ds = sc->vmx_ds; 1675254738Sbryanv 1676263259Sbryanv ds->mtu = ifp->if_mtu; 1677263259Sbryanv ds->ntxqueue = sc->vmx_ntxqueues; 1678263259Sbryanv ds->nrxqueue = sc->vmx_nrxqueues; 1679263259Sbryanv 1680254738Sbryanv ds->upt_features = 0; 1681255055Sbryanv if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1682255055Sbryanv ds->upt_features |= UPT1_F_CSUM; 1683254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1684254738Sbryanv ds->upt_features |= UPT1_F_VLAN; 1685254738Sbryanv if (ifp->if_capenable & IFCAP_LRO) 1686254738Sbryanv ds->upt_features |= UPT1_F_LRO; 1687254738Sbryanv 1688263259Sbryanv if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1689263259Sbryanv ds->upt_features |= UPT1_F_RSS; 1690263259Sbryanv vmxnet3_reinit_rss_shared_data(sc); 1691263259Sbryanv } 1692254738Sbryanv 1693254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1694254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1695254738Sbryanv (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1696254738Sbryanv} 1697254738Sbryanv 1698254738Sbryanvstatic int 1699254738Sbryanvvmxnet3_alloc_data(struct vmxnet3_softc *sc) 1700254738Sbryanv{ 1701254738Sbryanv int error; 1702254738Sbryanv 1703254738Sbryanv error = vmxnet3_alloc_shared_data(sc); 1704254738Sbryanv if (error) 1705254738Sbryanv return (error); 1706254738Sbryanv 1707254738Sbryanv error = vmxnet3_alloc_queue_data(sc); 1708254738Sbryanv if (error) 1709254738Sbryanv return (error); 1710254738Sbryanv 1711254738Sbryanv error = vmxnet3_alloc_mcast_table(sc); 1712254738Sbryanv if (error) 1713254738Sbryanv return (error); 1714254738Sbryanv 1715254738Sbryanv vmxnet3_init_shared_data(sc); 1716254738Sbryanv 1717254738Sbryanv return (0); 1718254738Sbryanv} 1719254738Sbryanv 1720254738Sbryanvstatic void 1721254738Sbryanvvmxnet3_free_data(struct vmxnet3_softc *sc) 1722254738Sbryanv{ 1723254738Sbryanv 1724254738Sbryanv vmxnet3_free_mcast_table(sc); 1725254738Sbryanv vmxnet3_free_queue_data(sc); 1726254738Sbryanv vmxnet3_free_shared_data(sc); 1727254738Sbryanv} 1728254738Sbryanv 1729254738Sbryanvstatic int 1730254738Sbryanvvmxnet3_setup_interface(struct vmxnet3_softc *sc) 1731254738Sbryanv{ 1732254738Sbryanv device_t dev; 1733254738Sbryanv struct ifnet *ifp; 1734254738Sbryanv 1735254738Sbryanv dev = sc->vmx_dev; 1736254738Sbryanv 1737254738Sbryanv ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1738254738Sbryanv if (ifp == NULL) { 1739254738Sbryanv device_printf(dev, "cannot allocate ifnet structure\n"); 1740254738Sbryanv return (ENOSPC); 1741254738Sbryanv } 1742254738Sbryanv 1743254738Sbryanv if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1744263259Sbryanv#if __FreeBSD_version < 1000025 1745263259Sbryanv ifp->if_baudrate = 1000000000; 1746263259Sbryanv#elif __FreeBSD_version < 1100011 1747263259Sbryanv if_initbaudrate(ifp, IF_Gbps(10)); 1748263259Sbryanv#else 1749263102Sglebius ifp->if_baudrate = IF_Gbps(10); 1750263259Sbryanv#endif 1751254738Sbryanv ifp->if_softc = sc; 1752254738Sbryanv ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1753254738Sbryanv ifp->if_init = vmxnet3_init; 1754254738Sbryanv ifp->if_ioctl = vmxnet3_ioctl; 1755272099Sglebius ifp->if_get_counter = vmxnet3_get_counter; 1756271946Shselasky ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1757271946Shselasky ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1758271946Shselasky ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1759263259Sbryanv 1760263259Sbryanv#ifdef VMXNET3_LEGACY_TX 1761254738Sbryanv ifp->if_start = vmxnet3_start; 1762254738Sbryanv ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1763254738Sbryanv IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1764254738Sbryanv IFQ_SET_READY(&ifp->if_snd); 1765263259Sbryanv#else 1766263259Sbryanv ifp->if_transmit = vmxnet3_txq_mq_start; 1767263259Sbryanv ifp->if_qflush = vmxnet3_qflush; 1768263259Sbryanv#endif 1769254738Sbryanv 1770254738Sbryanv vmxnet3_get_lladdr(sc); 1771254738Sbryanv ether_ifattach(ifp, sc->vmx_lladdr); 1772254738Sbryanv 1773254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1774254738Sbryanv ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1775254738Sbryanv ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1776255055Sbryanv ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1777255055Sbryanv IFCAP_VLAN_HWCSUM; 1778254738Sbryanv ifp->if_capenable = ifp->if_capabilities; 1779254738Sbryanv 1780255055Sbryanv /* These capabilities are not enabled by default. */ 1781255055Sbryanv ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1782254738Sbryanv 1783254738Sbryanv sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1784254738Sbryanv vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1785254738Sbryanv sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1786254738Sbryanv vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1787254738Sbryanv 1788254738Sbryanv ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1789254738Sbryanv vmxnet3_media_status); 1790254738Sbryanv ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1791254738Sbryanv ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1792254738Sbryanv 1793254738Sbryanv return (0); 1794254738Sbryanv} 1795254738Sbryanv 1796254738Sbryanvstatic void 1797254738Sbryanvvmxnet3_evintr(struct vmxnet3_softc *sc) 1798254738Sbryanv{ 1799254738Sbryanv device_t dev; 1800254738Sbryanv struct ifnet *ifp; 1801254738Sbryanv struct vmxnet3_txq_shared *ts; 1802254738Sbryanv struct vmxnet3_rxq_shared *rs; 1803254738Sbryanv uint32_t event; 1804254738Sbryanv int reset; 1805254738Sbryanv 1806254738Sbryanv dev = sc->vmx_dev; 1807254738Sbryanv ifp = sc->vmx_ifp; 1808254738Sbryanv reset = 0; 1809254738Sbryanv 1810254738Sbryanv VMXNET3_CORE_LOCK(sc); 1811254738Sbryanv 1812254738Sbryanv /* Clear events. */ 1813254738Sbryanv event = sc->vmx_ds->event; 1814254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1815254738Sbryanv 1816263259Sbryanv if (event & VMXNET3_EVENT_LINK) { 1817254738Sbryanv vmxnet3_link_status(sc); 1818263259Sbryanv if (sc->vmx_link_active != 0) 1819263259Sbryanv vmxnet3_tx_start_all(sc); 1820263259Sbryanv } 1821254738Sbryanv 1822254738Sbryanv if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1823254738Sbryanv reset = 1; 1824254738Sbryanv vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1825254738Sbryanv ts = sc->vmx_txq[0].vxtxq_ts; 1826254738Sbryanv if (ts->stopped != 0) 1827254738Sbryanv device_printf(dev, "Tx queue error %#x\n", ts->error); 1828254738Sbryanv rs = sc->vmx_rxq[0].vxrxq_rs; 1829254738Sbryanv if (rs->stopped != 0) 1830254738Sbryanv device_printf(dev, "Rx queue error %#x\n", rs->error); 1831254738Sbryanv device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1832254738Sbryanv } 1833254738Sbryanv 1834254738Sbryanv if (event & VMXNET3_EVENT_DIC) 1835254738Sbryanv device_printf(dev, "device implementation change event\n"); 1836254738Sbryanv if (event & VMXNET3_EVENT_DEBUG) 1837254738Sbryanv device_printf(dev, "debug event\n"); 1838254738Sbryanv 1839254738Sbryanv if (reset != 0) { 1840254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1841254738Sbryanv vmxnet3_init_locked(sc); 1842254738Sbryanv } 1843254738Sbryanv 1844254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 1845254738Sbryanv} 1846254738Sbryanv 1847254738Sbryanvstatic void 1848254738Sbryanvvmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1849254738Sbryanv{ 1850254738Sbryanv struct vmxnet3_softc *sc; 1851254738Sbryanv struct ifnet *ifp; 1852254738Sbryanv struct vmxnet3_txring *txr; 1853254738Sbryanv struct vmxnet3_comp_ring *txc; 1854254738Sbryanv struct vmxnet3_txcompdesc *txcd; 1855254738Sbryanv struct vmxnet3_txbuf *txb; 1856263259Sbryanv struct mbuf *m; 1857254738Sbryanv u_int sop; 1858254738Sbryanv 1859254738Sbryanv sc = txq->vxtxq_sc; 1860254738Sbryanv ifp = sc->vmx_ifp; 1861254738Sbryanv txr = &txq->vxtxq_cmd_ring; 1862254738Sbryanv txc = &txq->vxtxq_comp_ring; 1863254738Sbryanv 1864344272Svmaffione#ifdef DEV_NETMAP 1865344272Svmaffione if (netmap_tx_irq(sc->vmx_ifp, txq - sc->vmx_txq) != NM_IRQ_PASS) 1866344272Svmaffione return; 1867344272Svmaffione#endif 1868344272Svmaffione 1869254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 1870254738Sbryanv 1871254738Sbryanv for (;;) { 1872254738Sbryanv txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1873254738Sbryanv if (txcd->gen != txc->vxcr_gen) 1874254738Sbryanv break; 1875254950Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1876254738Sbryanv 1877254738Sbryanv if (++txc->vxcr_next == txc->vxcr_ndesc) { 1878254738Sbryanv txc->vxcr_next = 0; 1879254738Sbryanv txc->vxcr_gen ^= 1; 1880254738Sbryanv } 1881254738Sbryanv 1882254738Sbryanv sop = txr->vxtxr_next; 1883254738Sbryanv txb = &txr->vxtxr_txbuf[sop]; 1884254738Sbryanv 1885263259Sbryanv if ((m = txb->vtxb_m) != NULL) { 1886254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1887254738Sbryanv BUS_DMASYNC_POSTWRITE); 1888254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1889254738Sbryanv 1890263259Sbryanv txq->vxtxq_stats.vmtxs_opackets++; 1891263259Sbryanv txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1892263259Sbryanv if (m->m_flags & M_MCAST) 1893263259Sbryanv txq->vxtxq_stats.vmtxs_omcasts++; 1894263259Sbryanv 1895263259Sbryanv m_freem(m); 1896254738Sbryanv txb->vtxb_m = NULL; 1897254738Sbryanv } 1898254738Sbryanv 1899254738Sbryanv txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1900254738Sbryanv } 1901254738Sbryanv 1902254738Sbryanv if (txr->vxtxr_head == txr->vxtxr_next) 1903254738Sbryanv txq->vxtxq_watchdog = 0; 1904254738Sbryanv} 1905254738Sbryanv 1906254738Sbryanvstatic int 1907254738Sbryanvvmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1908254738Sbryanv{ 1909254738Sbryanv struct ifnet *ifp; 1910254738Sbryanv struct mbuf *m; 1911254738Sbryanv struct vmxnet3_rxdesc *rxd; 1912254738Sbryanv struct vmxnet3_rxbuf *rxb; 1913254738Sbryanv bus_dma_tag_t tag; 1914254738Sbryanv bus_dmamap_t dmap; 1915254738Sbryanv bus_dma_segment_t segs[1]; 1916254738Sbryanv int idx, clsize, btype, flags, nsegs, error; 1917254738Sbryanv 1918254738Sbryanv ifp = sc->vmx_ifp; 1919254738Sbryanv tag = rxr->vxrxr_rxtag; 1920254738Sbryanv dmap = rxr->vxrxr_spare_dmap; 1921254738Sbryanv idx = rxr->vxrxr_fill; 1922254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 1923254738Sbryanv rxb = &rxr->vxrxr_rxbuf[idx]; 1924254738Sbryanv 1925254738Sbryanv#ifdef VMXNET3_FAILPOINTS 1926254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1927254738Sbryanv if (rxr->vxrxr_rid != 0) 1928254738Sbryanv KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1929254738Sbryanv#endif 1930254738Sbryanv 1931254738Sbryanv if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1932254738Sbryanv flags = M_PKTHDR; 1933254738Sbryanv clsize = MCLBYTES; 1934254738Sbryanv btype = VMXNET3_BTYPE_HEAD; 1935254738Sbryanv } else { 1936254738Sbryanv#if __FreeBSD_version < 902001 1937254738Sbryanv /* 1938254738Sbryanv * These mbufs will never be used for the start of a frame. 1939254738Sbryanv * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1940254738Sbryanv * required the mbuf to always be a packet header. Avoid 1941254738Sbryanv * unnecessary mbuf initialization in newer versions where 1942254738Sbryanv * that is not the case. 1943254738Sbryanv */ 1944254738Sbryanv flags = M_PKTHDR; 1945254738Sbryanv#else 1946254738Sbryanv flags = 0; 1947254738Sbryanv#endif 1948254738Sbryanv clsize = MJUMPAGESIZE; 1949254738Sbryanv btype = VMXNET3_BTYPE_BODY; 1950254738Sbryanv } 1951254738Sbryanv 1952254738Sbryanv m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1953254738Sbryanv if (m == NULL) { 1954254738Sbryanv sc->vmx_stats.vmst_mgetcl_failed++; 1955254738Sbryanv return (ENOBUFS); 1956254738Sbryanv } 1957254738Sbryanv 1958254738Sbryanv if (btype == VMXNET3_BTYPE_HEAD) { 1959254738Sbryanv m->m_len = m->m_pkthdr.len = clsize; 1960254738Sbryanv m_adj(m, ETHER_ALIGN); 1961254738Sbryanv } else 1962254738Sbryanv m->m_len = clsize; 1963254738Sbryanv 1964254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1965254738Sbryanv BUS_DMA_NOWAIT); 1966254738Sbryanv if (error) { 1967254738Sbryanv m_freem(m); 1968254950Sbryanv sc->vmx_stats.vmst_mbuf_load_failed++; 1969254738Sbryanv return (error); 1970254738Sbryanv } 1971254738Sbryanv KASSERT(nsegs == 1, 1972254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1973254738Sbryanv#if __FreeBSD_version < 902001 1974254738Sbryanv if (btype == VMXNET3_BTYPE_BODY) 1975254738Sbryanv m->m_flags &= ~M_PKTHDR; 1976254738Sbryanv#endif 1977254738Sbryanv 1978254738Sbryanv if (rxb->vrxb_m != NULL) { 1979254738Sbryanv bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1980254738Sbryanv bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1981254738Sbryanv } 1982254738Sbryanv 1983254738Sbryanv rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1984254738Sbryanv rxb->vrxb_dmamap = dmap; 1985254738Sbryanv rxb->vrxb_m = m; 1986254738Sbryanv 1987254738Sbryanv rxd->addr = segs[0].ds_addr; 1988254738Sbryanv rxd->len = segs[0].ds_len; 1989254738Sbryanv rxd->btype = btype; 1990254738Sbryanv rxd->gen = rxr->vxrxr_gen; 1991254738Sbryanv 1992254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 1993254738Sbryanv return (0); 1994254738Sbryanv} 1995254738Sbryanv 1996254738Sbryanvstatic void 1997254738Sbryanvvmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1998254738Sbryanv struct vmxnet3_rxring *rxr, int idx) 1999254738Sbryanv{ 2000254738Sbryanv struct vmxnet3_rxdesc *rxd; 2001254738Sbryanv 2002254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 2003254738Sbryanv rxd->gen = rxr->vxrxr_gen; 2004254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 2005254738Sbryanv} 2006254738Sbryanv 2007254738Sbryanvstatic void 2008254738Sbryanvvmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 2009254738Sbryanv{ 2010254738Sbryanv struct vmxnet3_softc *sc; 2011254738Sbryanv struct vmxnet3_rxring *rxr; 2012254738Sbryanv struct vmxnet3_comp_ring *rxc; 2013254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 2014254738Sbryanv int idx, eof; 2015254738Sbryanv 2016254738Sbryanv sc = rxq->vxrxq_sc; 2017254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2018254738Sbryanv 2019254738Sbryanv do { 2020254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2021254738Sbryanv if (rxcd->gen != rxc->vxcr_gen) 2022254738Sbryanv break; /* Not expected. */ 2023254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2024254738Sbryanv 2025254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2026254738Sbryanv rxc->vxcr_next = 0; 2027254738Sbryanv rxc->vxcr_gen ^= 1; 2028254738Sbryanv } 2029254738Sbryanv 2030254738Sbryanv idx = rxcd->rxd_idx; 2031254738Sbryanv eof = rxcd->eop; 2032254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2033254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2034254738Sbryanv else 2035254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2036254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2037254738Sbryanv } while (!eof); 2038254738Sbryanv} 2039254738Sbryanv 2040254738Sbryanvstatic void 2041254738Sbryanvvmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2042254738Sbryanv{ 2043254738Sbryanv 2044254738Sbryanv if (rxcd->ipv4) { 2045254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2046254738Sbryanv if (rxcd->ipcsum_ok) 2047254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2048254738Sbryanv } 2049254738Sbryanv 2050254738Sbryanv if (!rxcd->fragment) { 2051254738Sbryanv if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2052254738Sbryanv m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2053254738Sbryanv CSUM_PSEUDO_HDR; 2054254738Sbryanv m->m_pkthdr.csum_data = 0xFFFF; 2055254738Sbryanv } 2056254738Sbryanv } 2057254738Sbryanv} 2058254738Sbryanv 2059254738Sbryanvstatic void 2060254738Sbryanvvmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2061254738Sbryanv struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2062254738Sbryanv{ 2063254738Sbryanv struct vmxnet3_softc *sc; 2064254738Sbryanv struct ifnet *ifp; 2065254738Sbryanv 2066254738Sbryanv sc = rxq->vxrxq_sc; 2067254738Sbryanv ifp = sc->vmx_ifp; 2068254738Sbryanv 2069254738Sbryanv if (rxcd->error) { 2070263259Sbryanv rxq->vxrxq_stats.vmrxs_ierrors++; 2071254738Sbryanv m_freem(m); 2072254738Sbryanv return; 2073254738Sbryanv } 2074254738Sbryanv 2075263259Sbryanv#ifdef notyet 2076263259Sbryanv switch (rxcd->rss_type) { 2077263259Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV4: 2078263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2079263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2080263259Sbryanv break; 2081263259Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2082263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2083263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2084263259Sbryanv break; 2085263259Sbryanv case VMXNET3_RCD_RSS_TYPE_IPV6: 2086263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2087263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2088263259Sbryanv break; 2089263259Sbryanv case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2090263259Sbryanv m->m_pkthdr.flowid = rxcd->rss_hash; 2091263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2092263259Sbryanv break; 2093263259Sbryanv default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2094263259Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2095263259Sbryanv M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2096263259Sbryanv break; 2097263259Sbryanv } 2098263259Sbryanv#else 2099263259Sbryanv m->m_pkthdr.flowid = rxq->vxrxq_id; 2100275358Shselasky M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2101263259Sbryanv#endif 2102263259Sbryanv 2103254738Sbryanv if (!rxcd->no_csum) 2104254738Sbryanv vmxnet3_rx_csum(rxcd, m); 2105254738Sbryanv if (rxcd->vlan) { 2106254738Sbryanv m->m_flags |= M_VLANTAG; 2107254738Sbryanv m->m_pkthdr.ether_vtag = rxcd->vtag; 2108254738Sbryanv } 2109254738Sbryanv 2110263259Sbryanv rxq->vxrxq_stats.vmrxs_ipackets++; 2111263259Sbryanv rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2112263259Sbryanv 2113254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2114254738Sbryanv (*ifp->if_input)(ifp, m); 2115254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2116254738Sbryanv} 2117254738Sbryanv 2118254738Sbryanvstatic void 2119254738Sbryanvvmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2120254738Sbryanv{ 2121254738Sbryanv struct vmxnet3_softc *sc; 2122254738Sbryanv struct ifnet *ifp; 2123254738Sbryanv struct vmxnet3_rxring *rxr; 2124254738Sbryanv struct vmxnet3_comp_ring *rxc; 2125254738Sbryanv struct vmxnet3_rxdesc *rxd; 2126254738Sbryanv struct vmxnet3_rxcompdesc *rxcd; 2127254738Sbryanv struct mbuf *m, *m_head, *m_tail; 2128254738Sbryanv int idx, length; 2129254738Sbryanv 2130254738Sbryanv sc = rxq->vxrxq_sc; 2131254738Sbryanv ifp = sc->vmx_ifp; 2132254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2133254738Sbryanv 2134344272Svmaffione#ifdef DEV_NETMAP 2135344272Svmaffione { 2136344272Svmaffione int dummy; 2137344272Svmaffione if (netmap_rx_irq(ifp, rxq - sc->vmx_rxq, &dummy) != 2138344272Svmaffione NM_IRQ_PASS) 2139344272Svmaffione return; 2140344272Svmaffione } 2141344272Svmaffione#endif 2142344272Svmaffione 2143254738Sbryanv VMXNET3_RXQ_LOCK_ASSERT(rxq); 2144254738Sbryanv 2145254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2146254738Sbryanv return; 2147254738Sbryanv 2148267662Sbryanv m_head = rxq->vxrxq_mhead; 2149267662Sbryanv rxq->vxrxq_mhead = NULL; 2150267662Sbryanv m_tail = rxq->vxrxq_mtail; 2151267662Sbryanv rxq->vxrxq_mtail = NULL; 2152267662Sbryanv MPASS(m_head == NULL || m_tail != NULL); 2153267662Sbryanv 2154254738Sbryanv for (;;) { 2155254738Sbryanv rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2156267662Sbryanv if (rxcd->gen != rxc->vxcr_gen) { 2157267662Sbryanv rxq->vxrxq_mhead = m_head; 2158267662Sbryanv rxq->vxrxq_mtail = m_tail; 2159254738Sbryanv break; 2160267662Sbryanv } 2161254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2162254738Sbryanv 2163254738Sbryanv if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2164254738Sbryanv rxc->vxcr_next = 0; 2165254738Sbryanv rxc->vxcr_gen ^= 1; 2166254738Sbryanv } 2167254738Sbryanv 2168254738Sbryanv idx = rxcd->rxd_idx; 2169254738Sbryanv length = rxcd->len; 2170254738Sbryanv if (rxcd->qid < sc->vmx_nrxqueues) 2171254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[0]; 2172254738Sbryanv else 2173254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[1]; 2174254738Sbryanv rxd = &rxr->vxrxr_rxd[idx]; 2175254738Sbryanv 2176254738Sbryanv m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2177254738Sbryanv KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2178254738Sbryanv __func__, rxcd->qid, idx)); 2179254738Sbryanv 2180254738Sbryanv /* 2181254738Sbryanv * The host may skip descriptors. We detect this when this 2182254738Sbryanv * descriptor does not match the previous fill index. Catch 2183254738Sbryanv * up with the host now. 2184254738Sbryanv */ 2185254738Sbryanv if (__predict_false(rxr->vxrxr_fill != idx)) { 2186254738Sbryanv while (rxr->vxrxr_fill != idx) { 2187254738Sbryanv rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2188254738Sbryanv rxr->vxrxr_gen; 2189254738Sbryanv vmxnet3_rxr_increment_fill(rxr); 2190254738Sbryanv } 2191254738Sbryanv } 2192254738Sbryanv 2193254738Sbryanv if (rxcd->sop) { 2194254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2195254738Sbryanv ("%s: start of frame w/o head buffer", __func__)); 2196254738Sbryanv KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2197254738Sbryanv ("%s: start of frame not in ring 0", __func__)); 2198254738Sbryanv KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2199254738Sbryanv ("%s: start of frame at unexcepted index %d (%d)", 2200254738Sbryanv __func__, idx, sc->vmx_rx_max_chain)); 2201254738Sbryanv KASSERT(m_head == NULL, 2202254738Sbryanv ("%s: duplicate start of frame?", __func__)); 2203254738Sbryanv 2204254738Sbryanv if (length == 0) { 2205254738Sbryanv /* Just ignore this descriptor. */ 2206254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2207254738Sbryanv goto nextp; 2208254738Sbryanv } 2209254738Sbryanv 2210254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2211263259Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2212254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2213254738Sbryanv if (!rxcd->eop) 2214254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2215254738Sbryanv goto nextp; 2216254738Sbryanv } 2217254738Sbryanv 2218254738Sbryanv m->m_pkthdr.rcvif = ifp; 2219254738Sbryanv m->m_pkthdr.len = m->m_len = length; 2220254738Sbryanv m->m_pkthdr.csum_flags = 0; 2221254738Sbryanv m_head = m_tail = m; 2222254738Sbryanv 2223254738Sbryanv } else { 2224254738Sbryanv KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2225254738Sbryanv ("%s: non start of frame w/o body buffer", __func__)); 2226320099Savg 2227320099Savg if (m_head == NULL && m_tail == NULL) { 2228320099Savg /* 2229320099Savg * This is a continuation of a packet that we 2230320099Savg * started to drop, but could not drop entirely 2231320099Savg * because this segment was still owned by the 2232320099Savg * host. So, drop the remainder now. 2233320099Savg */ 2234320099Savg vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2235320099Savg if (!rxcd->eop) 2236320099Savg vmxnet3_rxq_discard_chain(rxq); 2237320099Savg goto nextp; 2238320099Savg } 2239320099Savg 2240254738Sbryanv KASSERT(m_head != NULL, 2241254738Sbryanv ("%s: frame not started?", __func__)); 2242254738Sbryanv 2243254738Sbryanv if (vmxnet3_newbuf(sc, rxr) != 0) { 2244263259Sbryanv rxq->vxrxq_stats.vmrxs_iqdrops++; 2245254738Sbryanv vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2246254738Sbryanv if (!rxcd->eop) 2247254738Sbryanv vmxnet3_rxq_discard_chain(rxq); 2248254738Sbryanv m_freem(m_head); 2249254738Sbryanv m_head = m_tail = NULL; 2250254738Sbryanv goto nextp; 2251254738Sbryanv } 2252254738Sbryanv 2253254738Sbryanv m->m_len = length; 2254254738Sbryanv m_head->m_pkthdr.len += length; 2255254738Sbryanv m_tail->m_next = m; 2256254738Sbryanv m_tail = m; 2257254738Sbryanv } 2258254738Sbryanv 2259254738Sbryanv if (rxcd->eop) { 2260254738Sbryanv vmxnet3_rxq_input(rxq, rxcd, m_head); 2261254738Sbryanv m_head = m_tail = NULL; 2262254738Sbryanv 2263254738Sbryanv /* Must recheck after dropping the Rx lock. */ 2264254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2265254738Sbryanv break; 2266254738Sbryanv } 2267254738Sbryanv 2268254738Sbryanvnextp: 2269254738Sbryanv if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2270254738Sbryanv int qid = rxcd->qid; 2271254738Sbryanv bus_size_t r; 2272254738Sbryanv 2273254738Sbryanv idx = (idx + 1) % rxr->vxrxr_ndesc; 2274254738Sbryanv if (qid >= sc->vmx_nrxqueues) { 2275254738Sbryanv qid -= sc->vmx_nrxqueues; 2276254738Sbryanv r = VMXNET3_BAR0_RXH2(qid); 2277254738Sbryanv } else 2278254738Sbryanv r = VMXNET3_BAR0_RXH1(qid); 2279254738Sbryanv vmxnet3_write_bar0(sc, r, idx); 2280254738Sbryanv } 2281254738Sbryanv } 2282254738Sbryanv} 2283254738Sbryanv 2284254738Sbryanvstatic void 2285254738Sbryanvvmxnet3_legacy_intr(void *xsc) 2286254738Sbryanv{ 2287254738Sbryanv struct vmxnet3_softc *sc; 2288254738Sbryanv struct vmxnet3_rxqueue *rxq; 2289254738Sbryanv struct vmxnet3_txqueue *txq; 2290254738Sbryanv 2291254738Sbryanv sc = xsc; 2292254738Sbryanv rxq = &sc->vmx_rxq[0]; 2293254738Sbryanv txq = &sc->vmx_txq[0]; 2294254738Sbryanv 2295254738Sbryanv if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2296254738Sbryanv if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2297254738Sbryanv return; 2298254738Sbryanv } 2299254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2300254738Sbryanv vmxnet3_disable_all_intrs(sc); 2301254738Sbryanv 2302254738Sbryanv if (sc->vmx_ds->event != 0) 2303254738Sbryanv vmxnet3_evintr(sc); 2304254738Sbryanv 2305254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2306254738Sbryanv vmxnet3_rxq_eof(rxq); 2307254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2308254738Sbryanv 2309254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2310254738Sbryanv vmxnet3_txq_eof(txq); 2311263259Sbryanv vmxnet3_txq_start(txq); 2312254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2313254738Sbryanv 2314254738Sbryanv vmxnet3_enable_all_intrs(sc); 2315254738Sbryanv} 2316254738Sbryanv 2317254738Sbryanvstatic void 2318254738Sbryanvvmxnet3_txq_intr(void *xtxq) 2319254738Sbryanv{ 2320254738Sbryanv struct vmxnet3_softc *sc; 2321254738Sbryanv struct vmxnet3_txqueue *txq; 2322254738Sbryanv 2323254738Sbryanv txq = xtxq; 2324254738Sbryanv sc = txq->vxtxq_sc; 2325254738Sbryanv 2326254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2327254738Sbryanv vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2328254738Sbryanv 2329254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2330254738Sbryanv vmxnet3_txq_eof(txq); 2331263259Sbryanv vmxnet3_txq_start(txq); 2332254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2333254738Sbryanv 2334254738Sbryanv vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2335254738Sbryanv} 2336254738Sbryanv 2337254738Sbryanvstatic void 2338254738Sbryanvvmxnet3_rxq_intr(void *xrxq) 2339254738Sbryanv{ 2340254738Sbryanv struct vmxnet3_softc *sc; 2341254738Sbryanv struct vmxnet3_rxqueue *rxq; 2342254738Sbryanv 2343254738Sbryanv rxq = xrxq; 2344254738Sbryanv sc = rxq->vxrxq_sc; 2345254738Sbryanv 2346254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2347254738Sbryanv vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2348254738Sbryanv 2349254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2350254738Sbryanv vmxnet3_rxq_eof(rxq); 2351254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2352254738Sbryanv 2353254738Sbryanv vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2354254738Sbryanv} 2355254738Sbryanv 2356254738Sbryanvstatic void 2357254738Sbryanvvmxnet3_event_intr(void *xsc) 2358254738Sbryanv{ 2359254738Sbryanv struct vmxnet3_softc *sc; 2360254738Sbryanv 2361254738Sbryanv sc = xsc; 2362254738Sbryanv 2363254738Sbryanv if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2364254738Sbryanv vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2365254738Sbryanv 2366254738Sbryanv if (sc->vmx_ds->event != 0) 2367254738Sbryanv vmxnet3_evintr(sc); 2368254738Sbryanv 2369254738Sbryanv vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2370254738Sbryanv} 2371254738Sbryanv 2372254738Sbryanvstatic void 2373254738Sbryanvvmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2374254738Sbryanv{ 2375254738Sbryanv struct vmxnet3_txring *txr; 2376254738Sbryanv struct vmxnet3_txbuf *txb; 2377254738Sbryanv int i; 2378254738Sbryanv 2379254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2380254738Sbryanv 2381254738Sbryanv for (i = 0; i < txr->vxtxr_ndesc; i++) { 2382254738Sbryanv txb = &txr->vxtxr_txbuf[i]; 2383254738Sbryanv 2384254738Sbryanv if (txb->vtxb_m == NULL) 2385254738Sbryanv continue; 2386254738Sbryanv 2387254738Sbryanv bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2388254738Sbryanv BUS_DMASYNC_POSTWRITE); 2389254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2390254738Sbryanv m_freem(txb->vtxb_m); 2391254738Sbryanv txb->vtxb_m = NULL; 2392254738Sbryanv } 2393254738Sbryanv} 2394254738Sbryanv 2395254738Sbryanvstatic void 2396254738Sbryanvvmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2397254738Sbryanv{ 2398254738Sbryanv struct vmxnet3_rxring *rxr; 2399254738Sbryanv struct vmxnet3_rxbuf *rxb; 2400254738Sbryanv int i, j; 2401254738Sbryanv 2402267662Sbryanv if (rxq->vxrxq_mhead != NULL) { 2403267662Sbryanv m_freem(rxq->vxrxq_mhead); 2404267662Sbryanv rxq->vxrxq_mhead = NULL; 2405267662Sbryanv rxq->vxrxq_mtail = NULL; 2406267662Sbryanv } 2407267662Sbryanv 2408254738Sbryanv for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2409254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2410254738Sbryanv 2411254738Sbryanv for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2412254738Sbryanv rxb = &rxr->vxrxr_rxbuf[j]; 2413254738Sbryanv 2414254738Sbryanv if (rxb->vrxb_m == NULL) 2415254738Sbryanv continue; 2416263259Sbryanv 2417254738Sbryanv bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2418254738Sbryanv BUS_DMASYNC_POSTREAD); 2419254738Sbryanv bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2420254738Sbryanv m_freem(rxb->vrxb_m); 2421254738Sbryanv rxb->vrxb_m = NULL; 2422254738Sbryanv } 2423254738Sbryanv } 2424254738Sbryanv} 2425254738Sbryanv 2426254738Sbryanvstatic void 2427254738Sbryanvvmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2428254738Sbryanv{ 2429254738Sbryanv struct vmxnet3_rxqueue *rxq; 2430254738Sbryanv struct vmxnet3_txqueue *txq; 2431254738Sbryanv int i; 2432254738Sbryanv 2433344272Svmaffione#ifdef DEV_NETMAP 2434344272Svmaffione netmap_disable_all_rings(sc->vmx_ifp); 2435344272Svmaffione#endif 2436344272Svmaffione 2437254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 2438254738Sbryanv rxq = &sc->vmx_rxq[i]; 2439254738Sbryanv VMXNET3_RXQ_LOCK(rxq); 2440254738Sbryanv VMXNET3_RXQ_UNLOCK(rxq); 2441254738Sbryanv } 2442254738Sbryanv 2443254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 2444254738Sbryanv txq = &sc->vmx_txq[i]; 2445254738Sbryanv VMXNET3_TXQ_LOCK(txq); 2446254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 2447254738Sbryanv } 2448254738Sbryanv} 2449254738Sbryanv 2450254738Sbryanvstatic void 2451254738Sbryanvvmxnet3_stop(struct vmxnet3_softc *sc) 2452254738Sbryanv{ 2453254738Sbryanv struct ifnet *ifp; 2454254738Sbryanv int q; 2455254738Sbryanv 2456254738Sbryanv ifp = sc->vmx_ifp; 2457254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 2458254738Sbryanv 2459254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2460254738Sbryanv sc->vmx_link_active = 0; 2461254738Sbryanv callout_stop(&sc->vmx_tick); 2462254738Sbryanv 2463254738Sbryanv /* Disable interrupts. */ 2464254738Sbryanv vmxnet3_disable_all_intrs(sc); 2465254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2466254738Sbryanv 2467254738Sbryanv vmxnet3_stop_rendezvous(sc); 2468254738Sbryanv 2469254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2470254738Sbryanv vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2471254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) 2472254738Sbryanv vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2473254738Sbryanv 2474254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2475254738Sbryanv} 2476254738Sbryanv 2477254738Sbryanvstatic void 2478254738Sbryanvvmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2479254738Sbryanv{ 2480254738Sbryanv struct vmxnet3_txring *txr; 2481254738Sbryanv struct vmxnet3_comp_ring *txc; 2482254738Sbryanv 2483254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2484254738Sbryanv txr->vxtxr_head = 0; 2485254738Sbryanv txr->vxtxr_next = 0; 2486254738Sbryanv txr->vxtxr_gen = VMXNET3_INIT_GEN; 2487254738Sbryanv bzero(txr->vxtxr_txd, 2488254738Sbryanv txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2489254738Sbryanv 2490344272Svmaffione#ifdef DEV_NETMAP 2491344272Svmaffione vmxnet3_netmap_txq_init(sc, txq); 2492344272Svmaffione#endif 2493344272Svmaffione 2494254738Sbryanv txc = &txq->vxtxq_comp_ring; 2495254738Sbryanv txc->vxcr_next = 0; 2496254738Sbryanv txc->vxcr_gen = VMXNET3_INIT_GEN; 2497254738Sbryanv bzero(txc->vxcr_u.txcd, 2498254738Sbryanv txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2499254738Sbryanv} 2500254738Sbryanv 2501254738Sbryanvstatic int 2502254738Sbryanvvmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2503254738Sbryanv{ 2504254738Sbryanv struct ifnet *ifp; 2505254738Sbryanv struct vmxnet3_rxring *rxr; 2506254738Sbryanv struct vmxnet3_comp_ring *rxc; 2507254738Sbryanv int i, populate, idx, frame_size, error; 2508344272Svmaffione#ifdef DEV_NETMAP 2509344272Svmaffione struct netmap_adapter *na; 2510344272Svmaffione struct netmap_slot *slot; 2511344272Svmaffione#endif 2512254738Sbryanv 2513254738Sbryanv ifp = sc->vmx_ifp; 2514254950Sbryanv frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2515254950Sbryanv ifp->if_mtu; 2516254738Sbryanv 2517254738Sbryanv /* 2518254950Sbryanv * If the MTU causes us to exceed what a regular sized cluster can 2519254950Sbryanv * handle, we allocate a second MJUMPAGESIZE cluster after it in 2520254950Sbryanv * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2521254738Sbryanv * 2522254950Sbryanv * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2523254950Sbryanv * our life easier. We do not support changing the ring size after 2524254950Sbryanv * the attach. 2525254738Sbryanv */ 2526254950Sbryanv if (frame_size <= MCLBYTES) 2527254738Sbryanv sc->vmx_rx_max_chain = 1; 2528254738Sbryanv else 2529254738Sbryanv sc->vmx_rx_max_chain = 2; 2530254738Sbryanv 2531254738Sbryanv /* 2532254738Sbryanv * Only populate ring 1 if the configuration will take advantage 2533254738Sbryanv * of it. That is either when LRO is enabled or the frame size 2534254738Sbryanv * exceeds what ring 0 can contain. 2535254738Sbryanv */ 2536254738Sbryanv if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2537254738Sbryanv frame_size <= MCLBYTES + MJUMPAGESIZE) 2538254738Sbryanv populate = 1; 2539254738Sbryanv else 2540254738Sbryanv populate = VMXNET3_RXRINGS_PERQ; 2541254738Sbryanv 2542344272Svmaffione#ifdef DEV_NETMAP 2543344272Svmaffione na = NA(ifp); 2544344272Svmaffione slot = netmap_reset(na, NR_RX, rxq - sc->vmx_rxq, 0); 2545344272Svmaffione#endif 2546344272Svmaffione 2547254738Sbryanv for (i = 0; i < populate; i++) { 2548254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2549254738Sbryanv rxr->vxrxr_fill = 0; 2550254738Sbryanv rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2551254738Sbryanv bzero(rxr->vxrxr_rxd, 2552254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2553344272Svmaffione#ifdef DEV_NETMAP 2554344272Svmaffione if (slot != NULL) { 2555344272Svmaffione vmxnet3_netmap_rxq_init(sc, rxq, rxr, slot); 2556344272Svmaffione i = populate; 2557344272Svmaffione break; 2558344272Svmaffione } 2559344272Svmaffione#endif 2560254738Sbryanv 2561254738Sbryanv for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2562254738Sbryanv error = vmxnet3_newbuf(sc, rxr); 2563254738Sbryanv if (error) 2564254738Sbryanv return (error); 2565254738Sbryanv } 2566254738Sbryanv } 2567254738Sbryanv 2568254738Sbryanv for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2569254738Sbryanv rxr = &rxq->vxrxq_cmd_ring[i]; 2570254738Sbryanv rxr->vxrxr_fill = 0; 2571254738Sbryanv rxr->vxrxr_gen = 0; 2572254738Sbryanv bzero(rxr->vxrxr_rxd, 2573254738Sbryanv rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2574254738Sbryanv } 2575254738Sbryanv 2576254738Sbryanv rxc = &rxq->vxrxq_comp_ring; 2577254738Sbryanv rxc->vxcr_next = 0; 2578254738Sbryanv rxc->vxcr_gen = VMXNET3_INIT_GEN; 2579254738Sbryanv bzero(rxc->vxcr_u.rxcd, 2580254738Sbryanv rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2581254738Sbryanv 2582254738Sbryanv return (0); 2583254738Sbryanv} 2584254738Sbryanv 2585254738Sbryanvstatic int 2586254738Sbryanvvmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2587254738Sbryanv{ 2588254738Sbryanv device_t dev; 2589254738Sbryanv int q, error; 2590254738Sbryanv 2591254738Sbryanv dev = sc->vmx_dev; 2592254738Sbryanv 2593254738Sbryanv for (q = 0; q < sc->vmx_ntxqueues; q++) 2594254738Sbryanv vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2595254738Sbryanv 2596254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2597254738Sbryanv error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2598254738Sbryanv if (error) { 2599254738Sbryanv device_printf(dev, "cannot populate Rx queue %d\n", q); 2600254738Sbryanv return (error); 2601254738Sbryanv } 2602254738Sbryanv } 2603254738Sbryanv 2604254738Sbryanv return (0); 2605254738Sbryanv} 2606254738Sbryanv 2607254738Sbryanvstatic int 2608254738Sbryanvvmxnet3_enable_device(struct vmxnet3_softc *sc) 2609254738Sbryanv{ 2610254738Sbryanv int q; 2611254738Sbryanv 2612254738Sbryanv if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2613254738Sbryanv device_printf(sc->vmx_dev, "device enable command failed!\n"); 2614254738Sbryanv return (1); 2615254738Sbryanv } 2616254738Sbryanv 2617254738Sbryanv /* Reset the Rx queue heads. */ 2618254738Sbryanv for (q = 0; q < sc->vmx_nrxqueues; q++) { 2619254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2620254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2621254738Sbryanv } 2622254738Sbryanv 2623254738Sbryanv return (0); 2624254738Sbryanv} 2625254738Sbryanv 2626254738Sbryanvstatic void 2627254738Sbryanvvmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2628254738Sbryanv{ 2629254738Sbryanv struct ifnet *ifp; 2630254738Sbryanv 2631254738Sbryanv ifp = sc->vmx_ifp; 2632254738Sbryanv 2633254738Sbryanv vmxnet3_set_rxfilter(sc); 2634254738Sbryanv 2635254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2636254738Sbryanv bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2637254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2638254738Sbryanv else 2639254738Sbryanv bzero(sc->vmx_ds->vlan_filter, 2640254738Sbryanv sizeof(sc->vmx_ds->vlan_filter)); 2641254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2642254738Sbryanv} 2643254738Sbryanv 2644254738Sbryanvstatic int 2645254738Sbryanvvmxnet3_reinit(struct vmxnet3_softc *sc) 2646254738Sbryanv{ 2647254738Sbryanv 2648254738Sbryanv vmxnet3_reinit_interface(sc); 2649254738Sbryanv vmxnet3_reinit_shared_data(sc); 2650254738Sbryanv 2651254738Sbryanv if (vmxnet3_reinit_queues(sc) != 0) 2652254738Sbryanv return (ENXIO); 2653254738Sbryanv 2654254738Sbryanv if (vmxnet3_enable_device(sc) != 0) 2655254738Sbryanv return (ENXIO); 2656254738Sbryanv 2657254738Sbryanv vmxnet3_reinit_rxfilters(sc); 2658254738Sbryanv 2659254738Sbryanv return (0); 2660254738Sbryanv} 2661254738Sbryanv 2662254738Sbryanvstatic void 2663254738Sbryanvvmxnet3_init_locked(struct vmxnet3_softc *sc) 2664254738Sbryanv{ 2665254738Sbryanv struct ifnet *ifp; 2666254738Sbryanv 2667254738Sbryanv ifp = sc->vmx_ifp; 2668254738Sbryanv 2669254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2670254738Sbryanv return; 2671254738Sbryanv 2672254738Sbryanv vmxnet3_stop(sc); 2673254738Sbryanv 2674254738Sbryanv if (vmxnet3_reinit(sc) != 0) { 2675254738Sbryanv vmxnet3_stop(sc); 2676254738Sbryanv return; 2677254738Sbryanv } 2678254738Sbryanv 2679254738Sbryanv ifp->if_drv_flags |= IFF_DRV_RUNNING; 2680254738Sbryanv vmxnet3_link_status(sc); 2681254738Sbryanv 2682344272Svmaffione#ifdef DEV_NETMAP 2683344272Svmaffione netmap_enable_all_rings(ifp); 2684344272Svmaffione#endif 2685344272Svmaffione 2686254738Sbryanv vmxnet3_enable_all_intrs(sc); 2687254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2688254738Sbryanv} 2689254738Sbryanv 2690254738Sbryanvstatic void 2691254738Sbryanvvmxnet3_init(void *xsc) 2692254738Sbryanv{ 2693254738Sbryanv struct vmxnet3_softc *sc; 2694254738Sbryanv 2695254738Sbryanv sc = xsc; 2696254738Sbryanv 2697254738Sbryanv VMXNET3_CORE_LOCK(sc); 2698254738Sbryanv vmxnet3_init_locked(sc); 2699254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 2700254738Sbryanv} 2701254738Sbryanv 2702254738Sbryanv/* 2703254738Sbryanv * BMV: Much of this can go away once we finally have offsets in 2704254738Sbryanv * the mbuf packet header. Bug andre@. 2705254738Sbryanv */ 2706254738Sbryanvstatic int 2707263259Sbryanvvmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2708263259Sbryanv int *etype, int *proto, int *start) 2709254738Sbryanv{ 2710254738Sbryanv struct ether_vlan_header *evh; 2711254738Sbryanv int offset; 2712267253Sbryanv#if defined(INET) 2713267632Shselasky struct ip *ip = NULL; 2714267632Shselasky struct ip iphdr; 2715267253Sbryanv#endif 2716267253Sbryanv#if defined(INET6) 2717267632Shselasky struct ip6_hdr *ip6 = NULL; 2718267632Shselasky struct ip6_hdr ip6hdr; 2719267253Sbryanv#endif 2720254738Sbryanv 2721254738Sbryanv evh = mtod(m, struct ether_vlan_header *); 2722254738Sbryanv if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2723254738Sbryanv /* BMV: We should handle nested VLAN tags too. */ 2724254738Sbryanv *etype = ntohs(evh->evl_proto); 2725254738Sbryanv offset = sizeof(struct ether_vlan_header); 2726254738Sbryanv } else { 2727254738Sbryanv *etype = ntohs(evh->evl_encap_proto); 2728254738Sbryanv offset = sizeof(struct ether_header); 2729254738Sbryanv } 2730254738Sbryanv 2731254738Sbryanv switch (*etype) { 2732254738Sbryanv#if defined(INET) 2733267253Sbryanv case ETHERTYPE_IP: 2734254738Sbryanv if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2735254738Sbryanv m_copydata(m, offset, sizeof(struct ip), 2736254738Sbryanv (caddr_t) &iphdr); 2737254738Sbryanv ip = &iphdr; 2738254738Sbryanv } else 2739263259Sbryanv ip = mtodo(m, offset); 2740254738Sbryanv *proto = ip->ip_p; 2741254738Sbryanv *start = offset + (ip->ip_hl << 2); 2742254738Sbryanv break; 2743254738Sbryanv#endif 2744254738Sbryanv#if defined(INET6) 2745254738Sbryanv case ETHERTYPE_IPV6: 2746267253Sbryanv if (__predict_false(m->m_len < 2747267253Sbryanv offset + sizeof(struct ip6_hdr))) { 2748267253Sbryanv m_copydata(m, offset, sizeof(struct ip6_hdr), 2749267253Sbryanv (caddr_t) &ip6hdr); 2750267253Sbryanv ip6 = &ip6hdr; 2751267253Sbryanv } else 2752267253Sbryanv ip6 = mtodo(m, offset); 2753254738Sbryanv *proto = -1; 2754254738Sbryanv *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2755254738Sbryanv /* Assert the network stack sent us a valid packet. */ 2756254738Sbryanv KASSERT(*start > offset, 2757254738Sbryanv ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2758254738Sbryanv *start, offset, *proto)); 2759254738Sbryanv break; 2760254738Sbryanv#endif 2761254738Sbryanv default: 2762254738Sbryanv return (EINVAL); 2763254738Sbryanv } 2764254738Sbryanv 2765254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2766254738Sbryanv struct tcphdr *tcp, tcphdr; 2767267253Sbryanv uint16_t sum; 2768254738Sbryanv 2769254738Sbryanv if (__predict_false(*proto != IPPROTO_TCP)) { 2770254738Sbryanv /* Likely failed to correctly parse the mbuf. */ 2771254738Sbryanv return (EINVAL); 2772254738Sbryanv } 2773254738Sbryanv 2774263259Sbryanv txq->vxtxq_stats.vmtxs_tso++; 2775263259Sbryanv 2776267253Sbryanv switch (*etype) { 2777267253Sbryanv#if defined(INET) 2778267253Sbryanv case ETHERTYPE_IP: 2779267253Sbryanv sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2780267253Sbryanv htons(IPPROTO_TCP)); 2781267253Sbryanv break; 2782267253Sbryanv#endif 2783267253Sbryanv#if defined(INET6) 2784267253Sbryanv case ETHERTYPE_IPV6: 2785267253Sbryanv sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2786267253Sbryanv break; 2787267253Sbryanv#endif 2788267253Sbryanv default: 2789267253Sbryanv sum = 0; 2790267253Sbryanv break; 2791267253Sbryanv } 2792267253Sbryanv 2793267253Sbryanv if (m->m_len < *start + sizeof(struct tcphdr)) { 2794267253Sbryanv m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2795267253Sbryanv sizeof(uint16_t), (caddr_t) &sum); 2796267253Sbryanv m_copydata(m, *start, sizeof(struct tcphdr), 2797267253Sbryanv (caddr_t) &tcphdr); 2798267253Sbryanv tcp = &tcphdr; 2799267253Sbryanv } else { 2800267253Sbryanv tcp = mtodo(m, *start); 2801267253Sbryanv tcp->th_sum = sum; 2802267253Sbryanv } 2803267253Sbryanv 2804263259Sbryanv /* 2805263259Sbryanv * For TSO, the size of the protocol header is also 2806263259Sbryanv * included in the descriptor header size. 2807263259Sbryanv */ 2808254738Sbryanv *start += (tcp->th_off << 2); 2809263259Sbryanv } else 2810263259Sbryanv txq->vxtxq_stats.vmtxs_csum++; 2811254738Sbryanv 2812254738Sbryanv return (0); 2813254738Sbryanv} 2814254738Sbryanv 2815254738Sbryanvstatic int 2816254738Sbryanvvmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2817254738Sbryanv bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2818254738Sbryanv{ 2819254738Sbryanv struct vmxnet3_txring *txr; 2820254738Sbryanv struct mbuf *m; 2821254738Sbryanv bus_dma_tag_t tag; 2822263259Sbryanv int error; 2823254738Sbryanv 2824254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2825254738Sbryanv m = *m0; 2826254738Sbryanv tag = txr->vxtxr_txtag; 2827254738Sbryanv 2828254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2829254738Sbryanv if (error == 0 || error != EFBIG) 2830254738Sbryanv return (error); 2831254738Sbryanv 2832263259Sbryanv m = m_defrag(m, M_NOWAIT); 2833254738Sbryanv if (m != NULL) { 2834254738Sbryanv *m0 = m; 2835254738Sbryanv error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2836254738Sbryanv } else 2837254738Sbryanv error = ENOBUFS; 2838254738Sbryanv 2839254738Sbryanv if (error) { 2840254738Sbryanv m_freem(*m0); 2841254738Sbryanv *m0 = NULL; 2842263259Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2843254738Sbryanv } else 2844263259Sbryanv txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2845254738Sbryanv 2846254738Sbryanv return (error); 2847254738Sbryanv} 2848254738Sbryanv 2849254738Sbryanvstatic void 2850254738Sbryanvvmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2851254738Sbryanv{ 2852254738Sbryanv struct vmxnet3_txring *txr; 2853254738Sbryanv 2854254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2855254738Sbryanv bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2856254738Sbryanv} 2857254738Sbryanv 2858254738Sbryanvstatic int 2859254738Sbryanvvmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2860254738Sbryanv{ 2861254738Sbryanv struct vmxnet3_softc *sc; 2862254738Sbryanv struct vmxnet3_txring *txr; 2863254738Sbryanv struct vmxnet3_txdesc *txd, *sop; 2864254738Sbryanv struct mbuf *m; 2865254738Sbryanv bus_dmamap_t dmap; 2866254738Sbryanv bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2867254738Sbryanv int i, gen, nsegs, etype, proto, start, error; 2868254738Sbryanv 2869254738Sbryanv sc = txq->vxtxq_sc; 2870254738Sbryanv start = 0; 2871254738Sbryanv txd = NULL; 2872254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2873254738Sbryanv dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2874254738Sbryanv 2875254738Sbryanv error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2876254738Sbryanv if (error) 2877254738Sbryanv return (error); 2878254738Sbryanv 2879254738Sbryanv m = *m0; 2880254738Sbryanv M_ASSERTPKTHDR(m); 2881254738Sbryanv KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2882254738Sbryanv ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2883254738Sbryanv 2884254738Sbryanv if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2885263259Sbryanv txq->vxtxq_stats.vmtxs_full++; 2886254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2887254738Sbryanv return (ENOSPC); 2888254738Sbryanv } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2889263259Sbryanv error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2890254738Sbryanv if (error) { 2891263259Sbryanv txq->vxtxq_stats.vmtxs_offload_failed++; 2892254738Sbryanv vmxnet3_txq_unload_mbuf(txq, dmap); 2893254738Sbryanv m_freem(m); 2894254738Sbryanv *m0 = NULL; 2895254738Sbryanv return (error); 2896254738Sbryanv } 2897254738Sbryanv } 2898254738Sbryanv 2899267252Sbryanv txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2900254738Sbryanv sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2901254738Sbryanv gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2902254738Sbryanv 2903254738Sbryanv for (i = 0; i < nsegs; i++) { 2904254738Sbryanv txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2905254738Sbryanv 2906254738Sbryanv txd->addr = segs[i].ds_addr; 2907254738Sbryanv txd->len = segs[i].ds_len; 2908254738Sbryanv txd->gen = gen; 2909254738Sbryanv txd->dtype = 0; 2910254738Sbryanv txd->offload_mode = VMXNET3_OM_NONE; 2911254738Sbryanv txd->offload_pos = 0; 2912254738Sbryanv txd->hlen = 0; 2913254738Sbryanv txd->eop = 0; 2914254738Sbryanv txd->compreq = 0; 2915254738Sbryanv txd->vtag_mode = 0; 2916254738Sbryanv txd->vtag = 0; 2917254738Sbryanv 2918254738Sbryanv if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2919254738Sbryanv txr->vxtxr_head = 0; 2920254738Sbryanv txr->vxtxr_gen ^= 1; 2921254738Sbryanv } 2922254738Sbryanv gen = txr->vxtxr_gen; 2923254738Sbryanv } 2924254738Sbryanv txd->eop = 1; 2925254738Sbryanv txd->compreq = 1; 2926254738Sbryanv 2927254738Sbryanv if (m->m_flags & M_VLANTAG) { 2928254738Sbryanv sop->vtag_mode = 1; 2929254738Sbryanv sop->vtag = m->m_pkthdr.ether_vtag; 2930254738Sbryanv } 2931254738Sbryanv 2932254738Sbryanv if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2933254738Sbryanv sop->offload_mode = VMXNET3_OM_TSO; 2934254738Sbryanv sop->hlen = start; 2935254738Sbryanv sop->offload_pos = m->m_pkthdr.tso_segsz; 2936254738Sbryanv } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2937254738Sbryanv VMXNET3_CSUM_OFFLOAD_IPV6)) { 2938254738Sbryanv sop->offload_mode = VMXNET3_OM_CSUM; 2939254738Sbryanv sop->hlen = start; 2940254738Sbryanv sop->offload_pos = start + m->m_pkthdr.csum_data; 2941254738Sbryanv } 2942254738Sbryanv 2943254738Sbryanv /* Finally, change the ownership. */ 2944254738Sbryanv vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2945254738Sbryanv sop->gen ^= 1; 2946254738Sbryanv 2947267663Sbryanv txq->vxtxq_ts->npending += nsegs; 2948267663Sbryanv if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2949254738Sbryanv txq->vxtxq_ts->npending = 0; 2950254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2951254738Sbryanv txr->vxtxr_head); 2952254738Sbryanv } 2953254738Sbryanv 2954254738Sbryanv return (0); 2955254738Sbryanv} 2956254738Sbryanv 2957263259Sbryanv#ifdef VMXNET3_LEGACY_TX 2958263259Sbryanv 2959263259Sbryanvstatic void 2960254738Sbryanvvmxnet3_start_locked(struct ifnet *ifp) 2961254738Sbryanv{ 2962254738Sbryanv struct vmxnet3_softc *sc; 2963254738Sbryanv struct vmxnet3_txqueue *txq; 2964254738Sbryanv struct vmxnet3_txring *txr; 2965254738Sbryanv struct mbuf *m_head; 2966255055Sbryanv int tx, avail; 2967254738Sbryanv 2968254738Sbryanv sc = ifp->if_softc; 2969254738Sbryanv txq = &sc->vmx_txq[0]; 2970254738Sbryanv txr = &txq->vxtxq_cmd_ring; 2971254738Sbryanv tx = 0; 2972254738Sbryanv 2973254738Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 2974254738Sbryanv 2975254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2976254738Sbryanv sc->vmx_link_active == 0) 2977254738Sbryanv return; 2978254738Sbryanv 2979255055Sbryanv while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2980255055Sbryanv if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2981255055Sbryanv break; 2982255055Sbryanv 2983254738Sbryanv IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2984254738Sbryanv if (m_head == NULL) 2985254738Sbryanv break; 2986254738Sbryanv 2987255055Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 2988255055Sbryanv if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2989255055Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2990255055Sbryanv break; 2991255055Sbryanv } 2992255055Sbryanv 2993254738Sbryanv if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2994254738Sbryanv if (m_head != NULL) 2995254738Sbryanv IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2996254738Sbryanv break; 2997254738Sbryanv } 2998254738Sbryanv 2999254738Sbryanv tx++; 3000254738Sbryanv ETHER_BPF_MTAP(ifp, m_head); 3001254738Sbryanv } 3002254738Sbryanv 3003267661Sbryanv if (tx > 0) 3004254738Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3005254738Sbryanv} 3006254738Sbryanv 3007254738Sbryanvstatic void 3008254738Sbryanvvmxnet3_start(struct ifnet *ifp) 3009254738Sbryanv{ 3010254738Sbryanv struct vmxnet3_softc *sc; 3011254738Sbryanv struct vmxnet3_txqueue *txq; 3012254738Sbryanv 3013254738Sbryanv sc = ifp->if_softc; 3014254738Sbryanv txq = &sc->vmx_txq[0]; 3015254738Sbryanv 3016254738Sbryanv VMXNET3_TXQ_LOCK(txq); 3017254738Sbryanv vmxnet3_start_locked(ifp); 3018254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3019254738Sbryanv} 3020254738Sbryanv 3021263259Sbryanv#else /* !VMXNET3_LEGACY_TX */ 3022263259Sbryanv 3023263259Sbryanvstatic int 3024263259Sbryanvvmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 3025263259Sbryanv{ 3026263259Sbryanv struct vmxnet3_softc *sc; 3027263259Sbryanv struct vmxnet3_txring *txr; 3028263259Sbryanv struct buf_ring *br; 3029263259Sbryanv struct ifnet *ifp; 3030263259Sbryanv int tx, avail, error; 3031263259Sbryanv 3032263259Sbryanv sc = txq->vxtxq_sc; 3033263259Sbryanv br = txq->vxtxq_br; 3034263259Sbryanv ifp = sc->vmx_ifp; 3035263259Sbryanv txr = &txq->vxtxq_cmd_ring; 3036263259Sbryanv tx = 0; 3037263259Sbryanv error = 0; 3038263259Sbryanv 3039263259Sbryanv VMXNET3_TXQ_LOCK_ASSERT(txq); 3040263259Sbryanv 3041263259Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 3042263259Sbryanv sc->vmx_link_active == 0) { 3043263259Sbryanv if (m != NULL) 3044263259Sbryanv error = drbr_enqueue(ifp, br, m); 3045263259Sbryanv return (error); 3046263259Sbryanv } 3047263259Sbryanv 3048263259Sbryanv if (m != NULL) { 3049263259Sbryanv error = drbr_enqueue(ifp, br, m); 3050263259Sbryanv if (error) 3051263259Sbryanv return (error); 3052263259Sbryanv } 3053263259Sbryanv 3054263259Sbryanv while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 3055263259Sbryanv m = drbr_peek(ifp, br); 3056263259Sbryanv if (m == NULL) 3057263259Sbryanv break; 3058263259Sbryanv 3059263259Sbryanv /* Assume worse case if this mbuf is the head of a chain. */ 3060263259Sbryanv if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 3061263259Sbryanv drbr_putback(ifp, br, m); 3062263259Sbryanv break; 3063263259Sbryanv } 3064263259Sbryanv 3065267663Sbryanv if (vmxnet3_txq_encap(txq, &m) != 0) { 3066263259Sbryanv if (m != NULL) 3067263259Sbryanv drbr_putback(ifp, br, m); 3068263259Sbryanv else 3069263259Sbryanv drbr_advance(ifp, br); 3070263259Sbryanv break; 3071263259Sbryanv } 3072263259Sbryanv drbr_advance(ifp, br); 3073263259Sbryanv 3074263259Sbryanv tx++; 3075263259Sbryanv ETHER_BPF_MTAP(ifp, m); 3076263259Sbryanv } 3077263259Sbryanv 3078267661Sbryanv if (tx > 0) 3079263259Sbryanv txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 3080263259Sbryanv 3081267187Sluigi return (0); 3082263259Sbryanv} 3083263259Sbryanv 3084263259Sbryanvstatic int 3085263259Sbryanvvmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 3086263259Sbryanv{ 3087263259Sbryanv struct vmxnet3_softc *sc; 3088263259Sbryanv struct vmxnet3_txqueue *txq; 3089263259Sbryanv int i, ntxq, error; 3090263259Sbryanv 3091263259Sbryanv sc = ifp->if_softc; 3092263259Sbryanv ntxq = sc->vmx_ntxqueues; 3093263259Sbryanv 3094275358Shselasky /* check if flowid is set */ 3095275358Shselasky if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3096263259Sbryanv i = m->m_pkthdr.flowid % ntxq; 3097263259Sbryanv else 3098263259Sbryanv i = curcpu % ntxq; 3099263259Sbryanv 3100263259Sbryanv txq = &sc->vmx_txq[i]; 3101263259Sbryanv 3102263259Sbryanv if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3103263259Sbryanv error = vmxnet3_txq_mq_start_locked(txq, m); 3104263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3105263259Sbryanv } else { 3106263259Sbryanv error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3107263259Sbryanv taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3108263259Sbryanv } 3109263259Sbryanv 3110263259Sbryanv return (error); 3111263259Sbryanv} 3112263259Sbryanv 3113254738Sbryanvstatic void 3114263259Sbryanvvmxnet3_txq_tq_deferred(void *xtxq, int pending) 3115263259Sbryanv{ 3116263259Sbryanv struct vmxnet3_softc *sc; 3117263259Sbryanv struct vmxnet3_txqueue *txq; 3118263259Sbryanv 3119263259Sbryanv txq = xtxq; 3120263259Sbryanv sc = txq->vxtxq_sc; 3121263259Sbryanv 3122263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3123263259Sbryanv if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3124263259Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3125263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3126263259Sbryanv} 3127263259Sbryanv 3128263259Sbryanv#endif /* VMXNET3_LEGACY_TX */ 3129263259Sbryanv 3130263259Sbryanvstatic void 3131263259Sbryanvvmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3132263259Sbryanv{ 3133263259Sbryanv struct vmxnet3_softc *sc; 3134263259Sbryanv struct ifnet *ifp; 3135263259Sbryanv 3136263259Sbryanv sc = txq->vxtxq_sc; 3137263259Sbryanv ifp = sc->vmx_ifp; 3138263259Sbryanv 3139263259Sbryanv#ifdef VMXNET3_LEGACY_TX 3140263259Sbryanv if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3141263259Sbryanv vmxnet3_start_locked(ifp); 3142263259Sbryanv#else 3143263259Sbryanv if (!drbr_empty(ifp, txq->vxtxq_br)) 3144263259Sbryanv vmxnet3_txq_mq_start_locked(txq, NULL); 3145263259Sbryanv#endif 3146263259Sbryanv} 3147263259Sbryanv 3148263259Sbryanvstatic void 3149263259Sbryanvvmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3150263259Sbryanv{ 3151263259Sbryanv struct vmxnet3_txqueue *txq; 3152263259Sbryanv int i; 3153263259Sbryanv 3154263259Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3155263259Sbryanv 3156263259Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3157263259Sbryanv txq = &sc->vmx_txq[i]; 3158263259Sbryanv 3159263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3160263259Sbryanv vmxnet3_txq_start(txq); 3161263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3162263259Sbryanv } 3163263259Sbryanv} 3164263259Sbryanv 3165263259Sbryanvstatic void 3166254738Sbryanvvmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3167254738Sbryanv{ 3168254738Sbryanv struct ifnet *ifp; 3169254738Sbryanv int idx, bit; 3170254738Sbryanv 3171254738Sbryanv ifp = sc->vmx_ifp; 3172254738Sbryanv idx = (tag >> 5) & 0x7F; 3173254738Sbryanv bit = tag & 0x1F; 3174254738Sbryanv 3175254738Sbryanv if (tag == 0 || tag > 4095) 3176254738Sbryanv return; 3177254738Sbryanv 3178254738Sbryanv VMXNET3_CORE_LOCK(sc); 3179254738Sbryanv 3180254738Sbryanv /* Update our private VLAN bitvector. */ 3181254738Sbryanv if (add) 3182254738Sbryanv sc->vmx_vlan_filter[idx] |= (1 << bit); 3183254738Sbryanv else 3184254738Sbryanv sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3185254738Sbryanv 3186254738Sbryanv if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3187254738Sbryanv if (add) 3188254738Sbryanv sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3189254738Sbryanv else 3190254738Sbryanv sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3191254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3192254738Sbryanv } 3193254738Sbryanv 3194254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3195254738Sbryanv} 3196254738Sbryanv 3197254738Sbryanvstatic void 3198254738Sbryanvvmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3199254738Sbryanv{ 3200254738Sbryanv 3201254738Sbryanv if (ifp->if_softc == arg) 3202254738Sbryanv vmxnet3_update_vlan_filter(arg, 1, tag); 3203254738Sbryanv} 3204254738Sbryanv 3205254738Sbryanvstatic void 3206254738Sbryanvvmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3207254738Sbryanv{ 3208254738Sbryanv 3209254738Sbryanv if (ifp->if_softc == arg) 3210254738Sbryanv vmxnet3_update_vlan_filter(arg, 0, tag); 3211254738Sbryanv} 3212254738Sbryanv 3213254738Sbryanvstatic void 3214254738Sbryanvvmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3215254738Sbryanv{ 3216254738Sbryanv struct ifnet *ifp; 3217254738Sbryanv struct vmxnet3_driver_shared *ds; 3218254738Sbryanv struct ifmultiaddr *ifma; 3219254738Sbryanv u_int mode; 3220254738Sbryanv 3221254738Sbryanv ifp = sc->vmx_ifp; 3222254738Sbryanv ds = sc->vmx_ds; 3223254738Sbryanv 3224263259Sbryanv mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3225254738Sbryanv if (ifp->if_flags & IFF_PROMISC) 3226254738Sbryanv mode |= VMXNET3_RXMODE_PROMISC; 3227254738Sbryanv if (ifp->if_flags & IFF_ALLMULTI) 3228254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3229254738Sbryanv else { 3230254738Sbryanv int cnt = 0, overflow = 0; 3231254738Sbryanv 3232254738Sbryanv if_maddr_rlock(ifp); 3233254738Sbryanv TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3234254738Sbryanv if (ifma->ifma_addr->sa_family != AF_LINK) 3235254738Sbryanv continue; 3236254738Sbryanv else if (cnt == VMXNET3_MULTICAST_MAX) { 3237254738Sbryanv overflow = 1; 3238254738Sbryanv break; 3239254738Sbryanv } 3240254738Sbryanv 3241254738Sbryanv bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3242254738Sbryanv &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3243254738Sbryanv cnt++; 3244254738Sbryanv } 3245254738Sbryanv if_maddr_runlock(ifp); 3246254738Sbryanv 3247254738Sbryanv if (overflow != 0) { 3248254738Sbryanv cnt = 0; 3249254738Sbryanv mode |= VMXNET3_RXMODE_ALLMULTI; 3250254738Sbryanv } else if (cnt > 0) 3251254738Sbryanv mode |= VMXNET3_RXMODE_MCAST; 3252254738Sbryanv ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3253254738Sbryanv } 3254254738Sbryanv 3255254738Sbryanv ds->rxmode = mode; 3256254738Sbryanv 3257254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3258254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3259254738Sbryanv} 3260254738Sbryanv 3261254738Sbryanvstatic int 3262254738Sbryanvvmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3263254738Sbryanv{ 3264254738Sbryanv struct ifnet *ifp; 3265254738Sbryanv 3266254738Sbryanv ifp = sc->vmx_ifp; 3267254738Sbryanv 3268254738Sbryanv if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3269254738Sbryanv return (EINVAL); 3270254738Sbryanv 3271254738Sbryanv ifp->if_mtu = mtu; 3272254738Sbryanv 3273254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3274254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3275254738Sbryanv vmxnet3_init_locked(sc); 3276254738Sbryanv } 3277254738Sbryanv 3278254738Sbryanv return (0); 3279254738Sbryanv} 3280254738Sbryanv 3281254738Sbryanvstatic int 3282254738Sbryanvvmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3283254738Sbryanv{ 3284254738Sbryanv struct vmxnet3_softc *sc; 3285254738Sbryanv struct ifreq *ifr; 3286254738Sbryanv int reinit, mask, error; 3287254738Sbryanv 3288254738Sbryanv sc = ifp->if_softc; 3289254738Sbryanv ifr = (struct ifreq *) data; 3290254738Sbryanv error = 0; 3291254738Sbryanv 3292254738Sbryanv switch (cmd) { 3293254738Sbryanv case SIOCSIFMTU: 3294254738Sbryanv if (ifp->if_mtu != ifr->ifr_mtu) { 3295254738Sbryanv VMXNET3_CORE_LOCK(sc); 3296254738Sbryanv error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3297254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3298254738Sbryanv } 3299254738Sbryanv break; 3300254738Sbryanv 3301254738Sbryanv case SIOCSIFFLAGS: 3302254738Sbryanv VMXNET3_CORE_LOCK(sc); 3303254738Sbryanv if (ifp->if_flags & IFF_UP) { 3304254738Sbryanv if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3305254738Sbryanv if ((ifp->if_flags ^ sc->vmx_if_flags) & 3306254738Sbryanv (IFF_PROMISC | IFF_ALLMULTI)) { 3307254738Sbryanv vmxnet3_set_rxfilter(sc); 3308254738Sbryanv } 3309254738Sbryanv } else 3310254738Sbryanv vmxnet3_init_locked(sc); 3311254738Sbryanv } else { 3312254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3313254738Sbryanv vmxnet3_stop(sc); 3314254738Sbryanv } 3315254738Sbryanv sc->vmx_if_flags = ifp->if_flags; 3316254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3317254738Sbryanv break; 3318254738Sbryanv 3319254738Sbryanv case SIOCADDMULTI: 3320254738Sbryanv case SIOCDELMULTI: 3321254738Sbryanv VMXNET3_CORE_LOCK(sc); 3322254738Sbryanv if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3323254738Sbryanv vmxnet3_set_rxfilter(sc); 3324254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3325254738Sbryanv break; 3326254738Sbryanv 3327254738Sbryanv case SIOCSIFMEDIA: 3328254738Sbryanv case SIOCGIFMEDIA: 3329254738Sbryanv error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3330254738Sbryanv break; 3331254738Sbryanv 3332254738Sbryanv case SIOCSIFCAP: 3333254738Sbryanv VMXNET3_CORE_LOCK(sc); 3334254738Sbryanv mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3335254738Sbryanv 3336254738Sbryanv if (mask & IFCAP_TXCSUM) 3337254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM; 3338254738Sbryanv if (mask & IFCAP_TXCSUM_IPV6) 3339254738Sbryanv ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3340254738Sbryanv if (mask & IFCAP_TSO4) 3341254738Sbryanv ifp->if_capenable ^= IFCAP_TSO4; 3342254738Sbryanv if (mask & IFCAP_TSO6) 3343254738Sbryanv ifp->if_capenable ^= IFCAP_TSO6; 3344254738Sbryanv 3345254738Sbryanv if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3346255055Sbryanv IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3347255055Sbryanv /* Changing these features requires us to reinit. */ 3348254738Sbryanv reinit = 1; 3349254738Sbryanv 3350254738Sbryanv if (mask & IFCAP_RXCSUM) 3351254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM; 3352254738Sbryanv if (mask & IFCAP_RXCSUM_IPV6) 3353254738Sbryanv ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3354254738Sbryanv if (mask & IFCAP_LRO) 3355254738Sbryanv ifp->if_capenable ^= IFCAP_LRO; 3356255055Sbryanv if (mask & IFCAP_VLAN_HWTAGGING) 3357255055Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3358254738Sbryanv if (mask & IFCAP_VLAN_HWFILTER) 3359254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3360254738Sbryanv } else 3361254738Sbryanv reinit = 0; 3362254738Sbryanv 3363254738Sbryanv if (mask & IFCAP_VLAN_HWTSO) 3364254738Sbryanv ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3365254738Sbryanv 3366254738Sbryanv if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3367254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3368254738Sbryanv vmxnet3_init_locked(sc); 3369303136Smav } else { 3370303136Smav vmxnet3_init_hwassist(sc); 3371254738Sbryanv } 3372254738Sbryanv 3373254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3374254738Sbryanv VLAN_CAPABILITIES(ifp); 3375254738Sbryanv break; 3376254738Sbryanv 3377254738Sbryanv default: 3378254738Sbryanv error = ether_ioctl(ifp, cmd, data); 3379254738Sbryanv break; 3380254738Sbryanv } 3381254738Sbryanv 3382254738Sbryanv VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3383254738Sbryanv 3384254738Sbryanv return (error); 3385254738Sbryanv} 3386254738Sbryanv 3387263259Sbryanv#ifndef VMXNET3_LEGACY_TX 3388263259Sbryanvstatic void 3389263259Sbryanvvmxnet3_qflush(struct ifnet *ifp) 3390263259Sbryanv{ 3391263259Sbryanv struct vmxnet3_softc *sc; 3392263259Sbryanv struct vmxnet3_txqueue *txq; 3393263259Sbryanv struct mbuf *m; 3394263259Sbryanv int i; 3395263259Sbryanv 3396263259Sbryanv sc = ifp->if_softc; 3397263259Sbryanv 3398263259Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3399263259Sbryanv txq = &sc->vmx_txq[i]; 3400263259Sbryanv 3401263259Sbryanv VMXNET3_TXQ_LOCK(txq); 3402263259Sbryanv while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3403263259Sbryanv m_freem(m); 3404263259Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3405263259Sbryanv } 3406263259Sbryanv 3407263259Sbryanv if_qflush(ifp); 3408263259Sbryanv} 3409263259Sbryanv#endif 3410263259Sbryanv 3411254738Sbryanvstatic int 3412254738Sbryanvvmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3413254738Sbryanv{ 3414254738Sbryanv struct vmxnet3_softc *sc; 3415254738Sbryanv 3416254738Sbryanv sc = txq->vxtxq_sc; 3417254738Sbryanv 3418254738Sbryanv VMXNET3_TXQ_LOCK(txq); 3419254738Sbryanv if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3420254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3421254738Sbryanv return (0); 3422254738Sbryanv } 3423254738Sbryanv VMXNET3_TXQ_UNLOCK(txq); 3424254738Sbryanv 3425254738Sbryanv if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3426254738Sbryanv txq->vxtxq_id); 3427254738Sbryanv return (1); 3428254738Sbryanv} 3429254738Sbryanv 3430254738Sbryanvstatic void 3431263259Sbryanvvmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3432254738Sbryanv{ 3433254738Sbryanv 3434254738Sbryanv vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3435254738Sbryanv} 3436254738Sbryanv 3437272099Sglebiusstatic uint64_t 3438272099Sglebiusvmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt) 3439263259Sbryanv{ 3440272099Sglebius struct vmxnet3_softc *sc; 3441272099Sglebius uint64_t rv; 3442263259Sbryanv 3443272099Sglebius sc = if_getsoftc(ifp); 3444272099Sglebius rv = 0; 3445263259Sbryanv 3446263259Sbryanv /* 3447263259Sbryanv * With the exception of if_ierrors, these ifnet statistics are 3448263259Sbryanv * only updated in the driver, so just set them to our accumulated 3449263259Sbryanv * values. if_ierrors is updated in ether_input() for malformed 3450263259Sbryanv * frames that we should have already discarded. 3451263259Sbryanv */ 3452272099Sglebius switch (cnt) { 3453272099Sglebius case IFCOUNTER_IPACKETS: 3454272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3455272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets; 3456272099Sglebius return (rv); 3457272099Sglebius case IFCOUNTER_IQDROPS: 3458272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3459272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops; 3460272099Sglebius return (rv); 3461272099Sglebius case IFCOUNTER_IERRORS: 3462272099Sglebius for (int i = 0; i < sc->vmx_nrxqueues; i++) 3463272099Sglebius rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors; 3464272099Sglebius return (rv); 3465272099Sglebius case IFCOUNTER_OPACKETS: 3466272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3467272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets; 3468272099Sglebius return (rv); 3469263259Sbryanv#ifndef VMXNET3_LEGACY_TX 3470272099Sglebius case IFCOUNTER_OBYTES: 3471272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3472272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes; 3473272099Sglebius return (rv); 3474272099Sglebius case IFCOUNTER_OMCASTS: 3475272099Sglebius for (int i = 0; i < sc->vmx_ntxqueues; i++) 3476272099Sglebius rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts; 3477272099Sglebius return (rv); 3478263259Sbryanv#endif 3479272099Sglebius default: 3480272099Sglebius return (if_get_counter_default(ifp, cnt)); 3481272099Sglebius } 3482263259Sbryanv} 3483263259Sbryanv 3484263259Sbryanvstatic void 3485254738Sbryanvvmxnet3_tick(void *xsc) 3486254738Sbryanv{ 3487254738Sbryanv struct vmxnet3_softc *sc; 3488254738Sbryanv struct ifnet *ifp; 3489254738Sbryanv int i, timedout; 3490254738Sbryanv 3491254738Sbryanv sc = xsc; 3492254738Sbryanv ifp = sc->vmx_ifp; 3493254738Sbryanv timedout = 0; 3494254738Sbryanv 3495254738Sbryanv VMXNET3_CORE_LOCK_ASSERT(sc); 3496254738Sbryanv 3497263259Sbryanv vmxnet3_refresh_host_stats(sc); 3498263259Sbryanv 3499254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3500254738Sbryanv timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3501254738Sbryanv 3502254738Sbryanv if (timedout != 0) { 3503254738Sbryanv ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3504254738Sbryanv vmxnet3_init_locked(sc); 3505254738Sbryanv } else 3506254738Sbryanv callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3507254738Sbryanv} 3508254738Sbryanv 3509254738Sbryanvstatic int 3510254738Sbryanvvmxnet3_link_is_up(struct vmxnet3_softc *sc) 3511254738Sbryanv{ 3512254738Sbryanv uint32_t status; 3513254738Sbryanv 3514254738Sbryanv /* Also update the link speed while here. */ 3515254738Sbryanv status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3516254738Sbryanv sc->vmx_link_speed = status >> 16; 3517254738Sbryanv return !!(status & 0x1); 3518254738Sbryanv} 3519254738Sbryanv 3520254738Sbryanvstatic void 3521254738Sbryanvvmxnet3_link_status(struct vmxnet3_softc *sc) 3522254738Sbryanv{ 3523254738Sbryanv struct ifnet *ifp; 3524254738Sbryanv int link; 3525254738Sbryanv 3526254738Sbryanv ifp = sc->vmx_ifp; 3527254738Sbryanv link = vmxnet3_link_is_up(sc); 3528254738Sbryanv 3529254738Sbryanv if (link != 0 && sc->vmx_link_active == 0) { 3530254738Sbryanv sc->vmx_link_active = 1; 3531254738Sbryanv if_link_state_change(ifp, LINK_STATE_UP); 3532254738Sbryanv } else if (link == 0 && sc->vmx_link_active != 0) { 3533254738Sbryanv sc->vmx_link_active = 0; 3534254738Sbryanv if_link_state_change(ifp, LINK_STATE_DOWN); 3535254738Sbryanv } 3536254738Sbryanv} 3537254738Sbryanv 3538254738Sbryanvstatic void 3539254738Sbryanvvmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3540254738Sbryanv{ 3541254738Sbryanv struct vmxnet3_softc *sc; 3542254738Sbryanv 3543254738Sbryanv sc = ifp->if_softc; 3544254738Sbryanv 3545254738Sbryanv ifmr->ifm_status = IFM_AVALID; 3546326751Sbryanv ifmr->ifm_active = IFM_ETHER; 3547254738Sbryanv 3548254738Sbryanv VMXNET3_CORE_LOCK(sc); 3549326751Sbryanv if (vmxnet3_link_is_up(sc) != 0) { 3550254738Sbryanv ifmr->ifm_status |= IFM_ACTIVE; 3551326751Sbryanv ifmr->ifm_active |= IFM_AUTO; 3552326751Sbryanv } else 3553326751Sbryanv ifmr->ifm_active |= IFM_NONE; 3554254738Sbryanv VMXNET3_CORE_UNLOCK(sc); 3555254738Sbryanv} 3556254738Sbryanv 3557254738Sbryanvstatic int 3558254738Sbryanvvmxnet3_media_change(struct ifnet *ifp) 3559254738Sbryanv{ 3560254738Sbryanv 3561254738Sbryanv /* Ignore. */ 3562254738Sbryanv return (0); 3563254738Sbryanv} 3564254738Sbryanv 3565254738Sbryanvstatic void 3566254738Sbryanvvmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3567254738Sbryanv{ 3568254738Sbryanv uint32_t ml, mh; 3569254738Sbryanv 3570254738Sbryanv ml = sc->vmx_lladdr[0]; 3571254738Sbryanv ml |= sc->vmx_lladdr[1] << 8; 3572254738Sbryanv ml |= sc->vmx_lladdr[2] << 16; 3573254738Sbryanv ml |= sc->vmx_lladdr[3] << 24; 3574254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3575254738Sbryanv 3576254738Sbryanv mh = sc->vmx_lladdr[4]; 3577254738Sbryanv mh |= sc->vmx_lladdr[5] << 8; 3578254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3579254738Sbryanv} 3580254738Sbryanv 3581254738Sbryanvstatic void 3582254738Sbryanvvmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3583254738Sbryanv{ 3584254738Sbryanv uint32_t ml, mh; 3585254738Sbryanv 3586254738Sbryanv ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3587254738Sbryanv mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3588254738Sbryanv 3589254738Sbryanv sc->vmx_lladdr[0] = ml; 3590254738Sbryanv sc->vmx_lladdr[1] = ml >> 8; 3591254738Sbryanv sc->vmx_lladdr[2] = ml >> 16; 3592254738Sbryanv sc->vmx_lladdr[3] = ml >> 24; 3593254738Sbryanv sc->vmx_lladdr[4] = mh; 3594254738Sbryanv sc->vmx_lladdr[5] = mh >> 8; 3595254738Sbryanv} 3596254738Sbryanv 3597254738Sbryanvstatic void 3598254738Sbryanvvmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3599254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3600254738Sbryanv{ 3601254738Sbryanv struct sysctl_oid *node, *txsnode; 3602254738Sbryanv struct sysctl_oid_list *list, *txslist; 3603254738Sbryanv struct vmxnet3_txq_stats *stats; 3604254738Sbryanv struct UPT1_TxStats *txstats; 3605254738Sbryanv char namebuf[16]; 3606254738Sbryanv 3607254738Sbryanv stats = &txq->vxtxq_stats; 3608254738Sbryanv txstats = &txq->vxtxq_ts->stats; 3609254738Sbryanv 3610254738Sbryanv snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3611254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3612254738Sbryanv NULL, "Transmit Queue"); 3613254738Sbryanv txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3614254738Sbryanv 3615263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3616263259Sbryanv &stats->vmtxs_opackets, "Transmit packets"); 3617263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3618263259Sbryanv &stats->vmtxs_obytes, "Transmit bytes"); 3619263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3620263259Sbryanv &stats->vmtxs_omcasts, "Transmit multicasts"); 3621263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3622263259Sbryanv &stats->vmtxs_csum, "Transmit checksum offloaded"); 3623263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3624263259Sbryanv &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3625254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3626263259Sbryanv &stats->vmtxs_full, "Transmit ring full"); 3627254738Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3628263259Sbryanv &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3629254738Sbryanv 3630254738Sbryanv /* 3631254738Sbryanv * Add statistics reported by the host. These are updated once 3632254738Sbryanv * per second. 3633254738Sbryanv */ 3634254738Sbryanv txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3635254738Sbryanv NULL, "Host Statistics"); 3636254738Sbryanv txslist = SYSCTL_CHILDREN(txsnode); 3637254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3638254738Sbryanv &txstats->TSO_packets, "TSO packets"); 3639254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3640254738Sbryanv &txstats->TSO_bytes, "TSO bytes"); 3641254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3642254738Sbryanv &txstats->ucast_packets, "Unicast packets"); 3643254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3644254738Sbryanv &txstats->ucast_bytes, "Unicast bytes"); 3645254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3646254738Sbryanv &txstats->mcast_packets, "Multicast packets"); 3647254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3648254738Sbryanv &txstats->mcast_bytes, "Multicast bytes"); 3649254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3650254738Sbryanv &txstats->error, "Errors"); 3651254738Sbryanv SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3652254738Sbryanv &txstats->discard, "Discards"); 3653254738Sbryanv} 3654254738Sbryanv 3655254738Sbryanvstatic void 3656254738Sbryanvvmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3657254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3658254738Sbryanv{ 3659254738Sbryanv struct sysctl_oid *node, *rxsnode; 3660254738Sbryanv struct sysctl_oid_list *list, *rxslist; 3661254738Sbryanv struct vmxnet3_rxq_stats *stats; 3662254738Sbryanv struct UPT1_RxStats *rxstats; 3663254738Sbryanv char namebuf[16]; 3664254738Sbryanv 3665254738Sbryanv stats = &rxq->vxrxq_stats; 3666254738Sbryanv rxstats = &rxq->vxrxq_rs->stats; 3667254738Sbryanv 3668254738Sbryanv snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3669254738Sbryanv node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3670254738Sbryanv NULL, "Receive Queue"); 3671254738Sbryanv rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3672254738Sbryanv 3673263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3674263259Sbryanv &stats->vmrxs_ipackets, "Receive packets"); 3675263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3676263259Sbryanv &stats->vmrxs_ibytes, "Receive bytes"); 3677263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3678263259Sbryanv &stats->vmrxs_iqdrops, "Receive drops"); 3679263259Sbryanv SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3680263259Sbryanv &stats->vmrxs_ierrors, "Receive errors"); 3681263259Sbryanv 3682254738Sbryanv /* 3683254738Sbryanv * Add statistics reported by the host. These are updated once 3684254738Sbryanv * per second. 3685254738Sbryanv */ 3686254738Sbryanv rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3687254738Sbryanv NULL, "Host Statistics"); 3688254738Sbryanv rxslist = SYSCTL_CHILDREN(rxsnode); 3689254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3690254738Sbryanv &rxstats->LRO_packets, "LRO packets"); 3691254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3692254738Sbryanv &rxstats->LRO_bytes, "LRO bytes"); 3693254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3694254738Sbryanv &rxstats->ucast_packets, "Unicast packets"); 3695254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3696254738Sbryanv &rxstats->ucast_bytes, "Unicast bytes"); 3697254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3698254738Sbryanv &rxstats->mcast_packets, "Multicast packets"); 3699254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3700254738Sbryanv &rxstats->mcast_bytes, "Multicast bytes"); 3701254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3702254738Sbryanv &rxstats->bcast_packets, "Broadcast packets"); 3703254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3704254738Sbryanv &rxstats->bcast_bytes, "Broadcast bytes"); 3705254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3706254738Sbryanv &rxstats->nobuffer, "No buffer"); 3707254738Sbryanv SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3708254738Sbryanv &rxstats->error, "Errors"); 3709254738Sbryanv} 3710254738Sbryanv 3711254738Sbryanvstatic void 3712254738Sbryanvvmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3713254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3714254738Sbryanv{ 3715254738Sbryanv struct sysctl_oid *node; 3716254738Sbryanv struct sysctl_oid_list *list; 3717254738Sbryanv int i; 3718254738Sbryanv 3719254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) { 3720254738Sbryanv struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3721254738Sbryanv 3722254738Sbryanv node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3723254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3724254738Sbryanv list = SYSCTL_CHILDREN(node); 3725254738Sbryanv 3726254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3727254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3728254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3729254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3730254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3731254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3732254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3733254738Sbryanv &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3734254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3735254738Sbryanv &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3736254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3737254738Sbryanv &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3738254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3739254738Sbryanv &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3740254738Sbryanv } 3741254738Sbryanv 3742254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) { 3743254738Sbryanv struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3744254738Sbryanv 3745254738Sbryanv node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3746254738Sbryanv "debug", CTLFLAG_RD, NULL, ""); 3747254738Sbryanv list = SYSCTL_CHILDREN(node); 3748254738Sbryanv 3749254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3750254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3751254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3752254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3753254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3754254738Sbryanv &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3755254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3756254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3757254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3758254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3759254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3760254738Sbryanv &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3761254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3762254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3763254738Sbryanv SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3764254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3765254738Sbryanv SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3766254738Sbryanv &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3767254738Sbryanv } 3768254738Sbryanv} 3769254738Sbryanv 3770254738Sbryanvstatic void 3771254738Sbryanvvmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3772254738Sbryanv struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3773254738Sbryanv{ 3774254738Sbryanv int i; 3775254738Sbryanv 3776254738Sbryanv for (i = 0; i < sc->vmx_ntxqueues; i++) 3777254738Sbryanv vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3778254738Sbryanv for (i = 0; i < sc->vmx_nrxqueues; i++) 3779254738Sbryanv vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3780254738Sbryanv 3781254738Sbryanv vmxnet3_setup_debug_sysctl(sc, ctx, child); 3782254738Sbryanv} 3783254738Sbryanv 3784254738Sbryanvstatic void 3785254738Sbryanvvmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3786254738Sbryanv{ 3787254738Sbryanv device_t dev; 3788254738Sbryanv struct vmxnet3_statistics *stats; 3789254738Sbryanv struct sysctl_ctx_list *ctx; 3790254738Sbryanv struct sysctl_oid *tree; 3791254738Sbryanv struct sysctl_oid_list *child; 3792254738Sbryanv 3793254738Sbryanv dev = sc->vmx_dev; 3794254738Sbryanv ctx = device_get_sysctl_ctx(dev); 3795254738Sbryanv tree = device_get_sysctl_tree(dev); 3796254738Sbryanv child = SYSCTL_CHILDREN(tree); 3797254738Sbryanv 3798263259Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3799263259Sbryanv &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3800263259Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3801263259Sbryanv &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3802254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3803254738Sbryanv &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3804254738Sbryanv SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3805254738Sbryanv &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3806254738Sbryanv 3807254738Sbryanv stats = &sc->vmx_stats; 3808263259Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3809263259Sbryanv &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3810263259Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3811263259Sbryanv &stats->vmst_defrag_failed, 0, 3812263259Sbryanv "Tx mbuf dropped because defrag failed"); 3813254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3814254738Sbryanv &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3815254738Sbryanv SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3816254738Sbryanv &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3817254738Sbryanv 3818254738Sbryanv vmxnet3_setup_queue_sysctl(sc, ctx, child); 3819254738Sbryanv} 3820254738Sbryanv 3821254738Sbryanvstatic void 3822254738Sbryanvvmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3823254738Sbryanv{ 3824254738Sbryanv 3825254738Sbryanv bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3826254738Sbryanv} 3827254738Sbryanv 3828254738Sbryanvstatic uint32_t 3829254738Sbryanvvmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3830254738Sbryanv{ 3831254738Sbryanv 3832254738Sbryanv return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3833254738Sbryanv} 3834254738Sbryanv 3835254738Sbryanvstatic void 3836254738Sbryanvvmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3837254738Sbryanv{ 3838254738Sbryanv 3839254738Sbryanv bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3840254738Sbryanv} 3841254738Sbryanv 3842254738Sbryanvstatic void 3843254738Sbryanvvmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3844254738Sbryanv{ 3845254738Sbryanv 3846254738Sbryanv vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3847254738Sbryanv} 3848254738Sbryanv 3849254738Sbryanvstatic uint32_t 3850254738Sbryanvvmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3851254738Sbryanv{ 3852254738Sbryanv 3853254738Sbryanv vmxnet3_write_cmd(sc, cmd); 3854254738Sbryanv bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3855254738Sbryanv BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3856254738Sbryanv return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3857254738Sbryanv} 3858254738Sbryanv 3859254738Sbryanvstatic void 3860254738Sbryanvvmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3861254738Sbryanv{ 3862254738Sbryanv 3863254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3864254738Sbryanv} 3865254738Sbryanv 3866254738Sbryanvstatic void 3867254738Sbryanvvmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3868254738Sbryanv{ 3869254738Sbryanv 3870254738Sbryanv vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3871254738Sbryanv} 3872254738Sbryanv 3873254738Sbryanvstatic void 3874254738Sbryanvvmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3875254738Sbryanv{ 3876254738Sbryanv int i; 3877254738Sbryanv 3878254738Sbryanv sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3879254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3880254738Sbryanv vmxnet3_enable_intr(sc, i); 3881254738Sbryanv} 3882254738Sbryanv 3883254738Sbryanvstatic void 3884254738Sbryanvvmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3885254738Sbryanv{ 3886254738Sbryanv int i; 3887254738Sbryanv 3888254738Sbryanv sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3889254738Sbryanv for (i = 0; i < sc->vmx_nintrs; i++) 3890254738Sbryanv vmxnet3_disable_intr(sc, i); 3891254738Sbryanv} 3892254738Sbryanv 3893254738Sbryanvstatic void 3894254738Sbryanvvmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3895254738Sbryanv{ 3896254738Sbryanv bus_addr_t *baddr = arg; 3897254738Sbryanv 3898254738Sbryanv if (error == 0) 3899254738Sbryanv *baddr = segs->ds_addr; 3900254738Sbryanv} 3901254738Sbryanv 3902254738Sbryanvstatic int 3903254738Sbryanvvmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3904254738Sbryanv struct vmxnet3_dma_alloc *dma) 3905254738Sbryanv{ 3906254738Sbryanv device_t dev; 3907254738Sbryanv int error; 3908254738Sbryanv 3909254738Sbryanv dev = sc->vmx_dev; 3910254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3911254738Sbryanv 3912254738Sbryanv error = bus_dma_tag_create(bus_get_dma_tag(dev), 3913254738Sbryanv align, 0, /* alignment, bounds */ 3914254738Sbryanv BUS_SPACE_MAXADDR, /* lowaddr */ 3915254738Sbryanv BUS_SPACE_MAXADDR, /* highaddr */ 3916254738Sbryanv NULL, NULL, /* filter, filterarg */ 3917254738Sbryanv size, /* maxsize */ 3918254738Sbryanv 1, /* nsegments */ 3919254738Sbryanv size, /* maxsegsize */ 3920254738Sbryanv BUS_DMA_ALLOCNOW, /* flags */ 3921254738Sbryanv NULL, /* lockfunc */ 3922254738Sbryanv NULL, /* lockfuncarg */ 3923254738Sbryanv &dma->dma_tag); 3924254738Sbryanv if (error) { 3925254738Sbryanv device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3926254738Sbryanv goto fail; 3927254738Sbryanv } 3928254738Sbryanv 3929254738Sbryanv error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3930254738Sbryanv BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3931254738Sbryanv if (error) { 3932254738Sbryanv device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3933254738Sbryanv goto fail; 3934254738Sbryanv } 3935254738Sbryanv 3936254738Sbryanv error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3937254738Sbryanv size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3938254738Sbryanv if (error) { 3939254738Sbryanv device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3940254738Sbryanv goto fail; 3941254738Sbryanv } 3942254738Sbryanv 3943254738Sbryanv dma->dma_size = size; 3944254738Sbryanv 3945254738Sbryanvfail: 3946254738Sbryanv if (error) 3947254738Sbryanv vmxnet3_dma_free(sc, dma); 3948254738Sbryanv 3949254738Sbryanv return (error); 3950254738Sbryanv} 3951254738Sbryanv 3952254738Sbryanvstatic void 3953254738Sbryanvvmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3954254738Sbryanv{ 3955254738Sbryanv 3956254738Sbryanv if (dma->dma_tag != NULL) { 3957267488Sbryanv if (dma->dma_paddr != 0) { 3958254738Sbryanv bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3959254738Sbryanv BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3960254738Sbryanv bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3961254738Sbryanv } 3962254738Sbryanv 3963254738Sbryanv if (dma->dma_vaddr != NULL) { 3964254738Sbryanv bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3965254738Sbryanv dma->dma_map); 3966254738Sbryanv } 3967254738Sbryanv 3968254738Sbryanv bus_dma_tag_destroy(dma->dma_tag); 3969254738Sbryanv } 3970254738Sbryanv bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3971254738Sbryanv} 3972254738Sbryanv 3973255055Sbryanvstatic int 3974255055Sbryanvvmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3975255055Sbryanv{ 3976255055Sbryanv char path[64]; 3977255055Sbryanv 3978255055Sbryanv snprintf(path, sizeof(path), 3979255055Sbryanv "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3980255055Sbryanv TUNABLE_INT_FETCH(path, &def); 3981255055Sbryanv 3982255055Sbryanv return (def); 3983255055Sbryanv} 3984255055Sbryanv 3985254738Sbryanv/* 3986254738Sbryanv * Since this is a purely paravirtualized device, we do not have 3987254738Sbryanv * to worry about DMA coherency. But at times, we must make sure 3988254738Sbryanv * both the compiler and CPU do not reorder memory operations. 3989254738Sbryanv */ 3990254738Sbryanvstatic inline void 3991254738Sbryanvvmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3992254738Sbryanv{ 3993254738Sbryanv 3994254738Sbryanv switch (type) { 3995254738Sbryanv case VMXNET3_BARRIER_RD: 3996254738Sbryanv rmb(); 3997254738Sbryanv break; 3998254738Sbryanv case VMXNET3_BARRIER_WR: 3999254738Sbryanv wmb(); 4000254738Sbryanv break; 4001254738Sbryanv case VMXNET3_BARRIER_RDWR: 4002254738Sbryanv mb(); 4003254738Sbryanv break; 4004254738Sbryanv default: 4005254738Sbryanv panic("%s: bad barrier type %d", __func__, type); 4006254738Sbryanv } 4007254738Sbryanv} 4008