if_vmx.c revision 267187
1/*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20/* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22#include <sys/cdefs.h> 23__FBSDID("$FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 267187 2014-06-06 20:49:56Z luigi $"); 24 25#include <sys/param.h> 26#include <sys/systm.h> 27#include <sys/eventhandler.h> 28#include <sys/kernel.h> 29#include <sys/endian.h> 30#include <sys/sockio.h> 31#include <sys/mbuf.h> 32#include <sys/malloc.h> 33#include <sys/module.h> 34#include <sys/socket.h> 35#include <sys/sysctl.h> 36#include <sys/smp.h> 37#include <sys/taskqueue.h> 38#include <vm/vm.h> 39#include <vm/pmap.h> 40 41#include <net/ethernet.h> 42#include <net/if.h> 43#include <net/if_var.h> 44#include <net/if_arp.h> 45#include <net/if_dl.h> 46#include <net/if_types.h> 47#include <net/if_media.h> 48#include <net/if_vlan_var.h> 49 50#include <net/bpf.h> 51 52#include <netinet/in_systm.h> 53#include <netinet/in.h> 54#include <netinet/ip.h> 55#include <netinet/ip6.h> 56#include <netinet6/ip6_var.h> 57#include <netinet/udp.h> 58#include <netinet/tcp.h> 59 60#include <machine/bus.h> 61#include <machine/resource.h> 62#include <sys/bus.h> 63#include <sys/rman.h> 64 65#include <dev/pci/pcireg.h> 66#include <dev/pci/pcivar.h> 67 68#include "if_vmxreg.h" 69#include "if_vmxvar.h" 70 71#include "opt_inet.h" 72#include "opt_inet6.h" 73 74#ifdef VMXNET3_FAILPOINTS 75#include <sys/fail.h> 76static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 77 "vmxnet3 fail points"); 78#define VMXNET3_FP _debug_fail_point_vmxnet3 79#endif 80 81static int vmxnet3_probe(device_t); 82static int vmxnet3_attach(device_t); 83static int vmxnet3_detach(device_t); 84static int vmxnet3_shutdown(device_t); 85 86static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 87static void vmxnet3_free_resources(struct vmxnet3_softc *); 88static int vmxnet3_check_version(struct vmxnet3_softc *); 89static void vmxnet3_initial_config(struct vmxnet3_softc *); 90static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 91 92static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 93static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 94static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 95static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 96 struct vmxnet3_interrupt *); 97static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 98static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 99static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 100static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 101static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 102 103static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 104 struct vmxnet3_interrupt *); 105static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 106 107#ifndef VMXNET3_LEGACY_TX 108static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 109static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 110static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 111static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 112#endif 113 114static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 115static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 116static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 117static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 118static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 119static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 120 121static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 122static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 123static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 124static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 125static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 126static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 127static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 128static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 129static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 130static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 131static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 132static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 133static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 134static int vmxnet3_alloc_data(struct vmxnet3_softc *); 135static void vmxnet3_free_data(struct vmxnet3_softc *); 136static int vmxnet3_setup_interface(struct vmxnet3_softc *); 137 138static void vmxnet3_evintr(struct vmxnet3_softc *); 139static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 140static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 141static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 142static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 143 struct vmxnet3_rxring *, int); 144static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 145static void vmxnet3_legacy_intr(void *); 146static void vmxnet3_txq_intr(void *); 147static void vmxnet3_rxq_intr(void *); 148static void vmxnet3_event_intr(void *); 149 150static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 151static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 152static void vmxnet3_stop(struct vmxnet3_softc *); 153 154static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 155static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 156static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 157static int vmxnet3_enable_device(struct vmxnet3_softc *); 158static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 159static int vmxnet3_reinit(struct vmxnet3_softc *); 160static void vmxnet3_init_locked(struct vmxnet3_softc *); 161static void vmxnet3_init(void *); 162 163static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 164 int *, int *, int *); 165static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 166 bus_dmamap_t, bus_dma_segment_t [], int *); 167static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 168static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 169static void vmxnet3_txq_update_pending(struct vmxnet3_txqueue *); 170#ifdef VMXNET3_LEGACY_TX 171static void vmxnet3_start_locked(struct ifnet *); 172static void vmxnet3_start(struct ifnet *); 173#else 174static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 175 struct mbuf *); 176static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 177static void vmxnet3_txq_tq_deferred(void *, int); 178#endif 179static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 180static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 181 182static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 183 uint16_t); 184static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 185static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 186static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 187static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 188static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 189 190#ifndef VMXNET3_LEGACY_TX 191static void vmxnet3_qflush(struct ifnet *); 192#endif 193 194static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 195static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 196static void vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *, 197 struct vmxnet3_txq_stats *); 198static void vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *, 199 struct vmxnet3_rxq_stats *); 200static void vmxnet3_tick(void *); 201static void vmxnet3_link_status(struct vmxnet3_softc *); 202static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 203static int vmxnet3_media_change(struct ifnet *); 204static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 205static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 206 207static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 208 struct sysctl_ctx_list *, struct sysctl_oid_list *); 209static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 210 struct sysctl_ctx_list *, struct sysctl_oid_list *); 211static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 212 struct sysctl_ctx_list *, struct sysctl_oid_list *); 213static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 214 215static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 216 uint32_t); 217static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 218static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 219 uint32_t); 220static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 221static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 222 223static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 224static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 225static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 226static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 227 228static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 229 bus_size_t, struct vmxnet3_dma_alloc *); 230static void vmxnet3_dma_free(struct vmxnet3_softc *, 231 struct vmxnet3_dma_alloc *); 232static int vmxnet3_tunable_int(struct vmxnet3_softc *, 233 const char *, int); 234 235typedef enum { 236 VMXNET3_BARRIER_RD, 237 VMXNET3_BARRIER_WR, 238 VMXNET3_BARRIER_RDWR, 239} vmxnet3_barrier_t; 240 241static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 242 243/* Tunables. */ 244static int vmxnet3_mq_disable = 0; 245TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 246static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 247TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 248static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 249TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 250static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 251TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 252static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 253TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 254 255static device_method_t vmxnet3_methods[] = { 256 /* Device interface. */ 257 DEVMETHOD(device_probe, vmxnet3_probe), 258 DEVMETHOD(device_attach, vmxnet3_attach), 259 DEVMETHOD(device_detach, vmxnet3_detach), 260 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 261 262 DEVMETHOD_END 263}; 264 265static driver_t vmxnet3_driver = { 266 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 267}; 268 269static devclass_t vmxnet3_devclass; 270DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 271 272MODULE_DEPEND(vmx, pci, 1, 1, 1); 273MODULE_DEPEND(vmx, ether, 1, 1, 1); 274 275#define VMXNET3_VMWARE_VENDOR_ID 0x15AD 276#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 277 278static int 279vmxnet3_probe(device_t dev) 280{ 281 282 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 283 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 284 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 285 return (BUS_PROBE_DEFAULT); 286 } 287 288 return (ENXIO); 289} 290 291static int 292vmxnet3_attach(device_t dev) 293{ 294 struct vmxnet3_softc *sc; 295 int error; 296 297 sc = device_get_softc(dev); 298 sc->vmx_dev = dev; 299 300 pci_enable_busmaster(dev); 301 302 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 303 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 304 305 vmxnet3_initial_config(sc); 306 307 error = vmxnet3_alloc_resources(sc); 308 if (error) 309 goto fail; 310 311 error = vmxnet3_check_version(sc); 312 if (error) 313 goto fail; 314 315 error = vmxnet3_alloc_rxtx_queues(sc); 316 if (error) 317 goto fail; 318 319#ifndef VMXNET3_LEGACY_TX 320 error = vmxnet3_alloc_taskqueue(sc); 321 if (error) 322 goto fail; 323#endif 324 325 error = vmxnet3_alloc_interrupts(sc); 326 if (error) 327 goto fail; 328 329 vmxnet3_check_multiqueue(sc); 330 331 error = vmxnet3_alloc_data(sc); 332 if (error) 333 goto fail; 334 335 error = vmxnet3_setup_interface(sc); 336 if (error) 337 goto fail; 338 339 error = vmxnet3_setup_interrupts(sc); 340 if (error) { 341 ether_ifdetach(sc->vmx_ifp); 342 device_printf(dev, "could not set up interrupt\n"); 343 goto fail; 344 } 345 346 vmxnet3_setup_sysctl(sc); 347#ifndef VMXNET3_LEGACY_TX 348 vmxnet3_start_taskqueue(sc); 349#endif 350 351fail: 352 if (error) 353 vmxnet3_detach(dev); 354 355 return (error); 356} 357 358static int 359vmxnet3_detach(device_t dev) 360{ 361 struct vmxnet3_softc *sc; 362 struct ifnet *ifp; 363 364 sc = device_get_softc(dev); 365 ifp = sc->vmx_ifp; 366 367 if (device_is_attached(dev)) { 368 VMXNET3_CORE_LOCK(sc); 369 vmxnet3_stop(sc); 370 VMXNET3_CORE_UNLOCK(sc); 371 372 callout_drain(&sc->vmx_tick); 373#ifndef VMXNET3_LEGACY_TX 374 vmxnet3_drain_taskqueue(sc); 375#endif 376 377 ether_ifdetach(ifp); 378 } 379 380 if (sc->vmx_vlan_attach != NULL) { 381 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 382 sc->vmx_vlan_attach = NULL; 383 } 384 if (sc->vmx_vlan_detach != NULL) { 385 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 386 sc->vmx_vlan_detach = NULL; 387 } 388 389#ifndef VMXNET3_LEGACY_TX 390 vmxnet3_free_taskqueue(sc); 391#endif 392 vmxnet3_free_interrupts(sc); 393 394 if (ifp != NULL) { 395 if_free(ifp); 396 sc->vmx_ifp = NULL; 397 } 398 399 ifmedia_removeall(&sc->vmx_media); 400 401 vmxnet3_free_data(sc); 402 vmxnet3_free_resources(sc); 403 vmxnet3_free_rxtx_queues(sc); 404 405 VMXNET3_CORE_LOCK_DESTROY(sc); 406 407 return (0); 408} 409 410static int 411vmxnet3_shutdown(device_t dev) 412{ 413 414 return (0); 415} 416 417static int 418vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 419{ 420 device_t dev; 421 int rid; 422 423 dev = sc->vmx_dev; 424 425 rid = PCIR_BAR(0); 426 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 427 RF_ACTIVE); 428 if (sc->vmx_res0 == NULL) { 429 device_printf(dev, 430 "could not map BAR0 memory\n"); 431 return (ENXIO); 432 } 433 434 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 435 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 436 437 rid = PCIR_BAR(1); 438 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 439 RF_ACTIVE); 440 if (sc->vmx_res1 == NULL) { 441 device_printf(dev, 442 "could not map BAR1 memory\n"); 443 return (ENXIO); 444 } 445 446 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 447 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 448 449 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 450 rid = PCIR_BAR(2); 451 sc->vmx_msix_res = bus_alloc_resource_any(dev, 452 SYS_RES_MEMORY, &rid, RF_ACTIVE); 453 } 454 455 if (sc->vmx_msix_res == NULL) 456 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 457 458 return (0); 459} 460 461static void 462vmxnet3_free_resources(struct vmxnet3_softc *sc) 463{ 464 device_t dev; 465 int rid; 466 467 dev = sc->vmx_dev; 468 469 if (sc->vmx_res0 != NULL) { 470 rid = PCIR_BAR(0); 471 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 472 sc->vmx_res0 = NULL; 473 } 474 475 if (sc->vmx_res1 != NULL) { 476 rid = PCIR_BAR(1); 477 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 478 sc->vmx_res1 = NULL; 479 } 480 481 if (sc->vmx_msix_res != NULL) { 482 rid = PCIR_BAR(2); 483 bus_release_resource(dev, SYS_RES_MEMORY, rid, 484 sc->vmx_msix_res); 485 sc->vmx_msix_res = NULL; 486 } 487} 488 489static int 490vmxnet3_check_version(struct vmxnet3_softc *sc) 491{ 492 device_t dev; 493 uint32_t version; 494 495 dev = sc->vmx_dev; 496 497 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 498 if ((version & 0x01) == 0) { 499 device_printf(dev, "unsupported hardware version %#x\n", 500 version); 501 return (ENOTSUP); 502 } 503 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 504 505 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 506 if ((version & 0x01) == 0) { 507 device_printf(dev, "unsupported UPT version %#x\n", version); 508 return (ENOTSUP); 509 } 510 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 511 512 return (0); 513} 514 515static void 516vmxnet3_initial_config(struct vmxnet3_softc *sc) 517{ 518 int nqueue, ndesc; 519 520 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 521 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 522 nqueue = VMXNET3_DEF_TX_QUEUES; 523 if (nqueue > mp_ncpus) 524 nqueue = mp_ncpus; 525 sc->vmx_max_ntxqueues = nqueue; 526 527 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 528 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 529 nqueue = VMXNET3_DEF_RX_QUEUES; 530 if (nqueue > mp_ncpus) 531 nqueue = mp_ncpus; 532 sc->vmx_max_nrxqueues = nqueue; 533 534 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 535 sc->vmx_max_nrxqueues = 1; 536 sc->vmx_max_ntxqueues = 1; 537 } 538 539 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 540 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 541 ndesc = VMXNET3_DEF_TX_NDESC; 542 if (ndesc & VMXNET3_MASK_TX_NDESC) 543 ndesc &= ~VMXNET3_MASK_TX_NDESC; 544 sc->vmx_ntxdescs = ndesc; 545 546 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 547 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 548 ndesc = VMXNET3_DEF_RX_NDESC; 549 if (ndesc & VMXNET3_MASK_RX_NDESC) 550 ndesc &= ~VMXNET3_MASK_RX_NDESC; 551 sc->vmx_nrxdescs = ndesc; 552 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 553} 554 555static void 556vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 557{ 558 559 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 560 goto out; 561 562 /* BMV: Just use the maximum configured for now. */ 563 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 564 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 565 566 if (sc->vmx_nrxqueues > 1) 567 sc->vmx_flags |= VMXNET3_FLAG_RSS; 568 569 return; 570 571out: 572 sc->vmx_ntxqueues = 1; 573 sc->vmx_nrxqueues = 1; 574} 575 576static int 577vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 578{ 579 device_t dev; 580 int nmsix, cnt, required; 581 582 dev = sc->vmx_dev; 583 584 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 585 return (1); 586 587 /* Allocate an additional vector for the events interrupt. */ 588 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 589 590 nmsix = pci_msix_count(dev); 591 if (nmsix < required) 592 return (1); 593 594 cnt = required; 595 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 596 sc->vmx_nintrs = required; 597 return (0); 598 } else 599 pci_release_msi(dev); 600 601 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 602 603 return (1); 604} 605 606static int 607vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 608{ 609 device_t dev; 610 int nmsi, cnt, required; 611 612 dev = sc->vmx_dev; 613 required = 1; 614 615 nmsi = pci_msi_count(dev); 616 if (nmsi < required) 617 return (1); 618 619 cnt = required; 620 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 621 sc->vmx_nintrs = 1; 622 return (0); 623 } else 624 pci_release_msi(dev); 625 626 return (1); 627} 628 629static int 630vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 631{ 632 633 sc->vmx_nintrs = 1; 634 return (0); 635} 636 637static int 638vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 639 struct vmxnet3_interrupt *intr) 640{ 641 struct resource *irq; 642 643 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 644 if (irq == NULL) 645 return (ENXIO); 646 647 intr->vmxi_irq = irq; 648 intr->vmxi_rid = rid; 649 650 return (0); 651} 652 653static int 654vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 655{ 656 int i, rid, flags, error; 657 658 rid = 0; 659 flags = RF_ACTIVE; 660 661 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 662 flags |= RF_SHAREABLE; 663 else 664 rid = 1; 665 666 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 667 error = vmxnet3_alloc_interrupt(sc, rid, flags, 668 &sc->vmx_intrs[i]); 669 if (error) 670 return (error); 671 } 672 673 return (0); 674} 675 676static int 677vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 678{ 679 device_t dev; 680 struct vmxnet3_txqueue *txq; 681 struct vmxnet3_rxqueue *rxq; 682 struct vmxnet3_interrupt *intr; 683 enum intr_type type; 684 int i, error; 685 686 dev = sc->vmx_dev; 687 intr = &sc->vmx_intrs[0]; 688 type = INTR_TYPE_NET | INTR_MPSAFE; 689 690 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 691 txq = &sc->vmx_txq[i]; 692 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 693 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 694 if (error) 695 return (error); 696 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 697 } 698 699 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 700 rxq = &sc->vmx_rxq[i]; 701 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 702 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 703 if (error) 704 return (error); 705 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 706 } 707 708 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 709 vmxnet3_event_intr, sc, &intr->vmxi_handler); 710 if (error) 711 return (error); 712 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 713 714 return (0); 715} 716 717static int 718vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 719{ 720 struct vmxnet3_interrupt *intr; 721 int i, error; 722 723 intr = &sc->vmx_intrs[0]; 724 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 725 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 726 &intr->vmxi_handler); 727 728 for (i = 0; i < sc->vmx_ntxqueues; i++) 729 sc->vmx_txq[i].vxtxq_intr_idx = 0; 730 for (i = 0; i < sc->vmx_nrxqueues; i++) 731 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 732 sc->vmx_event_intr_idx = 0; 733 734 return (error); 735} 736 737static void 738vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 739{ 740 struct vmxnet3_txqueue *txq; 741 struct vmxnet3_txq_shared *txs; 742 struct vmxnet3_rxqueue *rxq; 743 struct vmxnet3_rxq_shared *rxs; 744 int i; 745 746 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 747 748 for (i = 0; i < sc->vmx_ntxqueues; i++) { 749 txq = &sc->vmx_txq[i]; 750 txs = txq->vxtxq_ts; 751 txs->intr_idx = txq->vxtxq_intr_idx; 752 } 753 754 for (i = 0; i < sc->vmx_nrxqueues; i++) { 755 rxq = &sc->vmx_rxq[i]; 756 rxs = rxq->vxrxq_rs; 757 rxs->intr_idx = rxq->vxrxq_intr_idx; 758 } 759} 760 761static int 762vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 763{ 764 int error; 765 766 error = vmxnet3_alloc_intr_resources(sc); 767 if (error) 768 return (error); 769 770 switch (sc->vmx_intr_type) { 771 case VMXNET3_IT_MSIX: 772 error = vmxnet3_setup_msix_interrupts(sc); 773 break; 774 case VMXNET3_IT_MSI: 775 case VMXNET3_IT_LEGACY: 776 error = vmxnet3_setup_legacy_interrupt(sc); 777 break; 778 default: 779 panic("%s: invalid interrupt type %d", __func__, 780 sc->vmx_intr_type); 781 } 782 783 if (error == 0) 784 vmxnet3_set_interrupt_idx(sc); 785 786 return (error); 787} 788 789static int 790vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 791{ 792 device_t dev; 793 uint32_t config; 794 int error; 795 796 dev = sc->vmx_dev; 797 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 798 799 sc->vmx_intr_type = config & 0x03; 800 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 801 802 switch (sc->vmx_intr_type) { 803 case VMXNET3_IT_AUTO: 804 sc->vmx_intr_type = VMXNET3_IT_MSIX; 805 /* FALLTHROUGH */ 806 case VMXNET3_IT_MSIX: 807 error = vmxnet3_alloc_msix_interrupts(sc); 808 if (error == 0) 809 break; 810 sc->vmx_intr_type = VMXNET3_IT_MSI; 811 /* FALLTHROUGH */ 812 case VMXNET3_IT_MSI: 813 error = vmxnet3_alloc_msi_interrupts(sc); 814 if (error == 0) 815 break; 816 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 817 /* FALLTHROUGH */ 818 case VMXNET3_IT_LEGACY: 819 error = vmxnet3_alloc_legacy_interrupts(sc); 820 if (error == 0) 821 break; 822 /* FALLTHROUGH */ 823 default: 824 sc->vmx_intr_type = -1; 825 device_printf(dev, "cannot allocate any interrupt resources\n"); 826 return (ENXIO); 827 } 828 829 return (error); 830} 831 832static void 833vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 834 struct vmxnet3_interrupt *intr) 835{ 836 device_t dev; 837 838 dev = sc->vmx_dev; 839 840 if (intr->vmxi_handler != NULL) { 841 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 842 intr->vmxi_handler = NULL; 843 } 844 845 if (intr->vmxi_irq != NULL) { 846 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 847 intr->vmxi_irq); 848 intr->vmxi_irq = NULL; 849 intr->vmxi_rid = -1; 850 } 851} 852 853static void 854vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 855{ 856 int i; 857 858 for (i = 0; i < sc->vmx_nintrs; i++) 859 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 860 861 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 862 sc->vmx_intr_type == VMXNET3_IT_MSIX) 863 pci_release_msi(sc->vmx_dev); 864} 865 866#ifndef VMXNET3_LEGACY_TX 867static int 868vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 869{ 870 device_t dev; 871 872 dev = sc->vmx_dev; 873 874 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 875 taskqueue_thread_enqueue, &sc->vmx_tq); 876 if (sc->vmx_tq == NULL) 877 return (ENOMEM); 878 879 return (0); 880} 881 882static void 883vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 884{ 885 device_t dev; 886 int nthreads, error; 887 888 dev = sc->vmx_dev; 889 890 /* 891 * The taskqueue is typically not frequently used, so a dedicated 892 * thread for each queue is unnecessary. 893 */ 894 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 895 896 /* 897 * Most drivers just ignore the return value - it only fails 898 * with ENOMEM so an error is not likely. It is hard for us 899 * to recover from an error here. 900 */ 901 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 902 "%s taskq", device_get_nameunit(dev)); 903 if (error) 904 device_printf(dev, "failed to start taskqueue: %d", error); 905} 906 907static void 908vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 909{ 910 struct vmxnet3_txqueue *txq; 911 int i; 912 913 if (sc->vmx_tq != NULL) { 914 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 915 txq = &sc->vmx_txq[i]; 916 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 917 } 918 } 919} 920 921static void 922vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 923{ 924 if (sc->vmx_tq != NULL) { 925 taskqueue_free(sc->vmx_tq); 926 sc->vmx_tq = NULL; 927 } 928} 929#endif 930 931static int 932vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 933{ 934 struct vmxnet3_rxqueue *rxq; 935 struct vmxnet3_rxring *rxr; 936 int i; 937 938 rxq = &sc->vmx_rxq[q]; 939 940 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 941 device_get_nameunit(sc->vmx_dev), q); 942 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 943 944 rxq->vxrxq_sc = sc; 945 rxq->vxrxq_id = q; 946 947 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 948 rxr = &rxq->vxrxq_cmd_ring[i]; 949 rxr->vxrxr_rid = i; 950 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 951 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 952 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 953 if (rxr->vxrxr_rxbuf == NULL) 954 return (ENOMEM); 955 956 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 957 } 958 959 return (0); 960} 961 962static int 963vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 964{ 965 struct vmxnet3_txqueue *txq; 966 struct vmxnet3_txring *txr; 967 968 txq = &sc->vmx_txq[q]; 969 txr = &txq->vxtxq_cmd_ring; 970 971 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 972 device_get_nameunit(sc->vmx_dev), q); 973 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 974 975 txq->vxtxq_sc = sc; 976 txq->vxtxq_id = q; 977 978 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 979 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 980 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 981 if (txr->vxtxr_txbuf == NULL) 982 return (ENOMEM); 983 984 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 985 986#ifndef VMXNET3_LEGACY_TX 987 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 988 989 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 990 M_NOWAIT, &txq->vxtxq_mtx); 991 if (txq->vxtxq_br == NULL) 992 return (ENOMEM); 993#endif 994 995 return (0); 996} 997 998static int 999vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1000{ 1001 int i, error; 1002 1003 /* 1004 * Only attempt to create multiple queues if MSIX is available. MSIX is 1005 * disabled by default because its apparently broken for devices passed 1006 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1007 * must be set to zero for MSIX. This check prevents us from allocating 1008 * queue structures that we will not use. 1009 */ 1010 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1011 sc->vmx_max_nrxqueues = 1; 1012 sc->vmx_max_ntxqueues = 1; 1013 } 1014 1015 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1016 sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1017 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1018 sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1019 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1020 return (ENOMEM); 1021 1022 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1023 error = vmxnet3_init_rxq(sc, i); 1024 if (error) 1025 return (error); 1026 } 1027 1028 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1029 error = vmxnet3_init_txq(sc, i); 1030 if (error) 1031 return (error); 1032 } 1033 1034 return (0); 1035} 1036 1037static void 1038vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1039{ 1040 struct vmxnet3_rxring *rxr; 1041 int i; 1042 1043 rxq->vxrxq_sc = NULL; 1044 rxq->vxrxq_id = -1; 1045 1046 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1047 rxr = &rxq->vxrxq_cmd_ring[i]; 1048 1049 if (rxr->vxrxr_rxbuf != NULL) { 1050 free(rxr->vxrxr_rxbuf, M_DEVBUF); 1051 rxr->vxrxr_rxbuf = NULL; 1052 } 1053 } 1054 1055 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1056 mtx_destroy(&rxq->vxrxq_mtx); 1057} 1058 1059static void 1060vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1061{ 1062 struct vmxnet3_txring *txr; 1063 1064 txr = &txq->vxtxq_cmd_ring; 1065 1066 txq->vxtxq_sc = NULL; 1067 txq->vxtxq_id = -1; 1068 1069#ifndef VMXNET3_LEGACY_TX 1070 if (txq->vxtxq_br != NULL) { 1071 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1072 txq->vxtxq_br = NULL; 1073 } 1074#endif 1075 1076 if (txr->vxtxr_txbuf != NULL) { 1077 free(txr->vxtxr_txbuf, M_DEVBUF); 1078 txr->vxtxr_txbuf = NULL; 1079 } 1080 1081 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1082 mtx_destroy(&txq->vxtxq_mtx); 1083} 1084 1085static void 1086vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1087{ 1088 int i; 1089 1090 if (sc->vmx_rxq != NULL) { 1091 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1092 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1093 free(sc->vmx_rxq, M_DEVBUF); 1094 sc->vmx_rxq = NULL; 1095 } 1096 1097 if (sc->vmx_txq != NULL) { 1098 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1099 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1100 free(sc->vmx_txq, M_DEVBUF); 1101 sc->vmx_txq = NULL; 1102 } 1103} 1104 1105static int 1106vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1107{ 1108 device_t dev; 1109 uint8_t *kva; 1110 size_t size; 1111 int i, error; 1112 1113 dev = sc->vmx_dev; 1114 1115 size = sizeof(struct vmxnet3_driver_shared); 1116 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1117 if (error) { 1118 device_printf(dev, "cannot alloc shared memory\n"); 1119 return (error); 1120 } 1121 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1122 1123 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1124 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1125 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1126 if (error) { 1127 device_printf(dev, "cannot alloc queue shared memory\n"); 1128 return (error); 1129 } 1130 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1131 kva = sc->vmx_qs; 1132 1133 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1134 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1135 kva += sizeof(struct vmxnet3_txq_shared); 1136 } 1137 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1138 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1139 kva += sizeof(struct vmxnet3_rxq_shared); 1140 } 1141 1142 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1143 size = sizeof(struct vmxnet3_rss_shared); 1144 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1145 if (error) { 1146 device_printf(dev, "cannot alloc rss shared memory\n"); 1147 return (error); 1148 } 1149 sc->vmx_rss = 1150 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1151 } 1152 1153 return (0); 1154} 1155 1156static void 1157vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1158{ 1159 1160 if (sc->vmx_rss != NULL) { 1161 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1162 sc->vmx_rss = NULL; 1163 } 1164 1165 if (sc->vmx_qs != NULL) { 1166 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1167 sc->vmx_qs = NULL; 1168 } 1169 1170 if (sc->vmx_ds != NULL) { 1171 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1172 sc->vmx_ds = NULL; 1173 } 1174} 1175 1176static int 1177vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1178{ 1179 device_t dev; 1180 struct vmxnet3_txqueue *txq; 1181 struct vmxnet3_txring *txr; 1182 struct vmxnet3_comp_ring *txc; 1183 size_t descsz, compsz; 1184 int i, q, error; 1185 1186 dev = sc->vmx_dev; 1187 1188 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1189 txq = &sc->vmx_txq[q]; 1190 txr = &txq->vxtxq_cmd_ring; 1191 txc = &txq->vxtxq_comp_ring; 1192 1193 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1194 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1195 1196 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1197 1, 0, /* alignment, boundary */ 1198 BUS_SPACE_MAXADDR, /* lowaddr */ 1199 BUS_SPACE_MAXADDR, /* highaddr */ 1200 NULL, NULL, /* filter, filterarg */ 1201 VMXNET3_TX_MAXSIZE, /* maxsize */ 1202 VMXNET3_TX_MAXSEGS, /* nsegments */ 1203 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1204 0, /* flags */ 1205 NULL, NULL, /* lockfunc, lockarg */ 1206 &txr->vxtxr_txtag); 1207 if (error) { 1208 device_printf(dev, 1209 "unable to create Tx buffer tag for queue %d\n", q); 1210 return (error); 1211 } 1212 1213 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1214 if (error) { 1215 device_printf(dev, "cannot alloc Tx descriptors for " 1216 "queue %d error %d\n", q, error); 1217 return (error); 1218 } 1219 txr->vxtxr_txd = 1220 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1221 1222 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1223 if (error) { 1224 device_printf(dev, "cannot alloc Tx comp descriptors " 1225 "for queue %d error %d\n", q, error); 1226 return (error); 1227 } 1228 txc->vxcr_u.txcd = 1229 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1230 1231 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1232 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1233 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1234 if (error) { 1235 device_printf(dev, "unable to create Tx buf " 1236 "dmamap for queue %d idx %d\n", q, i); 1237 return (error); 1238 } 1239 } 1240 } 1241 1242 return (0); 1243} 1244 1245static void 1246vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1247{ 1248 device_t dev; 1249 struct vmxnet3_txqueue *txq; 1250 struct vmxnet3_txring *txr; 1251 struct vmxnet3_comp_ring *txc; 1252 struct vmxnet3_txbuf *txb; 1253 int i, q; 1254 1255 dev = sc->vmx_dev; 1256 1257 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1258 txq = &sc->vmx_txq[q]; 1259 txr = &txq->vxtxq_cmd_ring; 1260 txc = &txq->vxtxq_comp_ring; 1261 1262 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1263 txb = &txr->vxtxr_txbuf[i]; 1264 if (txb->vtxb_dmamap != NULL) { 1265 bus_dmamap_destroy(txr->vxtxr_txtag, 1266 txb->vtxb_dmamap); 1267 txb->vtxb_dmamap = NULL; 1268 } 1269 } 1270 1271 if (txc->vxcr_u.txcd != NULL) { 1272 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1273 txc->vxcr_u.txcd = NULL; 1274 } 1275 1276 if (txr->vxtxr_txd != NULL) { 1277 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1278 txr->vxtxr_txd = NULL; 1279 } 1280 1281 if (txr->vxtxr_txtag != NULL) { 1282 bus_dma_tag_destroy(txr->vxtxr_txtag); 1283 txr->vxtxr_txtag = NULL; 1284 } 1285 } 1286} 1287 1288static int 1289vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1290{ 1291 device_t dev; 1292 struct vmxnet3_rxqueue *rxq; 1293 struct vmxnet3_rxring *rxr; 1294 struct vmxnet3_comp_ring *rxc; 1295 int descsz, compsz; 1296 int i, j, q, error; 1297 1298 dev = sc->vmx_dev; 1299 1300 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1301 rxq = &sc->vmx_rxq[q]; 1302 rxc = &rxq->vxrxq_comp_ring; 1303 compsz = 0; 1304 1305 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1306 rxr = &rxq->vxrxq_cmd_ring[i]; 1307 1308 descsz = rxr->vxrxr_ndesc * 1309 sizeof(struct vmxnet3_rxdesc); 1310 compsz += rxr->vxrxr_ndesc * 1311 sizeof(struct vmxnet3_rxcompdesc); 1312 1313 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1314 1, 0, /* alignment, boundary */ 1315 BUS_SPACE_MAXADDR, /* lowaddr */ 1316 BUS_SPACE_MAXADDR, /* highaddr */ 1317 NULL, NULL, /* filter, filterarg */ 1318 MJUMPAGESIZE, /* maxsize */ 1319 1, /* nsegments */ 1320 MJUMPAGESIZE, /* maxsegsize */ 1321 0, /* flags */ 1322 NULL, NULL, /* lockfunc, lockarg */ 1323 &rxr->vxrxr_rxtag); 1324 if (error) { 1325 device_printf(dev, 1326 "unable to create Rx buffer tag for " 1327 "queue %d\n", q); 1328 return (error); 1329 } 1330 1331 error = vmxnet3_dma_malloc(sc, descsz, 512, 1332 &rxr->vxrxr_dma); 1333 if (error) { 1334 device_printf(dev, "cannot allocate Rx " 1335 "descriptors for queue %d/%d error %d\n", 1336 i, q, error); 1337 return (error); 1338 } 1339 rxr->vxrxr_rxd = 1340 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1341 } 1342 1343 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1344 if (error) { 1345 device_printf(dev, "cannot alloc Rx comp descriptors " 1346 "for queue %d error %d\n", q, error); 1347 return (error); 1348 } 1349 rxc->vxcr_u.rxcd = 1350 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1351 1352 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1353 rxr = &rxq->vxrxq_cmd_ring[i]; 1354 1355 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1356 &rxr->vxrxr_spare_dmap); 1357 if (error) { 1358 device_printf(dev, "unable to create spare " 1359 "dmamap for queue %d/%d error %d\n", 1360 q, i, error); 1361 return (error); 1362 } 1363 1364 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1365 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1366 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1367 if (error) { 1368 device_printf(dev, "unable to create " 1369 "dmamap for queue %d/%d slot %d " 1370 "error %d\n", 1371 q, i, j, error); 1372 return (error); 1373 } 1374 } 1375 } 1376 } 1377 1378 return (0); 1379} 1380 1381static void 1382vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1383{ 1384 device_t dev; 1385 struct vmxnet3_rxqueue *rxq; 1386 struct vmxnet3_rxring *rxr; 1387 struct vmxnet3_comp_ring *rxc; 1388 struct vmxnet3_rxbuf *rxb; 1389 int i, j, q; 1390 1391 dev = sc->vmx_dev; 1392 1393 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1394 rxq = &sc->vmx_rxq[q]; 1395 rxc = &rxq->vxrxq_comp_ring; 1396 1397 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1398 rxr = &rxq->vxrxq_cmd_ring[i]; 1399 1400 if (rxr->vxrxr_spare_dmap != NULL) { 1401 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1402 rxr->vxrxr_spare_dmap); 1403 rxr->vxrxr_spare_dmap = NULL; 1404 } 1405 1406 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1407 rxb = &rxr->vxrxr_rxbuf[j]; 1408 if (rxb->vrxb_dmamap != NULL) { 1409 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1410 rxb->vrxb_dmamap); 1411 rxb->vrxb_dmamap = NULL; 1412 } 1413 } 1414 } 1415 1416 if (rxc->vxcr_u.rxcd != NULL) { 1417 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1418 rxc->vxcr_u.rxcd = NULL; 1419 } 1420 1421 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1422 rxr = &rxq->vxrxq_cmd_ring[i]; 1423 1424 if (rxr->vxrxr_rxd != NULL) { 1425 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1426 rxr->vxrxr_rxd = NULL; 1427 } 1428 1429 if (rxr->vxrxr_rxtag != NULL) { 1430 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1431 rxr->vxrxr_rxtag = NULL; 1432 } 1433 } 1434 } 1435} 1436 1437static int 1438vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1439{ 1440 int error; 1441 1442 error = vmxnet3_alloc_txq_data(sc); 1443 if (error) 1444 return (error); 1445 1446 error = vmxnet3_alloc_rxq_data(sc); 1447 if (error) 1448 return (error); 1449 1450 return (0); 1451} 1452 1453static void 1454vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1455{ 1456 1457 if (sc->vmx_rxq != NULL) 1458 vmxnet3_free_rxq_data(sc); 1459 1460 if (sc->vmx_txq != NULL) 1461 vmxnet3_free_txq_data(sc); 1462} 1463 1464static int 1465vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1466{ 1467 int error; 1468 1469 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1470 32, &sc->vmx_mcast_dma); 1471 if (error) 1472 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1473 else 1474 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1475 1476 return (error); 1477} 1478 1479static void 1480vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1481{ 1482 1483 if (sc->vmx_mcast != NULL) { 1484 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1485 sc->vmx_mcast = NULL; 1486 } 1487} 1488 1489static void 1490vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1491{ 1492 struct vmxnet3_driver_shared *ds; 1493 struct vmxnet3_txqueue *txq; 1494 struct vmxnet3_txq_shared *txs; 1495 struct vmxnet3_rxqueue *rxq; 1496 struct vmxnet3_rxq_shared *rxs; 1497 int i; 1498 1499 ds = sc->vmx_ds; 1500 1501 /* 1502 * Initialize fields of the shared data that remains the same across 1503 * reinits. Note the shared data is zero'd when allocated. 1504 */ 1505 1506 ds->magic = VMXNET3_REV1_MAGIC; 1507 1508 /* DriverInfo */ 1509 ds->version = VMXNET3_DRIVER_VERSION; 1510 ds->guest = VMXNET3_GOS_FREEBSD | 1511#ifdef __LP64__ 1512 VMXNET3_GOS_64BIT; 1513#else 1514 VMXNET3_GOS_32BIT; 1515#endif 1516 ds->vmxnet3_revision = 1; 1517 ds->upt_version = 1; 1518 1519 /* Misc. conf */ 1520 ds->driver_data = vtophys(sc); 1521 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1522 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1523 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1524 ds->nrxsg_max = sc->vmx_max_rxsegs; 1525 1526 /* RSS conf */ 1527 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1528 ds->rss.version = 1; 1529 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1530 ds->rss.len = sc->vmx_rss_dma.dma_size; 1531 } 1532 1533 /* Interrupt control. */ 1534 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1535 ds->nintr = sc->vmx_nintrs; 1536 ds->evintr = sc->vmx_event_intr_idx; 1537 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1538 1539 for (i = 0; i < sc->vmx_nintrs; i++) 1540 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1541 1542 /* Receive filter. */ 1543 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1544 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1545 1546 /* Tx queues */ 1547 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1548 txq = &sc->vmx_txq[i]; 1549 txs = txq->vxtxq_ts; 1550 1551 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1552 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1553 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1554 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1555 txs->driver_data = vtophys(txq); 1556 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1557 } 1558 1559 /* Rx queues */ 1560 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1561 rxq = &sc->vmx_rxq[i]; 1562 rxs = rxq->vxrxq_rs; 1563 1564 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1565 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1566 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1567 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1568 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1569 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1570 rxs->driver_data = vtophys(rxq); 1571 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1572 } 1573} 1574 1575static void 1576vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1577{ 1578 struct ifnet *ifp; 1579 1580 ifp = sc->vmx_ifp; 1581 1582 /* Use the current MAC address. */ 1583 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1584 vmxnet3_set_lladdr(sc); 1585 1586 ifp->if_hwassist = 0; 1587 if (ifp->if_capenable & IFCAP_TXCSUM) 1588 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1589 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1590 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1591 if (ifp->if_capenable & IFCAP_TSO4) 1592 ifp->if_hwassist |= CSUM_IP_TSO; 1593 if (ifp->if_capenable & IFCAP_TSO6) 1594 ifp->if_hwassist |= CSUM_IP6_TSO; 1595} 1596 1597static void 1598vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1599{ 1600 /* 1601 * Use the same key as the Linux driver until FreeBSD can do 1602 * RSS (presumably Toeplitz) in software. 1603 */ 1604 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1605 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1606 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1607 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1608 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1609 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1610 }; 1611 1612 struct vmxnet3_driver_shared *ds; 1613 struct vmxnet3_rss_shared *rss; 1614 int i; 1615 1616 ds = sc->vmx_ds; 1617 rss = sc->vmx_rss; 1618 1619 rss->hash_type = 1620 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1621 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1622 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1623 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1624 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1625 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1626 1627 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1628 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1629} 1630 1631static void 1632vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1633{ 1634 struct ifnet *ifp; 1635 struct vmxnet3_driver_shared *ds; 1636 1637 ifp = sc->vmx_ifp; 1638 ds = sc->vmx_ds; 1639 1640 ds->mtu = ifp->if_mtu; 1641 ds->ntxqueue = sc->vmx_ntxqueues; 1642 ds->nrxqueue = sc->vmx_nrxqueues; 1643 1644 ds->upt_features = 0; 1645 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1646 ds->upt_features |= UPT1_F_CSUM; 1647 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1648 ds->upt_features |= UPT1_F_VLAN; 1649 if (ifp->if_capenable & IFCAP_LRO) 1650 ds->upt_features |= UPT1_F_LRO; 1651 1652 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1653 ds->upt_features |= UPT1_F_RSS; 1654 vmxnet3_reinit_rss_shared_data(sc); 1655 } 1656 1657 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1658 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1659 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1660} 1661 1662static int 1663vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1664{ 1665 int error; 1666 1667 error = vmxnet3_alloc_shared_data(sc); 1668 if (error) 1669 return (error); 1670 1671 error = vmxnet3_alloc_queue_data(sc); 1672 if (error) 1673 return (error); 1674 1675 error = vmxnet3_alloc_mcast_table(sc); 1676 if (error) 1677 return (error); 1678 1679 vmxnet3_init_shared_data(sc); 1680 1681 return (0); 1682} 1683 1684static void 1685vmxnet3_free_data(struct vmxnet3_softc *sc) 1686{ 1687 1688 vmxnet3_free_mcast_table(sc); 1689 vmxnet3_free_queue_data(sc); 1690 vmxnet3_free_shared_data(sc); 1691} 1692 1693static int 1694vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1695{ 1696 device_t dev; 1697 struct ifnet *ifp; 1698 1699 dev = sc->vmx_dev; 1700 1701 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1702 if (ifp == NULL) { 1703 device_printf(dev, "cannot allocate ifnet structure\n"); 1704 return (ENOSPC); 1705 } 1706 1707 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1708#if __FreeBSD_version < 1000025 1709 ifp->if_baudrate = 1000000000; 1710#elif __FreeBSD_version < 1100011 1711 if_initbaudrate(ifp, IF_Gbps(10)); 1712#else 1713 ifp->if_baudrate = IF_Gbps(10); 1714#endif 1715 ifp->if_softc = sc; 1716 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1717 ifp->if_init = vmxnet3_init; 1718 ifp->if_ioctl = vmxnet3_ioctl; 1719 ifp->if_hw_tsomax = VMXNET3_TSO_MAXSIZE; 1720 1721#ifdef VMXNET3_LEGACY_TX 1722 ifp->if_start = vmxnet3_start; 1723 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1724 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1725 IFQ_SET_READY(&ifp->if_snd); 1726#else 1727 ifp->if_transmit = vmxnet3_txq_mq_start; 1728 ifp->if_qflush = vmxnet3_qflush; 1729#endif 1730 1731 vmxnet3_get_lladdr(sc); 1732 ether_ifattach(ifp, sc->vmx_lladdr); 1733 1734 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1735 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1736 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1737 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1738 IFCAP_VLAN_HWCSUM; 1739 ifp->if_capenable = ifp->if_capabilities; 1740 1741 /* These capabilities are not enabled by default. */ 1742 ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1743 1744 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1745 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1746 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1747 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1748 1749 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1750 vmxnet3_media_status); 1751 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1752 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1753 1754 return (0); 1755} 1756 1757static void 1758vmxnet3_evintr(struct vmxnet3_softc *sc) 1759{ 1760 device_t dev; 1761 struct ifnet *ifp; 1762 struct vmxnet3_txq_shared *ts; 1763 struct vmxnet3_rxq_shared *rs; 1764 uint32_t event; 1765 int reset; 1766 1767 dev = sc->vmx_dev; 1768 ifp = sc->vmx_ifp; 1769 reset = 0; 1770 1771 VMXNET3_CORE_LOCK(sc); 1772 1773 /* Clear events. */ 1774 event = sc->vmx_ds->event; 1775 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1776 1777 if (event & VMXNET3_EVENT_LINK) { 1778 vmxnet3_link_status(sc); 1779 if (sc->vmx_link_active != 0) 1780 vmxnet3_tx_start_all(sc); 1781 } 1782 1783 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1784 reset = 1; 1785 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1786 ts = sc->vmx_txq[0].vxtxq_ts; 1787 if (ts->stopped != 0) 1788 device_printf(dev, "Tx queue error %#x\n", ts->error); 1789 rs = sc->vmx_rxq[0].vxrxq_rs; 1790 if (rs->stopped != 0) 1791 device_printf(dev, "Rx queue error %#x\n", rs->error); 1792 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1793 } 1794 1795 if (event & VMXNET3_EVENT_DIC) 1796 device_printf(dev, "device implementation change event\n"); 1797 if (event & VMXNET3_EVENT_DEBUG) 1798 device_printf(dev, "debug event\n"); 1799 1800 if (reset != 0) { 1801 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1802 vmxnet3_init_locked(sc); 1803 } 1804 1805 VMXNET3_CORE_UNLOCK(sc); 1806} 1807 1808static void 1809vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1810{ 1811 struct vmxnet3_softc *sc; 1812 struct ifnet *ifp; 1813 struct vmxnet3_txring *txr; 1814 struct vmxnet3_comp_ring *txc; 1815 struct vmxnet3_txcompdesc *txcd; 1816 struct vmxnet3_txbuf *txb; 1817 struct mbuf *m; 1818 u_int sop; 1819 1820 sc = txq->vxtxq_sc; 1821 ifp = sc->vmx_ifp; 1822 txr = &txq->vxtxq_cmd_ring; 1823 txc = &txq->vxtxq_comp_ring; 1824 1825 VMXNET3_TXQ_LOCK_ASSERT(txq); 1826 1827 for (;;) { 1828 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1829 if (txcd->gen != txc->vxcr_gen) 1830 break; 1831 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1832 1833 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1834 txc->vxcr_next = 0; 1835 txc->vxcr_gen ^= 1; 1836 } 1837 1838 sop = txr->vxtxr_next; 1839 txb = &txr->vxtxr_txbuf[sop]; 1840 1841 if ((m = txb->vtxb_m) != NULL) { 1842 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1843 BUS_DMASYNC_POSTWRITE); 1844 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1845 1846 txq->vxtxq_stats.vmtxs_opackets++; 1847 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1848 if (m->m_flags & M_MCAST) 1849 txq->vxtxq_stats.vmtxs_omcasts++; 1850 1851 m_freem(m); 1852 txb->vtxb_m = NULL; 1853 } 1854 1855 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1856 } 1857 1858 if (txr->vxtxr_head == txr->vxtxr_next) 1859 txq->vxtxq_watchdog = 0; 1860} 1861 1862static int 1863vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1864{ 1865 struct ifnet *ifp; 1866 struct mbuf *m; 1867 struct vmxnet3_rxdesc *rxd; 1868 struct vmxnet3_rxbuf *rxb; 1869 bus_dma_tag_t tag; 1870 bus_dmamap_t dmap; 1871 bus_dma_segment_t segs[1]; 1872 int idx, clsize, btype, flags, nsegs, error; 1873 1874 ifp = sc->vmx_ifp; 1875 tag = rxr->vxrxr_rxtag; 1876 dmap = rxr->vxrxr_spare_dmap; 1877 idx = rxr->vxrxr_fill; 1878 rxd = &rxr->vxrxr_rxd[idx]; 1879 rxb = &rxr->vxrxr_rxbuf[idx]; 1880 1881#ifdef VMXNET3_FAILPOINTS 1882 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1883 if (rxr->vxrxr_rid != 0) 1884 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1885#endif 1886 1887 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1888 flags = M_PKTHDR; 1889 clsize = MCLBYTES; 1890 btype = VMXNET3_BTYPE_HEAD; 1891 } else { 1892#if __FreeBSD_version < 902001 1893 /* 1894 * These mbufs will never be used for the start of a frame. 1895 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1896 * required the mbuf to always be a packet header. Avoid 1897 * unnecessary mbuf initialization in newer versions where 1898 * that is not the case. 1899 */ 1900 flags = M_PKTHDR; 1901#else 1902 flags = 0; 1903#endif 1904 clsize = MJUMPAGESIZE; 1905 btype = VMXNET3_BTYPE_BODY; 1906 } 1907 1908 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1909 if (m == NULL) { 1910 sc->vmx_stats.vmst_mgetcl_failed++; 1911 return (ENOBUFS); 1912 } 1913 1914 if (btype == VMXNET3_BTYPE_HEAD) { 1915 m->m_len = m->m_pkthdr.len = clsize; 1916 m_adj(m, ETHER_ALIGN); 1917 } else 1918 m->m_len = clsize; 1919 1920 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1921 BUS_DMA_NOWAIT); 1922 if (error) { 1923 m_freem(m); 1924 sc->vmx_stats.vmst_mbuf_load_failed++; 1925 return (error); 1926 } 1927 KASSERT(nsegs == 1, 1928 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1929#if __FreeBSD_version < 902001 1930 if (btype == VMXNET3_BTYPE_BODY) 1931 m->m_flags &= ~M_PKTHDR; 1932#endif 1933 1934 if (rxb->vrxb_m != NULL) { 1935 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1936 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1937 } 1938 1939 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1940 rxb->vrxb_dmamap = dmap; 1941 rxb->vrxb_m = m; 1942 1943 rxd->addr = segs[0].ds_addr; 1944 rxd->len = segs[0].ds_len; 1945 rxd->btype = btype; 1946 rxd->gen = rxr->vxrxr_gen; 1947 1948 vmxnet3_rxr_increment_fill(rxr); 1949 return (0); 1950} 1951 1952static void 1953vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1954 struct vmxnet3_rxring *rxr, int idx) 1955{ 1956 struct vmxnet3_rxdesc *rxd; 1957 1958 rxd = &rxr->vxrxr_rxd[idx]; 1959 rxd->gen = rxr->vxrxr_gen; 1960 vmxnet3_rxr_increment_fill(rxr); 1961} 1962 1963static void 1964vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1965{ 1966 struct vmxnet3_softc *sc; 1967 struct vmxnet3_rxring *rxr; 1968 struct vmxnet3_comp_ring *rxc; 1969 struct vmxnet3_rxcompdesc *rxcd; 1970 int idx, eof; 1971 1972 sc = rxq->vxrxq_sc; 1973 rxc = &rxq->vxrxq_comp_ring; 1974 1975 do { 1976 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1977 if (rxcd->gen != rxc->vxcr_gen) 1978 break; /* Not expected. */ 1979 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1980 1981 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1982 rxc->vxcr_next = 0; 1983 rxc->vxcr_gen ^= 1; 1984 } 1985 1986 idx = rxcd->rxd_idx; 1987 eof = rxcd->eop; 1988 if (rxcd->qid < sc->vmx_nrxqueues) 1989 rxr = &rxq->vxrxq_cmd_ring[0]; 1990 else 1991 rxr = &rxq->vxrxq_cmd_ring[1]; 1992 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1993 } while (!eof); 1994} 1995 1996static void 1997vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1998{ 1999 2000 if (rxcd->ipv4) { 2001 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2002 if (rxcd->ipcsum_ok) 2003 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2004 } 2005 2006 if (!rxcd->fragment) { 2007 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2008 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2009 CSUM_PSEUDO_HDR; 2010 m->m_pkthdr.csum_data = 0xFFFF; 2011 } 2012 } 2013} 2014 2015static void 2016vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2017 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2018{ 2019 struct vmxnet3_softc *sc; 2020 struct ifnet *ifp; 2021 2022 sc = rxq->vxrxq_sc; 2023 ifp = sc->vmx_ifp; 2024 2025 if (rxcd->error) { 2026 rxq->vxrxq_stats.vmrxs_ierrors++; 2027 m_freem(m); 2028 return; 2029 } 2030 2031#ifdef notyet 2032 switch (rxcd->rss_type) { 2033 case VMXNET3_RCD_RSS_TYPE_IPV4: 2034 m->m_pkthdr.flowid = rxcd->rss_hash; 2035 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2036 break; 2037 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2038 m->m_pkthdr.flowid = rxcd->rss_hash; 2039 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2040 break; 2041 case VMXNET3_RCD_RSS_TYPE_IPV6: 2042 m->m_pkthdr.flowid = rxcd->rss_hash; 2043 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2044 break; 2045 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2046 m->m_pkthdr.flowid = rxcd->rss_hash; 2047 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2048 break; 2049 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2050 m->m_pkthdr.flowid = rxq->vxrxq_id; 2051 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2052 break; 2053 } 2054#else 2055 m->m_pkthdr.flowid = rxq->vxrxq_id; 2056 m->m_flags |= M_FLOWID; 2057#endif 2058 2059 if (!rxcd->no_csum) 2060 vmxnet3_rx_csum(rxcd, m); 2061 if (rxcd->vlan) { 2062 m->m_flags |= M_VLANTAG; 2063 m->m_pkthdr.ether_vtag = rxcd->vtag; 2064 } 2065 2066 rxq->vxrxq_stats.vmrxs_ipackets++; 2067 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2068 2069 VMXNET3_RXQ_UNLOCK(rxq); 2070 (*ifp->if_input)(ifp, m); 2071 VMXNET3_RXQ_LOCK(rxq); 2072} 2073 2074static void 2075vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2076{ 2077 struct vmxnet3_softc *sc; 2078 struct ifnet *ifp; 2079 struct vmxnet3_rxring *rxr; 2080 struct vmxnet3_comp_ring *rxc; 2081 struct vmxnet3_rxdesc *rxd; 2082 struct vmxnet3_rxcompdesc *rxcd; 2083 struct mbuf *m, *m_head, *m_tail; 2084 int idx, length; 2085 2086 sc = rxq->vxrxq_sc; 2087 ifp = sc->vmx_ifp; 2088 rxc = &rxq->vxrxq_comp_ring; 2089 m_head = m_tail = NULL; 2090 2091 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2092 2093 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2094 return; 2095 2096 for (;;) { 2097 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2098 if (rxcd->gen != rxc->vxcr_gen) 2099 break; 2100 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2101 2102 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2103 rxc->vxcr_next = 0; 2104 rxc->vxcr_gen ^= 1; 2105 } 2106 2107 idx = rxcd->rxd_idx; 2108 length = rxcd->len; 2109 if (rxcd->qid < sc->vmx_nrxqueues) 2110 rxr = &rxq->vxrxq_cmd_ring[0]; 2111 else 2112 rxr = &rxq->vxrxq_cmd_ring[1]; 2113 rxd = &rxr->vxrxr_rxd[idx]; 2114 2115 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2116 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2117 __func__, rxcd->qid, idx)); 2118 2119 /* 2120 * The host may skip descriptors. We detect this when this 2121 * descriptor does not match the previous fill index. Catch 2122 * up with the host now. 2123 */ 2124 if (__predict_false(rxr->vxrxr_fill != idx)) { 2125 while (rxr->vxrxr_fill != idx) { 2126 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2127 rxr->vxrxr_gen; 2128 vmxnet3_rxr_increment_fill(rxr); 2129 } 2130 } 2131 2132 if (rxcd->sop) { 2133 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2134 ("%s: start of frame w/o head buffer", __func__)); 2135 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2136 ("%s: start of frame not in ring 0", __func__)); 2137 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2138 ("%s: start of frame at unexcepted index %d (%d)", 2139 __func__, idx, sc->vmx_rx_max_chain)); 2140 KASSERT(m_head == NULL, 2141 ("%s: duplicate start of frame?", __func__)); 2142 2143 if (length == 0) { 2144 /* Just ignore this descriptor. */ 2145 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2146 goto nextp; 2147 } 2148 2149 if (vmxnet3_newbuf(sc, rxr) != 0) { 2150 rxq->vxrxq_stats.vmrxs_iqdrops++; 2151 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2152 if (!rxcd->eop) 2153 vmxnet3_rxq_discard_chain(rxq); 2154 goto nextp; 2155 } 2156 2157 m->m_pkthdr.rcvif = ifp; 2158 m->m_pkthdr.len = m->m_len = length; 2159 m->m_pkthdr.csum_flags = 0; 2160 m_head = m_tail = m; 2161 2162 } else { 2163 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2164 ("%s: non start of frame w/o body buffer", __func__)); 2165 KASSERT(m_head != NULL, 2166 ("%s: frame not started?", __func__)); 2167 2168 if (vmxnet3_newbuf(sc, rxr) != 0) { 2169 rxq->vxrxq_stats.vmrxs_iqdrops++; 2170 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2171 if (!rxcd->eop) 2172 vmxnet3_rxq_discard_chain(rxq); 2173 m_freem(m_head); 2174 m_head = m_tail = NULL; 2175 goto nextp; 2176 } 2177 2178 m->m_len = length; 2179 m_head->m_pkthdr.len += length; 2180 m_tail->m_next = m; 2181 m_tail = m; 2182 } 2183 2184 if (rxcd->eop) { 2185 vmxnet3_rxq_input(rxq, rxcd, m_head); 2186 m_head = m_tail = NULL; 2187 2188 /* Must recheck after dropping the Rx lock. */ 2189 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2190 break; 2191 } 2192 2193nextp: 2194 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2195 int qid = rxcd->qid; 2196 bus_size_t r; 2197 2198 idx = (idx + 1) % rxr->vxrxr_ndesc; 2199 if (qid >= sc->vmx_nrxqueues) { 2200 qid -= sc->vmx_nrxqueues; 2201 r = VMXNET3_BAR0_RXH2(qid); 2202 } else 2203 r = VMXNET3_BAR0_RXH1(qid); 2204 vmxnet3_write_bar0(sc, r, idx); 2205 } 2206 } 2207} 2208 2209static void 2210vmxnet3_legacy_intr(void *xsc) 2211{ 2212 struct vmxnet3_softc *sc; 2213 struct vmxnet3_rxqueue *rxq; 2214 struct vmxnet3_txqueue *txq; 2215 struct ifnet *ifp; 2216 2217 sc = xsc; 2218 rxq = &sc->vmx_rxq[0]; 2219 txq = &sc->vmx_txq[0]; 2220 ifp = sc->vmx_ifp; 2221 2222 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2223 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2224 return; 2225 } 2226 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2227 vmxnet3_disable_all_intrs(sc); 2228 2229 if (sc->vmx_ds->event != 0) 2230 vmxnet3_evintr(sc); 2231 2232 VMXNET3_RXQ_LOCK(rxq); 2233 vmxnet3_rxq_eof(rxq); 2234 VMXNET3_RXQ_UNLOCK(rxq); 2235 2236 VMXNET3_TXQ_LOCK(txq); 2237 vmxnet3_txq_eof(txq); 2238 vmxnet3_txq_start(txq); 2239 VMXNET3_TXQ_UNLOCK(txq); 2240 2241 vmxnet3_enable_all_intrs(sc); 2242} 2243 2244static void 2245vmxnet3_txq_intr(void *xtxq) 2246{ 2247 struct vmxnet3_softc *sc; 2248 struct vmxnet3_txqueue *txq; 2249 struct ifnet *ifp; 2250 2251 txq = xtxq; 2252 sc = txq->vxtxq_sc; 2253 ifp = sc->vmx_ifp; 2254 2255 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2256 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2257 2258 VMXNET3_TXQ_LOCK(txq); 2259 vmxnet3_txq_eof(txq); 2260 vmxnet3_txq_start(txq); 2261 VMXNET3_TXQ_UNLOCK(txq); 2262 2263 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2264} 2265 2266static void 2267vmxnet3_rxq_intr(void *xrxq) 2268{ 2269 struct vmxnet3_softc *sc; 2270 struct vmxnet3_rxqueue *rxq; 2271 2272 rxq = xrxq; 2273 sc = rxq->vxrxq_sc; 2274 2275 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2276 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2277 2278 VMXNET3_RXQ_LOCK(rxq); 2279 vmxnet3_rxq_eof(rxq); 2280 VMXNET3_RXQ_UNLOCK(rxq); 2281 2282 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2283} 2284 2285static void 2286vmxnet3_event_intr(void *xsc) 2287{ 2288 struct vmxnet3_softc *sc; 2289 2290 sc = xsc; 2291 2292 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2293 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2294 2295 if (sc->vmx_ds->event != 0) 2296 vmxnet3_evintr(sc); 2297 2298 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2299} 2300 2301static void 2302vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2303{ 2304 struct vmxnet3_txring *txr; 2305 struct vmxnet3_txbuf *txb; 2306 int i; 2307 2308 txr = &txq->vxtxq_cmd_ring; 2309 2310 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2311 txb = &txr->vxtxr_txbuf[i]; 2312 2313 if (txb->vtxb_m == NULL) 2314 continue; 2315 2316 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2317 BUS_DMASYNC_POSTWRITE); 2318 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2319 m_freem(txb->vtxb_m); 2320 txb->vtxb_m = NULL; 2321 } 2322} 2323 2324static void 2325vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2326{ 2327 struct vmxnet3_rxring *rxr; 2328 struct vmxnet3_rxbuf *rxb; 2329 int i, j; 2330 2331 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2332 rxr = &rxq->vxrxq_cmd_ring[i]; 2333 2334 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2335 rxb = &rxr->vxrxr_rxbuf[j]; 2336 2337 if (rxb->vrxb_m == NULL) 2338 continue; 2339 2340 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2341 BUS_DMASYNC_POSTREAD); 2342 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2343 m_freem(rxb->vrxb_m); 2344 rxb->vrxb_m = NULL; 2345 } 2346 } 2347} 2348 2349static void 2350vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2351{ 2352 struct vmxnet3_rxqueue *rxq; 2353 struct vmxnet3_txqueue *txq; 2354 int i; 2355 2356 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2357 rxq = &sc->vmx_rxq[i]; 2358 VMXNET3_RXQ_LOCK(rxq); 2359 VMXNET3_RXQ_UNLOCK(rxq); 2360 } 2361 2362 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2363 txq = &sc->vmx_txq[i]; 2364 VMXNET3_TXQ_LOCK(txq); 2365 VMXNET3_TXQ_UNLOCK(txq); 2366 } 2367} 2368 2369static void 2370vmxnet3_stop(struct vmxnet3_softc *sc) 2371{ 2372 struct ifnet *ifp; 2373 int q; 2374 2375 ifp = sc->vmx_ifp; 2376 VMXNET3_CORE_LOCK_ASSERT(sc); 2377 2378 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2379 sc->vmx_link_active = 0; 2380 callout_stop(&sc->vmx_tick); 2381 2382 /* Disable interrupts. */ 2383 vmxnet3_disable_all_intrs(sc); 2384 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2385 2386 vmxnet3_stop_rendezvous(sc); 2387 2388 for (q = 0; q < sc->vmx_ntxqueues; q++) 2389 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2390 for (q = 0; q < sc->vmx_nrxqueues; q++) 2391 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2392 2393 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2394} 2395 2396static void 2397vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2398{ 2399 struct vmxnet3_txring *txr; 2400 struct vmxnet3_comp_ring *txc; 2401 2402 txr = &txq->vxtxq_cmd_ring; 2403 txr->vxtxr_head = 0; 2404 txr->vxtxr_next = 0; 2405 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2406 bzero(txr->vxtxr_txd, 2407 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2408 2409 txc = &txq->vxtxq_comp_ring; 2410 txc->vxcr_next = 0; 2411 txc->vxcr_gen = VMXNET3_INIT_GEN; 2412 bzero(txc->vxcr_u.txcd, 2413 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2414} 2415 2416static int 2417vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2418{ 2419 struct ifnet *ifp; 2420 struct vmxnet3_rxring *rxr; 2421 struct vmxnet3_comp_ring *rxc; 2422 int i, populate, idx, frame_size, error; 2423 2424 ifp = sc->vmx_ifp; 2425 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2426 ifp->if_mtu; 2427 2428 /* 2429 * If the MTU causes us to exceed what a regular sized cluster can 2430 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2431 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2432 * 2433 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2434 * our life easier. We do not support changing the ring size after 2435 * the attach. 2436 */ 2437 if (frame_size <= MCLBYTES) 2438 sc->vmx_rx_max_chain = 1; 2439 else 2440 sc->vmx_rx_max_chain = 2; 2441 2442 /* 2443 * Only populate ring 1 if the configuration will take advantage 2444 * of it. That is either when LRO is enabled or the frame size 2445 * exceeds what ring 0 can contain. 2446 */ 2447 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2448 frame_size <= MCLBYTES + MJUMPAGESIZE) 2449 populate = 1; 2450 else 2451 populate = VMXNET3_RXRINGS_PERQ; 2452 2453 for (i = 0; i < populate; i++) { 2454 rxr = &rxq->vxrxq_cmd_ring[i]; 2455 rxr->vxrxr_fill = 0; 2456 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2457 bzero(rxr->vxrxr_rxd, 2458 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2459 2460 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2461 error = vmxnet3_newbuf(sc, rxr); 2462 if (error) 2463 return (error); 2464 } 2465 } 2466 2467 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2468 rxr = &rxq->vxrxq_cmd_ring[i]; 2469 rxr->vxrxr_fill = 0; 2470 rxr->vxrxr_gen = 0; 2471 bzero(rxr->vxrxr_rxd, 2472 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2473 } 2474 2475 rxc = &rxq->vxrxq_comp_ring; 2476 rxc->vxcr_next = 0; 2477 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2478 bzero(rxc->vxcr_u.rxcd, 2479 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2480 2481 return (0); 2482} 2483 2484static int 2485vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2486{ 2487 device_t dev; 2488 int q, error; 2489 2490 dev = sc->vmx_dev; 2491 2492 for (q = 0; q < sc->vmx_ntxqueues; q++) 2493 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2494 2495 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2496 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2497 if (error) { 2498 device_printf(dev, "cannot populate Rx queue %d\n", q); 2499 return (error); 2500 } 2501 } 2502 2503 return (0); 2504} 2505 2506static int 2507vmxnet3_enable_device(struct vmxnet3_softc *sc) 2508{ 2509 int q; 2510 2511 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2512 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2513 return (1); 2514 } 2515 2516 /* Reset the Rx queue heads. */ 2517 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2518 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2519 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2520 } 2521 2522 return (0); 2523} 2524 2525static void 2526vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2527{ 2528 struct ifnet *ifp; 2529 2530 ifp = sc->vmx_ifp; 2531 2532 vmxnet3_set_rxfilter(sc); 2533 2534 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2535 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2536 sizeof(sc->vmx_ds->vlan_filter)); 2537 else 2538 bzero(sc->vmx_ds->vlan_filter, 2539 sizeof(sc->vmx_ds->vlan_filter)); 2540 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2541} 2542 2543static int 2544vmxnet3_reinit(struct vmxnet3_softc *sc) 2545{ 2546 2547 vmxnet3_reinit_interface(sc); 2548 vmxnet3_reinit_shared_data(sc); 2549 2550 if (vmxnet3_reinit_queues(sc) != 0) 2551 return (ENXIO); 2552 2553 if (vmxnet3_enable_device(sc) != 0) 2554 return (ENXIO); 2555 2556 vmxnet3_reinit_rxfilters(sc); 2557 2558 return (0); 2559} 2560 2561static void 2562vmxnet3_init_locked(struct vmxnet3_softc *sc) 2563{ 2564 struct ifnet *ifp; 2565 2566 ifp = sc->vmx_ifp; 2567 2568 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2569 return; 2570 2571 vmxnet3_stop(sc); 2572 2573 if (vmxnet3_reinit(sc) != 0) { 2574 vmxnet3_stop(sc); 2575 return; 2576 } 2577 2578 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2579 vmxnet3_link_status(sc); 2580 2581 vmxnet3_enable_all_intrs(sc); 2582 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2583} 2584 2585static void 2586vmxnet3_init(void *xsc) 2587{ 2588 struct vmxnet3_softc *sc; 2589 2590 sc = xsc; 2591 2592 VMXNET3_CORE_LOCK(sc); 2593 vmxnet3_init_locked(sc); 2594 VMXNET3_CORE_UNLOCK(sc); 2595} 2596 2597/* 2598 * BMV: Much of this can go away once we finally have offsets in 2599 * the mbuf packet header. Bug andre@. 2600 */ 2601static int 2602vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2603 int *etype, int *proto, int *start) 2604{ 2605 struct ether_vlan_header *evh; 2606 int offset; 2607 2608 evh = mtod(m, struct ether_vlan_header *); 2609 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2610 /* BMV: We should handle nested VLAN tags too. */ 2611 *etype = ntohs(evh->evl_proto); 2612 offset = sizeof(struct ether_vlan_header); 2613 } else { 2614 *etype = ntohs(evh->evl_encap_proto); 2615 offset = sizeof(struct ether_header); 2616 } 2617 2618 switch (*etype) { 2619#if defined(INET) 2620 case ETHERTYPE_IP: { 2621 struct ip *ip, iphdr; 2622 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2623 m_copydata(m, offset, sizeof(struct ip), 2624 (caddr_t) &iphdr); 2625 ip = &iphdr; 2626 } else 2627 ip = mtodo(m, offset); 2628 *proto = ip->ip_p; 2629 *start = offset + (ip->ip_hl << 2); 2630 break; 2631 } 2632#endif 2633#if defined(INET6) 2634 case ETHERTYPE_IPV6: 2635 *proto = -1; 2636 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2637 /* Assert the network stack sent us a valid packet. */ 2638 KASSERT(*start > offset, 2639 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2640 *start, offset, *proto)); 2641 break; 2642#endif 2643 default: 2644 return (EINVAL); 2645 } 2646 2647 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2648 struct tcphdr *tcp, tcphdr; 2649 2650 if (__predict_false(*proto != IPPROTO_TCP)) { 2651 /* Likely failed to correctly parse the mbuf. */ 2652 return (EINVAL); 2653 } 2654 2655 txq->vxtxq_stats.vmtxs_tso++; 2656 2657 /* 2658 * For TSO, the size of the protocol header is also 2659 * included in the descriptor header size. 2660 */ 2661 if (m->m_len < *start + sizeof(struct tcphdr)) { 2662 m_copydata(m, offset, sizeof(struct tcphdr), 2663 (caddr_t) &tcphdr); 2664 tcp = &tcphdr; 2665 } else 2666 tcp = mtodo(m, *start); 2667 *start += (tcp->th_off << 2); 2668 } else 2669 txq->vxtxq_stats.vmtxs_csum++; 2670 2671 return (0); 2672} 2673 2674static int 2675vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2676 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2677{ 2678 struct vmxnet3_txring *txr; 2679 struct mbuf *m; 2680 bus_dma_tag_t tag; 2681 int error; 2682 2683 txr = &txq->vxtxq_cmd_ring; 2684 m = *m0; 2685 tag = txr->vxtxr_txtag; 2686 2687 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2688 if (error == 0 || error != EFBIG) 2689 return (error); 2690 2691 m = m_defrag(m, M_NOWAIT); 2692 if (m != NULL) { 2693 *m0 = m; 2694 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2695 } else 2696 error = ENOBUFS; 2697 2698 if (error) { 2699 m_freem(*m0); 2700 *m0 = NULL; 2701 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2702 } else 2703 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2704 2705 return (error); 2706} 2707 2708static void 2709vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2710{ 2711 struct vmxnet3_txring *txr; 2712 2713 txr = &txq->vxtxq_cmd_ring; 2714 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2715} 2716 2717static int 2718vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2719{ 2720 struct vmxnet3_softc *sc; 2721 struct ifnet *ifp; 2722 struct vmxnet3_txring *txr; 2723 struct vmxnet3_txdesc *txd, *sop; 2724 struct mbuf *m; 2725 bus_dmamap_t dmap; 2726 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2727 int i, gen, nsegs, etype, proto, start, error; 2728 2729 sc = txq->vxtxq_sc; 2730 ifp = sc->vmx_ifp; 2731 start = 0; 2732 txd = NULL; 2733 txr = &txq->vxtxq_cmd_ring; 2734 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2735 2736 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2737 if (error) 2738 return (error); 2739 2740 m = *m0; 2741 M_ASSERTPKTHDR(m); 2742 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2743 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2744 2745 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2746 txq->vxtxq_stats.vmtxs_full++; 2747 vmxnet3_txq_unload_mbuf(txq, dmap); 2748 return (ENOSPC); 2749 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2750 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2751 if (error) { 2752 txq->vxtxq_stats.vmtxs_offload_failed++; 2753 vmxnet3_txq_unload_mbuf(txq, dmap); 2754 m_freem(m); 2755 *m0 = NULL; 2756 return (error); 2757 } 2758 } 2759 2760 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m = *m0; 2761 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2762 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2763 2764 for (i = 0; i < nsegs; i++) { 2765 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2766 2767 txd->addr = segs[i].ds_addr; 2768 txd->len = segs[i].ds_len; 2769 txd->gen = gen; 2770 txd->dtype = 0; 2771 txd->offload_mode = VMXNET3_OM_NONE; 2772 txd->offload_pos = 0; 2773 txd->hlen = 0; 2774 txd->eop = 0; 2775 txd->compreq = 0; 2776 txd->vtag_mode = 0; 2777 txd->vtag = 0; 2778 2779 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2780 txr->vxtxr_head = 0; 2781 txr->vxtxr_gen ^= 1; 2782 } 2783 gen = txr->vxtxr_gen; 2784 } 2785 txd->eop = 1; 2786 txd->compreq = 1; 2787 2788 if (m->m_flags & M_VLANTAG) { 2789 sop->vtag_mode = 1; 2790 sop->vtag = m->m_pkthdr.ether_vtag; 2791 } 2792 2793 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2794 sop->offload_mode = VMXNET3_OM_TSO; 2795 sop->hlen = start; 2796 sop->offload_pos = m->m_pkthdr.tso_segsz; 2797 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2798 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2799 sop->offload_mode = VMXNET3_OM_CSUM; 2800 sop->hlen = start; 2801 sop->offload_pos = start + m->m_pkthdr.csum_data; 2802 } 2803 2804 /* Finally, change the ownership. */ 2805 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2806 sop->gen ^= 1; 2807 2808 if (++txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2809 txq->vxtxq_ts->npending = 0; 2810 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2811 txr->vxtxr_head); 2812 } 2813 2814 return (0); 2815} 2816 2817static void 2818vmxnet3_txq_update_pending(struct vmxnet3_txqueue *txq) 2819{ 2820 struct vmxnet3_txring *txr; 2821 2822 txr = &txq->vxtxq_cmd_ring; 2823 2824 if (txq->vxtxq_ts->npending > 0) { 2825 txq->vxtxq_ts->npending = 0; 2826 vmxnet3_write_bar0(txq->vxtxq_sc, 2827 VMXNET3_BAR0_TXH(txq->vxtxq_id), txr->vxtxr_head); 2828 } 2829} 2830 2831#ifdef VMXNET3_LEGACY_TX 2832 2833static void 2834vmxnet3_start_locked(struct ifnet *ifp) 2835{ 2836 struct vmxnet3_softc *sc; 2837 struct vmxnet3_txqueue *txq; 2838 struct vmxnet3_txring *txr; 2839 struct mbuf *m_head; 2840 int tx, avail; 2841 2842 sc = ifp->if_softc; 2843 txq = &sc->vmx_txq[0]; 2844 txr = &txq->vxtxq_cmd_ring; 2845 tx = 0; 2846 2847 VMXNET3_TXQ_LOCK_ASSERT(txq); 2848 2849 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2850 sc->vmx_link_active == 0) 2851 return; 2852 2853 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2854 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2855 break; 2856 2857 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2858 if (m_head == NULL) 2859 break; 2860 2861 /* Assume worse case if this mbuf is the head of a chain. */ 2862 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2863 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2864 break; 2865 } 2866 2867 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2868 if (m_head != NULL) 2869 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2870 break; 2871 } 2872 2873 tx++; 2874 ETHER_BPF_MTAP(ifp, m_head); 2875 } 2876 2877 if (tx > 0) { 2878 vmxnet3_txq_update_pending(txq); 2879 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2880 } 2881} 2882 2883static void 2884vmxnet3_start(struct ifnet *ifp) 2885{ 2886 struct vmxnet3_softc *sc; 2887 struct vmxnet3_txqueue *txq; 2888 2889 sc = ifp->if_softc; 2890 txq = &sc->vmx_txq[0]; 2891 2892 VMXNET3_TXQ_LOCK(txq); 2893 vmxnet3_start_locked(ifp); 2894 VMXNET3_TXQ_UNLOCK(txq); 2895} 2896 2897#else /* !VMXNET3_LEGACY_TX */ 2898 2899static int 2900vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2901{ 2902 struct vmxnet3_softc *sc; 2903 struct vmxnet3_txring *txr; 2904 struct buf_ring *br; 2905 struct ifnet *ifp; 2906 int tx, avail, error; 2907 2908 sc = txq->vxtxq_sc; 2909 br = txq->vxtxq_br; 2910 ifp = sc->vmx_ifp; 2911 txr = &txq->vxtxq_cmd_ring; 2912 tx = 0; 2913 error = 0; 2914 2915 VMXNET3_TXQ_LOCK_ASSERT(txq); 2916 2917 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2918 sc->vmx_link_active == 0) { 2919 if (m != NULL) 2920 error = drbr_enqueue(ifp, br, m); 2921 return (error); 2922 } 2923 2924 if (m != NULL) { 2925 error = drbr_enqueue(ifp, br, m); 2926 if (error) 2927 return (error); 2928 } 2929 2930 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2931 m = drbr_peek(ifp, br); 2932 if (m == NULL) 2933 break; 2934 2935 /* Assume worse case if this mbuf is the head of a chain. */ 2936 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2937 drbr_putback(ifp, br, m); 2938 break; 2939 } 2940 2941 error = vmxnet3_txq_encap(txq, &m); 2942 if (error) { 2943 if (m != NULL) 2944 drbr_putback(ifp, br, m); 2945 else 2946 drbr_advance(ifp, br); 2947 break; 2948 } 2949 drbr_advance(ifp, br); 2950 2951 tx++; 2952 ETHER_BPF_MTAP(ifp, m); 2953 } 2954 2955 if (tx > 0) { 2956 vmxnet3_txq_update_pending(txq); 2957 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2958 } 2959 2960 return (0); 2961} 2962 2963static int 2964vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2965{ 2966 struct vmxnet3_softc *sc; 2967 struct vmxnet3_txqueue *txq; 2968 int i, ntxq, error; 2969 2970 sc = ifp->if_softc; 2971 ntxq = sc->vmx_ntxqueues; 2972 2973 if (m->m_flags & M_FLOWID) 2974 i = m->m_pkthdr.flowid % ntxq; 2975 else 2976 i = curcpu % ntxq; 2977 2978 txq = &sc->vmx_txq[i]; 2979 2980 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 2981 error = vmxnet3_txq_mq_start_locked(txq, m); 2982 VMXNET3_TXQ_UNLOCK(txq); 2983 } else { 2984 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 2985 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 2986 } 2987 2988 return (error); 2989} 2990 2991static void 2992vmxnet3_txq_tq_deferred(void *xtxq, int pending) 2993{ 2994 struct vmxnet3_softc *sc; 2995 struct vmxnet3_txqueue *txq; 2996 2997 txq = xtxq; 2998 sc = txq->vxtxq_sc; 2999 3000 VMXNET3_TXQ_LOCK(txq); 3001 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3002 vmxnet3_txq_mq_start_locked(txq, NULL); 3003 VMXNET3_TXQ_UNLOCK(txq); 3004} 3005 3006#endif /* VMXNET3_LEGACY_TX */ 3007 3008static void 3009vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3010{ 3011 struct vmxnet3_softc *sc; 3012 struct ifnet *ifp; 3013 3014 sc = txq->vxtxq_sc; 3015 ifp = sc->vmx_ifp; 3016 3017#ifdef VMXNET3_LEGACY_TX 3018 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3019 vmxnet3_start_locked(ifp); 3020#else 3021 if (!drbr_empty(ifp, txq->vxtxq_br)) 3022 vmxnet3_txq_mq_start_locked(txq, NULL); 3023#endif 3024} 3025 3026static void 3027vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3028{ 3029 struct vmxnet3_txqueue *txq; 3030 int i; 3031 3032 VMXNET3_CORE_LOCK_ASSERT(sc); 3033 3034 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3035 txq = &sc->vmx_txq[i]; 3036 3037 VMXNET3_TXQ_LOCK(txq); 3038 vmxnet3_txq_start(txq); 3039 VMXNET3_TXQ_UNLOCK(txq); 3040 } 3041} 3042 3043static void 3044vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3045{ 3046 struct ifnet *ifp; 3047 int idx, bit; 3048 3049 ifp = sc->vmx_ifp; 3050 idx = (tag >> 5) & 0x7F; 3051 bit = tag & 0x1F; 3052 3053 if (tag == 0 || tag > 4095) 3054 return; 3055 3056 VMXNET3_CORE_LOCK(sc); 3057 3058 /* Update our private VLAN bitvector. */ 3059 if (add) 3060 sc->vmx_vlan_filter[idx] |= (1 << bit); 3061 else 3062 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3063 3064 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3065 if (add) 3066 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3067 else 3068 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3069 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3070 } 3071 3072 VMXNET3_CORE_UNLOCK(sc); 3073} 3074 3075static void 3076vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3077{ 3078 3079 if (ifp->if_softc == arg) 3080 vmxnet3_update_vlan_filter(arg, 1, tag); 3081} 3082 3083static void 3084vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3085{ 3086 3087 if (ifp->if_softc == arg) 3088 vmxnet3_update_vlan_filter(arg, 0, tag); 3089} 3090 3091static void 3092vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3093{ 3094 struct ifnet *ifp; 3095 struct vmxnet3_driver_shared *ds; 3096 struct ifmultiaddr *ifma; 3097 u_int mode; 3098 3099 ifp = sc->vmx_ifp; 3100 ds = sc->vmx_ds; 3101 3102 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3103 if (ifp->if_flags & IFF_PROMISC) 3104 mode |= VMXNET3_RXMODE_PROMISC; 3105 if (ifp->if_flags & IFF_ALLMULTI) 3106 mode |= VMXNET3_RXMODE_ALLMULTI; 3107 else { 3108 int cnt = 0, overflow = 0; 3109 3110 if_maddr_rlock(ifp); 3111 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3112 if (ifma->ifma_addr->sa_family != AF_LINK) 3113 continue; 3114 else if (cnt == VMXNET3_MULTICAST_MAX) { 3115 overflow = 1; 3116 break; 3117 } 3118 3119 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3120 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3121 cnt++; 3122 } 3123 if_maddr_runlock(ifp); 3124 3125 if (overflow != 0) { 3126 cnt = 0; 3127 mode |= VMXNET3_RXMODE_ALLMULTI; 3128 } else if (cnt > 0) 3129 mode |= VMXNET3_RXMODE_MCAST; 3130 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3131 } 3132 3133 ds->rxmode = mode; 3134 3135 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3136 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3137} 3138 3139static int 3140vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3141{ 3142 struct ifnet *ifp; 3143 3144 ifp = sc->vmx_ifp; 3145 3146 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3147 return (EINVAL); 3148 3149 ifp->if_mtu = mtu; 3150 3151 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3152 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3153 vmxnet3_init_locked(sc); 3154 } 3155 3156 return (0); 3157} 3158 3159static int 3160vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3161{ 3162 struct vmxnet3_softc *sc; 3163 struct ifreq *ifr; 3164 int reinit, mask, error; 3165 3166 sc = ifp->if_softc; 3167 ifr = (struct ifreq *) data; 3168 error = 0; 3169 3170 switch (cmd) { 3171 case SIOCSIFMTU: 3172 if (ifp->if_mtu != ifr->ifr_mtu) { 3173 VMXNET3_CORE_LOCK(sc); 3174 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3175 VMXNET3_CORE_UNLOCK(sc); 3176 } 3177 break; 3178 3179 case SIOCSIFFLAGS: 3180 VMXNET3_CORE_LOCK(sc); 3181 if (ifp->if_flags & IFF_UP) { 3182 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3183 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3184 (IFF_PROMISC | IFF_ALLMULTI)) { 3185 vmxnet3_set_rxfilter(sc); 3186 } 3187 } else 3188 vmxnet3_init_locked(sc); 3189 } else { 3190 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3191 vmxnet3_stop(sc); 3192 } 3193 sc->vmx_if_flags = ifp->if_flags; 3194 VMXNET3_CORE_UNLOCK(sc); 3195 break; 3196 3197 case SIOCADDMULTI: 3198 case SIOCDELMULTI: 3199 VMXNET3_CORE_LOCK(sc); 3200 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3201 vmxnet3_set_rxfilter(sc); 3202 VMXNET3_CORE_UNLOCK(sc); 3203 break; 3204 3205 case SIOCSIFMEDIA: 3206 case SIOCGIFMEDIA: 3207 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3208 break; 3209 3210 case SIOCSIFCAP: 3211 VMXNET3_CORE_LOCK(sc); 3212 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3213 3214 if (mask & IFCAP_TXCSUM) 3215 ifp->if_capenable ^= IFCAP_TXCSUM; 3216 if (mask & IFCAP_TXCSUM_IPV6) 3217 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3218 if (mask & IFCAP_TSO4) 3219 ifp->if_capenable ^= IFCAP_TSO4; 3220 if (mask & IFCAP_TSO6) 3221 ifp->if_capenable ^= IFCAP_TSO6; 3222 3223 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3224 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3225 /* Changing these features requires us to reinit. */ 3226 reinit = 1; 3227 3228 if (mask & IFCAP_RXCSUM) 3229 ifp->if_capenable ^= IFCAP_RXCSUM; 3230 if (mask & IFCAP_RXCSUM_IPV6) 3231 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3232 if (mask & IFCAP_LRO) 3233 ifp->if_capenable ^= IFCAP_LRO; 3234 if (mask & IFCAP_VLAN_HWTAGGING) 3235 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3236 if (mask & IFCAP_VLAN_HWFILTER) 3237 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3238 } else 3239 reinit = 0; 3240 3241 if (mask & IFCAP_VLAN_HWTSO) 3242 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3243 3244 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3245 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3246 vmxnet3_init_locked(sc); 3247 } 3248 3249 VMXNET3_CORE_UNLOCK(sc); 3250 VLAN_CAPABILITIES(ifp); 3251 break; 3252 3253 default: 3254 error = ether_ioctl(ifp, cmd, data); 3255 break; 3256 } 3257 3258 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3259 3260 return (error); 3261} 3262 3263#ifndef VMXNET3_LEGACY_TX 3264static void 3265vmxnet3_qflush(struct ifnet *ifp) 3266{ 3267 struct vmxnet3_softc *sc; 3268 struct vmxnet3_txqueue *txq; 3269 struct mbuf *m; 3270 int i; 3271 3272 sc = ifp->if_softc; 3273 3274 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3275 txq = &sc->vmx_txq[i]; 3276 3277 VMXNET3_TXQ_LOCK(txq); 3278 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3279 m_freem(m); 3280 VMXNET3_TXQ_UNLOCK(txq); 3281 } 3282 3283 if_qflush(ifp); 3284} 3285#endif 3286 3287static int 3288vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3289{ 3290 struct vmxnet3_softc *sc; 3291 3292 sc = txq->vxtxq_sc; 3293 3294 VMXNET3_TXQ_LOCK(txq); 3295 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3296 VMXNET3_TXQ_UNLOCK(txq); 3297 return (0); 3298 } 3299 VMXNET3_TXQ_UNLOCK(txq); 3300 3301 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3302 txq->vxtxq_id); 3303 return (1); 3304} 3305 3306static void 3307vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3308{ 3309 3310 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3311} 3312 3313static void 3314vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq, 3315 struct vmxnet3_txq_stats *accum) 3316{ 3317 struct vmxnet3_txq_stats *st; 3318 3319 st = &txq->vxtxq_stats; 3320 3321 accum->vmtxs_opackets += st->vmtxs_opackets; 3322 accum->vmtxs_obytes += st->vmtxs_obytes; 3323 accum->vmtxs_omcasts += st->vmtxs_omcasts; 3324 accum->vmtxs_csum += st->vmtxs_csum; 3325 accum->vmtxs_tso += st->vmtxs_tso; 3326 accum->vmtxs_full += st->vmtxs_full; 3327 accum->vmtxs_offload_failed += st->vmtxs_offload_failed; 3328} 3329 3330static void 3331vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq, 3332 struct vmxnet3_rxq_stats *accum) 3333{ 3334 struct vmxnet3_rxq_stats *st; 3335 3336 st = &rxq->vxrxq_stats; 3337 3338 accum->vmrxs_ipackets += st->vmrxs_ipackets; 3339 accum->vmrxs_ibytes += st->vmrxs_ibytes; 3340 accum->vmrxs_iqdrops += st->vmrxs_iqdrops; 3341 accum->vmrxs_ierrors += st->vmrxs_ierrors; 3342} 3343 3344static void 3345vmxnet3_accumulate_stats(struct vmxnet3_softc *sc) 3346{ 3347 struct ifnet *ifp; 3348 struct vmxnet3_statistics *st; 3349 struct vmxnet3_txq_stats txaccum; 3350 struct vmxnet3_rxq_stats rxaccum; 3351 int i; 3352 3353 ifp = sc->vmx_ifp; 3354 st = &sc->vmx_stats; 3355 3356 bzero(&txaccum, sizeof(struct vmxnet3_txq_stats)); 3357 bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats)); 3358 3359 for (i = 0; i < sc->vmx_ntxqueues; i++) 3360 vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum); 3361 for (i = 0; i < sc->vmx_nrxqueues; i++) 3362 vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum); 3363 3364 /* 3365 * With the exception of if_ierrors, these ifnet statistics are 3366 * only updated in the driver, so just set them to our accumulated 3367 * values. if_ierrors is updated in ether_input() for malformed 3368 * frames that we should have already discarded. 3369 */ 3370 ifp->if_ipackets = rxaccum.vmrxs_ipackets; 3371 ifp->if_iqdrops = rxaccum.vmrxs_iqdrops; 3372 ifp->if_ierrors = rxaccum.vmrxs_ierrors; 3373 ifp->if_opackets = txaccum.vmtxs_opackets; 3374#ifndef VMXNET3_LEGACY_TX 3375 ifp->if_obytes = txaccum.vmtxs_obytes; 3376 ifp->if_omcasts = txaccum.vmtxs_omcasts; 3377#endif 3378} 3379 3380static void 3381vmxnet3_tick(void *xsc) 3382{ 3383 struct vmxnet3_softc *sc; 3384 struct ifnet *ifp; 3385 int i, timedout; 3386 3387 sc = xsc; 3388 ifp = sc->vmx_ifp; 3389 timedout = 0; 3390 3391 VMXNET3_CORE_LOCK_ASSERT(sc); 3392 3393 vmxnet3_accumulate_stats(sc); 3394 vmxnet3_refresh_host_stats(sc); 3395 3396 for (i = 0; i < sc->vmx_ntxqueues; i++) 3397 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3398 3399 if (timedout != 0) { 3400 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3401 vmxnet3_init_locked(sc); 3402 } else 3403 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3404} 3405 3406static int 3407vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3408{ 3409 uint32_t status; 3410 3411 /* Also update the link speed while here. */ 3412 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3413 sc->vmx_link_speed = status >> 16; 3414 return !!(status & 0x1); 3415} 3416 3417static void 3418vmxnet3_link_status(struct vmxnet3_softc *sc) 3419{ 3420 struct ifnet *ifp; 3421 int link; 3422 3423 ifp = sc->vmx_ifp; 3424 link = vmxnet3_link_is_up(sc); 3425 3426 if (link != 0 && sc->vmx_link_active == 0) { 3427 sc->vmx_link_active = 1; 3428 if_link_state_change(ifp, LINK_STATE_UP); 3429 } else if (link == 0 && sc->vmx_link_active != 0) { 3430 sc->vmx_link_active = 0; 3431 if_link_state_change(ifp, LINK_STATE_DOWN); 3432 } 3433} 3434 3435static void 3436vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3437{ 3438 struct vmxnet3_softc *sc; 3439 3440 sc = ifp->if_softc; 3441 3442 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3443 ifmr->ifm_status = IFM_AVALID; 3444 3445 VMXNET3_CORE_LOCK(sc); 3446 if (vmxnet3_link_is_up(sc) != 0) 3447 ifmr->ifm_status |= IFM_ACTIVE; 3448 else 3449 ifmr->ifm_status |= IFM_NONE; 3450 VMXNET3_CORE_UNLOCK(sc); 3451} 3452 3453static int 3454vmxnet3_media_change(struct ifnet *ifp) 3455{ 3456 3457 /* Ignore. */ 3458 return (0); 3459} 3460 3461static void 3462vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3463{ 3464 uint32_t ml, mh; 3465 3466 ml = sc->vmx_lladdr[0]; 3467 ml |= sc->vmx_lladdr[1] << 8; 3468 ml |= sc->vmx_lladdr[2] << 16; 3469 ml |= sc->vmx_lladdr[3] << 24; 3470 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3471 3472 mh = sc->vmx_lladdr[4]; 3473 mh |= sc->vmx_lladdr[5] << 8; 3474 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3475} 3476 3477static void 3478vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3479{ 3480 uint32_t ml, mh; 3481 3482 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3483 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3484 3485 sc->vmx_lladdr[0] = ml; 3486 sc->vmx_lladdr[1] = ml >> 8; 3487 sc->vmx_lladdr[2] = ml >> 16; 3488 sc->vmx_lladdr[3] = ml >> 24; 3489 sc->vmx_lladdr[4] = mh; 3490 sc->vmx_lladdr[5] = mh >> 8; 3491} 3492 3493static void 3494vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3495 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3496{ 3497 struct sysctl_oid *node, *txsnode; 3498 struct sysctl_oid_list *list, *txslist; 3499 struct vmxnet3_txq_stats *stats; 3500 struct UPT1_TxStats *txstats; 3501 char namebuf[16]; 3502 3503 stats = &txq->vxtxq_stats; 3504 txstats = &txq->vxtxq_ts->stats; 3505 3506 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3507 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3508 NULL, "Transmit Queue"); 3509 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3510 3511 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3512 &stats->vmtxs_opackets, "Transmit packets"); 3513 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3514 &stats->vmtxs_obytes, "Transmit bytes"); 3515 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3516 &stats->vmtxs_omcasts, "Transmit multicasts"); 3517 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3518 &stats->vmtxs_csum, "Transmit checksum offloaded"); 3519 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3520 &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3521 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3522 &stats->vmtxs_full, "Transmit ring full"); 3523 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3524 &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3525 3526 /* 3527 * Add statistics reported by the host. These are updated once 3528 * per second. 3529 */ 3530 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3531 NULL, "Host Statistics"); 3532 txslist = SYSCTL_CHILDREN(txsnode); 3533 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3534 &txstats->TSO_packets, "TSO packets"); 3535 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3536 &txstats->TSO_bytes, "TSO bytes"); 3537 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3538 &txstats->ucast_packets, "Unicast packets"); 3539 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3540 &txstats->ucast_bytes, "Unicast bytes"); 3541 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3542 &txstats->mcast_packets, "Multicast packets"); 3543 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3544 &txstats->mcast_bytes, "Multicast bytes"); 3545 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3546 &txstats->error, "Errors"); 3547 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3548 &txstats->discard, "Discards"); 3549} 3550 3551static void 3552vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3553 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3554{ 3555 struct sysctl_oid *node, *rxsnode; 3556 struct sysctl_oid_list *list, *rxslist; 3557 struct vmxnet3_rxq_stats *stats; 3558 struct UPT1_RxStats *rxstats; 3559 char namebuf[16]; 3560 3561 stats = &rxq->vxrxq_stats; 3562 rxstats = &rxq->vxrxq_rs->stats; 3563 3564 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3565 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3566 NULL, "Receive Queue"); 3567 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3568 3569 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3570 &stats->vmrxs_ipackets, "Receive packets"); 3571 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3572 &stats->vmrxs_ibytes, "Receive bytes"); 3573 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3574 &stats->vmrxs_iqdrops, "Receive drops"); 3575 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3576 &stats->vmrxs_ierrors, "Receive errors"); 3577 3578 /* 3579 * Add statistics reported by the host. These are updated once 3580 * per second. 3581 */ 3582 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3583 NULL, "Host Statistics"); 3584 rxslist = SYSCTL_CHILDREN(rxsnode); 3585 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3586 &rxstats->LRO_packets, "LRO packets"); 3587 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3588 &rxstats->LRO_bytes, "LRO bytes"); 3589 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3590 &rxstats->ucast_packets, "Unicast packets"); 3591 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3592 &rxstats->ucast_bytes, "Unicast bytes"); 3593 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3594 &rxstats->mcast_packets, "Multicast packets"); 3595 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3596 &rxstats->mcast_bytes, "Multicast bytes"); 3597 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3598 &rxstats->bcast_packets, "Broadcast packets"); 3599 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3600 &rxstats->bcast_bytes, "Broadcast bytes"); 3601 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3602 &rxstats->nobuffer, "No buffer"); 3603 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3604 &rxstats->error, "Errors"); 3605} 3606 3607static void 3608vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3609 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3610{ 3611 struct sysctl_oid *node; 3612 struct sysctl_oid_list *list; 3613 int i; 3614 3615 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3616 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3617 3618 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3619 "debug", CTLFLAG_RD, NULL, ""); 3620 list = SYSCTL_CHILDREN(node); 3621 3622 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3623 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3624 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3625 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3626 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3627 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3628 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3629 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3630 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3631 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3632 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3633 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3634 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3635 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3636 } 3637 3638 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3639 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3640 3641 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3642 "debug", CTLFLAG_RD, NULL, ""); 3643 list = SYSCTL_CHILDREN(node); 3644 3645 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3646 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3647 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3648 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3649 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3650 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3651 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3652 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3653 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3654 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3655 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3656 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3657 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3658 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3659 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3660 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3661 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3662 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3663 } 3664} 3665 3666static void 3667vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3668 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3669{ 3670 int i; 3671 3672 for (i = 0; i < sc->vmx_ntxqueues; i++) 3673 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3674 for (i = 0; i < sc->vmx_nrxqueues; i++) 3675 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3676 3677 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3678} 3679 3680static void 3681vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3682{ 3683 device_t dev; 3684 struct vmxnet3_statistics *stats; 3685 struct sysctl_ctx_list *ctx; 3686 struct sysctl_oid *tree; 3687 struct sysctl_oid_list *child; 3688 3689 dev = sc->vmx_dev; 3690 ctx = device_get_sysctl_ctx(dev); 3691 tree = device_get_sysctl_tree(dev); 3692 child = SYSCTL_CHILDREN(tree); 3693 3694 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3695 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3696 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3697 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3698 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3699 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3700 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3701 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3702 3703 stats = &sc->vmx_stats; 3704 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3705 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3706 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3707 &stats->vmst_defrag_failed, 0, 3708 "Tx mbuf dropped because defrag failed"); 3709 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3710 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3711 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3712 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3713 3714 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3715} 3716 3717static void 3718vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3719{ 3720 3721 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3722} 3723 3724static uint32_t 3725vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3726{ 3727 3728 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3729} 3730 3731static void 3732vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3733{ 3734 3735 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3736} 3737 3738static void 3739vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3740{ 3741 3742 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3743} 3744 3745static uint32_t 3746vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3747{ 3748 3749 vmxnet3_write_cmd(sc, cmd); 3750 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3751 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3752 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3753} 3754 3755static void 3756vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3757{ 3758 3759 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3760} 3761 3762static void 3763vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3764{ 3765 3766 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3767} 3768 3769static void 3770vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3771{ 3772 int i; 3773 3774 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3775 for (i = 0; i < sc->vmx_nintrs; i++) 3776 vmxnet3_enable_intr(sc, i); 3777} 3778 3779static void 3780vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3781{ 3782 int i; 3783 3784 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3785 for (i = 0; i < sc->vmx_nintrs; i++) 3786 vmxnet3_disable_intr(sc, i); 3787} 3788 3789static void 3790vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3791{ 3792 bus_addr_t *baddr = arg; 3793 3794 if (error == 0) 3795 *baddr = segs->ds_addr; 3796} 3797 3798static int 3799vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3800 struct vmxnet3_dma_alloc *dma) 3801{ 3802 device_t dev; 3803 int error; 3804 3805 dev = sc->vmx_dev; 3806 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3807 3808 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3809 align, 0, /* alignment, bounds */ 3810 BUS_SPACE_MAXADDR, /* lowaddr */ 3811 BUS_SPACE_MAXADDR, /* highaddr */ 3812 NULL, NULL, /* filter, filterarg */ 3813 size, /* maxsize */ 3814 1, /* nsegments */ 3815 size, /* maxsegsize */ 3816 BUS_DMA_ALLOCNOW, /* flags */ 3817 NULL, /* lockfunc */ 3818 NULL, /* lockfuncarg */ 3819 &dma->dma_tag); 3820 if (error) { 3821 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3822 goto fail; 3823 } 3824 3825 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3826 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3827 if (error) { 3828 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3829 goto fail; 3830 } 3831 3832 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3833 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3834 if (error) { 3835 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3836 goto fail; 3837 } 3838 3839 dma->dma_size = size; 3840 3841fail: 3842 if (error) 3843 vmxnet3_dma_free(sc, dma); 3844 3845 return (error); 3846} 3847 3848static void 3849vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3850{ 3851 3852 if (dma->dma_tag != NULL) { 3853 if (dma->dma_map != NULL) { 3854 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3855 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3856 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3857 } 3858 3859 if (dma->dma_vaddr != NULL) { 3860 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3861 dma->dma_map); 3862 } 3863 3864 bus_dma_tag_destroy(dma->dma_tag); 3865 } 3866 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3867} 3868 3869static int 3870vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3871{ 3872 char path[64]; 3873 3874 snprintf(path, sizeof(path), 3875 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3876 TUNABLE_INT_FETCH(path, &def); 3877 3878 return (def); 3879} 3880 3881/* 3882 * Since this is a purely paravirtualized device, we do not have 3883 * to worry about DMA coherency. But at times, we must make sure 3884 * both the compiler and CPU do not reorder memory operations. 3885 */ 3886static inline void 3887vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3888{ 3889 3890 switch (type) { 3891 case VMXNET3_BARRIER_RD: 3892 rmb(); 3893 break; 3894 case VMXNET3_BARRIER_WR: 3895 wmb(); 3896 break; 3897 case VMXNET3_BARRIER_RDWR: 3898 mb(); 3899 break; 3900 default: 3901 panic("%s: bad barrier type %d", __func__, type); 3902 } 3903} 3904