if_vmx.c revision 254950
1/*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20/* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22#include <sys/cdefs.h> 23__FBSDID("$FreeBSD: head/sys/dev/vmware/vmxnet3/if_vmx.c 254950 2013-08-27 04:05:18Z bryanv $"); 24 25#include <sys/param.h> 26#include <sys/systm.h> 27#include <sys/kernel.h> 28#include <sys/endian.h> 29#include <sys/sockio.h> 30#include <sys/mbuf.h> 31#include <sys/malloc.h> 32#include <sys/module.h> 33#include <sys/socket.h> 34#include <sys/sysctl.h> 35#include <vm/vm.h> 36#include <vm/pmap.h> 37 38#include <net/ethernet.h> 39#include <net/if.h> 40#include <net/if_arp.h> 41#include <net/if_dl.h> 42#include <net/if_types.h> 43#include <net/if_media.h> 44#include <net/if_vlan_var.h> 45 46#include <net/bpf.h> 47 48#include <netinet/in_systm.h> 49#include <netinet/in.h> 50#include <netinet/ip.h> 51#include <netinet/ip6.h> 52#include <netinet6/ip6_var.h> 53#include <netinet/udp.h> 54#include <netinet/tcp.h> 55 56#include <machine/bus.h> 57#include <machine/resource.h> 58#include <sys/bus.h> 59#include <sys/rman.h> 60 61#include <dev/pci/pcireg.h> 62#include <dev/pci/pcivar.h> 63 64#include "if_vmxreg.h" 65#include "if_vmxvar.h" 66 67#include "opt_inet.h" 68#include "opt_inet6.h" 69 70/* Always enable for now - useful for queue hangs. */ 71#define VMXNET3_DEBUG_SYSCTL 72 73#ifdef VMXNET3_FAILPOINTS 74#include <sys/fail.h> 75static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 76 "vmxnet3 fail points"); 77#define VMXNET3_FP _debug_fail_point_vmxnet3 78#endif 79 80static int vmxnet3_probe(device_t); 81static int vmxnet3_attach(device_t); 82static int vmxnet3_detach(device_t); 83static int vmxnet3_shutdown(device_t); 84 85static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 86static void vmxnet3_free_resources(struct vmxnet3_softc *); 87static int vmxnet3_check_version(struct vmxnet3_softc *); 88static void vmxnet3_initial_config(struct vmxnet3_softc *); 89 90static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 91static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 92static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 93static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 94 struct vmxnet3_interrupt *); 95static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 96static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 97static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 98static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 99static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 100 101static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 102 struct vmxnet3_interrupt *); 103static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 104 105static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 106static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 107static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 108static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 109static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 110static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 111 112static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 113static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 114static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 115static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 116static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 117static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 118static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 119static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 120static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 121static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 122static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 123static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 124static int vmxnet3_alloc_data(struct vmxnet3_softc *); 125static void vmxnet3_free_data(struct vmxnet3_softc *); 126static int vmxnet3_setup_interface(struct vmxnet3_softc *); 127 128static void vmxnet3_evintr(struct vmxnet3_softc *); 129static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 130static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 131static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 132static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 133 struct vmxnet3_rxring *, int); 134static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 135static void vmxnet3_legacy_intr(void *); 136static void vmxnet3_txq_intr(void *); 137static void vmxnet3_rxq_intr(void *); 138static void vmxnet3_event_intr(void *); 139 140static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 141static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 142static void vmxnet3_stop(struct vmxnet3_softc *); 143 144static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 145static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 146static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 147static int vmxnet3_enable_device(struct vmxnet3_softc *); 148static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 149static int vmxnet3_reinit(struct vmxnet3_softc *); 150static void vmxnet3_init_locked(struct vmxnet3_softc *); 151static void vmxnet3_init(void *); 152 153static int vmxnet3_txq_offload_ctx(struct mbuf *, int *, int *, int *); 154static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 155 bus_dmamap_t, bus_dma_segment_t [], int *); 156static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 157static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 158static void vmxnet3_start_locked(struct ifnet *); 159static void vmxnet3_start(struct ifnet *); 160 161static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 162 uint16_t); 163static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 164static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 165static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 166static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 167static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 168 169static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 170static void vmxnet3_tick(void *); 171static void vmxnet3_link_status(struct vmxnet3_softc *); 172static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 173static int vmxnet3_media_change(struct ifnet *); 174static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 175static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 176 177static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 178 struct sysctl_ctx_list *, struct sysctl_oid_list *); 179static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 180 struct sysctl_ctx_list *, struct sysctl_oid_list *); 181static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 182 struct sysctl_ctx_list *, struct sysctl_oid_list *); 183static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 184 185static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 186 uint32_t); 187static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 188static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 189 uint32_t); 190static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 191static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 192 193static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 194static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 195static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 196static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 197 198static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 199 bus_size_t, struct vmxnet3_dma_alloc *); 200static void vmxnet3_dma_free(struct vmxnet3_softc *, 201 struct vmxnet3_dma_alloc *); 202 203typedef enum { 204 VMXNET3_BARRIER_RD, 205 VMXNET3_BARRIER_WR, 206 VMXNET3_BARRIER_RDWR, 207} vmxnet3_barrier_t; 208 209static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 210 211static device_method_t vmxnet3_methods[] = { 212 /* Device interface. */ 213 DEVMETHOD(device_probe, vmxnet3_probe), 214 DEVMETHOD(device_attach, vmxnet3_attach), 215 DEVMETHOD(device_detach, vmxnet3_detach), 216 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 217 218 DEVMETHOD_END 219}; 220 221static driver_t vmxnet3_driver = { 222 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 223}; 224 225static devclass_t vmxnet3_devclass; 226DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 227 228MODULE_DEPEND(vmx, pci, 1, 1, 1); 229MODULE_DEPEND(vmx, ether, 1, 1, 1); 230 231#define VMXNET3_VMWARE_VENDOR_ID 0x15AD 232#define VMXNET3_VMWARE_DEVICE_ID 0x07B0 233 234static int 235vmxnet3_probe(device_t dev) 236{ 237 238 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 239 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 240 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 241 return (BUS_PROBE_DEFAULT); 242 } 243 244 return (ENXIO); 245} 246 247static int 248vmxnet3_attach(device_t dev) 249{ 250 struct vmxnet3_softc *sc; 251 int error; 252 253 sc = device_get_softc(dev); 254 sc->vmx_dev = dev; 255 256 pci_enable_busmaster(dev); 257 258 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 259 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 260 261 vmxnet3_initial_config(sc); 262 263 error = vmxnet3_alloc_resources(sc); 264 if (error) 265 goto fail; 266 267 error = vmxnet3_check_version(sc); 268 if (error) 269 goto fail; 270 271 error = vmxnet3_alloc_rxtx_queues(sc); 272 if (error) 273 goto fail; 274 275 error = vmxnet3_alloc_interrupts(sc); 276 if (error) 277 goto fail; 278 279 error = vmxnet3_alloc_data(sc); 280 if (error) 281 goto fail; 282 283 error = vmxnet3_setup_interface(sc); 284 if (error) 285 goto fail; 286 287 error = vmxnet3_setup_interrupts(sc); 288 if (error) { 289 ether_ifdetach(sc->vmx_ifp); 290 device_printf(dev, "could not set up interrupt\n"); 291 goto fail; 292 } 293 294 vmxnet3_setup_sysctl(sc); 295 vmxnet3_link_status(sc); 296 297fail: 298 if (error) 299 vmxnet3_detach(dev); 300 301 return (error); 302} 303 304static int 305vmxnet3_detach(device_t dev) 306{ 307 struct vmxnet3_softc *sc; 308 struct ifnet *ifp; 309 310 sc = device_get_softc(dev); 311 ifp = sc->vmx_ifp; 312 313 if (device_is_attached(dev)) { 314 ether_ifdetach(ifp); 315 VMXNET3_CORE_LOCK(sc); 316 vmxnet3_stop(sc); 317 VMXNET3_CORE_UNLOCK(sc); 318 callout_drain(&sc->vmx_tick); 319 } 320 321 if (sc->vmx_vlan_attach != NULL) { 322 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 323 sc->vmx_vlan_attach = NULL; 324 } 325 if (sc->vmx_vlan_detach != NULL) { 326 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 327 sc->vmx_vlan_detach = NULL; 328 } 329 330 vmxnet3_free_interrupts(sc); 331 332 if (ifp != NULL) { 333 if_free(ifp); 334 sc->vmx_ifp = NULL; 335 } 336 337 ifmedia_removeall(&sc->vmx_media); 338 339 vmxnet3_free_data(sc); 340 vmxnet3_free_resources(sc); 341 vmxnet3_free_rxtx_queues(sc); 342 343 VMXNET3_CORE_LOCK_DESTROY(sc); 344 345 return (0); 346} 347 348static int 349vmxnet3_shutdown(device_t dev) 350{ 351 352 return (0); 353} 354 355static int 356vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 357{ 358 device_t dev; 359 int rid; 360 361 dev = sc->vmx_dev; 362 363 rid = PCIR_BAR(0); 364 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 365 RF_ACTIVE); 366 if (sc->vmx_res0 == NULL) { 367 device_printf(dev, 368 "could not map BAR0 memory\n"); 369 return (ENXIO); 370 } 371 372 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 373 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 374 375 rid = PCIR_BAR(1); 376 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 377 RF_ACTIVE); 378 if (sc->vmx_res1 == NULL) { 379 device_printf(dev, 380 "could not map BAR1 memory\n"); 381 return (ENXIO); 382 } 383 384 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 385 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 386 387 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 388 rid = PCIR_BAR(2); 389 sc->vmx_msix_res = bus_alloc_resource_any(dev, 390 SYS_RES_MEMORY, &rid, RF_ACTIVE); 391 } 392 393 if (sc->vmx_msix_res == NULL) 394 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 395 396 return (0); 397} 398 399static void 400vmxnet3_free_resources(struct vmxnet3_softc *sc) 401{ 402 device_t dev; 403 int rid; 404 405 dev = sc->vmx_dev; 406 407 if (sc->vmx_res0 != NULL) { 408 rid = PCIR_BAR(0); 409 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 410 sc->vmx_res0 = NULL; 411 } 412 413 if (sc->vmx_res1 != NULL) { 414 rid = PCIR_BAR(1); 415 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 416 sc->vmx_res1 = NULL; 417 } 418 419 if (sc->vmx_msix_res != NULL) { 420 rid = PCIR_BAR(2); 421 bus_release_resource(dev, SYS_RES_MEMORY, rid, 422 sc->vmx_msix_res); 423 sc->vmx_msix_res = NULL; 424 } 425} 426 427static int 428vmxnet3_check_version(struct vmxnet3_softc *sc) 429{ 430 device_t dev; 431 uint32_t version; 432 433 dev = sc->vmx_dev; 434 435 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 436 if ((version & 0x01) == 0) { 437 device_printf(dev, "unsupported hardware version %#x\n", 438 version); 439 return (ENOTSUP); 440 } 441 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 442 443 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 444 if ((version & 0x01) == 0) { 445 device_printf(dev, "unsupported UPT version %#x\n", version); 446 return (ENOTSUP); 447 } 448 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 449 450 return (0); 451} 452 453static void 454vmxnet3_initial_config(struct vmxnet3_softc *sc) 455{ 456 457 sc->vmx_ntxqueues = 1; 458 sc->vmx_nrxqueues = 1; 459 sc->vmx_ntxdescs = VMXNET3_MAX_TX_NDESC; 460 sc->vmx_nrxdescs = VMXNET3_MAX_RX_NDESC; 461 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 462} 463 464static int 465vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 466{ 467 device_t dev; 468 int nmsix, cnt, required; 469 470 dev = sc->vmx_dev; 471 472 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 473 return (1); 474 475 /* Allocate an additional vector for the events interrupt. */ 476 required = sc->vmx_nrxqueues + sc->vmx_ntxqueues + 1; 477 478 nmsix = pci_msix_count(dev); 479 if (nmsix < required) 480 return (1); 481 482 cnt = required; 483 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 484 sc->vmx_nintrs = required; 485 return (0); 486 } else 487 pci_release_msi(dev); 488 489 return (1); 490} 491 492static int 493vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 494{ 495 device_t dev; 496 int nmsi, cnt, required; 497 498 dev = sc->vmx_dev; 499 required = 1; 500 501 nmsi = pci_msi_count(dev); 502 if (nmsi < required) 503 return (1); 504 505 cnt = required; 506 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 507 sc->vmx_nintrs = 1; 508 return (0); 509 } else 510 pci_release_msi(dev); 511 512 return (1); 513} 514 515static int 516vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 517{ 518 519 sc->vmx_nintrs = 1; 520 return (0); 521} 522 523static int 524vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 525 struct vmxnet3_interrupt *intr) 526{ 527 struct resource *irq; 528 529 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 530 if (irq == NULL) 531 return (ENXIO); 532 533 intr->vmxi_irq = irq; 534 intr->vmxi_rid = rid; 535 536 return (0); 537} 538 539static int 540vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 541{ 542 int i, rid, flags, error; 543 544 rid = 0; 545 flags = RF_ACTIVE; 546 547 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 548 flags |= RF_SHAREABLE; 549 else 550 rid = 1; 551 552 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 553 error = vmxnet3_alloc_interrupt(sc, rid, flags, 554 &sc->vmx_intrs[i]); 555 if (error) 556 return (error); 557 } 558 559 return (0); 560} 561 562/* 563 * NOTE: We only support the simple case of each Rx and Tx queue on its 564 * own MSIX vector. This is good enough until we support mulitqueue. 565 */ 566static int 567vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 568{ 569 device_t dev; 570 struct vmxnet3_txqueue *txq; 571 struct vmxnet3_rxqueue *rxq; 572 struct vmxnet3_interrupt *intr; 573 enum intr_type type; 574 int i, error; 575 576 dev = sc->vmx_dev; 577 intr = &sc->vmx_intrs[0]; 578 type = INTR_TYPE_NET | INTR_MPSAFE; 579 580 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 581 txq = &sc->vmx_txq[i]; 582 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 583 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 584 if (error) 585 return (error); 586 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 587 } 588 589 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 590 rxq = &sc->vmx_rxq[i]; 591 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 592 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 593 if (error) 594 return (error); 595 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 596 } 597 598 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 599 vmxnet3_event_intr, sc, &intr->vmxi_handler); 600 if (error) 601 return (error); 602 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 603 604 return (0); 605} 606 607static int 608vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 609{ 610 struct vmxnet3_interrupt *intr; 611 int i, error; 612 613 intr = &sc->vmx_intrs[0]; 614 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 615 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 616 &intr->vmxi_handler); 617 618 for (i = 0; i < sc->vmx_ntxqueues; i++) 619 sc->vmx_txq[i].vxtxq_intr_idx = 0; 620 for (i = 0; i < sc->vmx_nrxqueues; i++) 621 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 622 sc->vmx_event_intr_idx = 0; 623 624 return (error); 625} 626 627/* 628 * XXX BMV Should probably reorganize the attach and just do 629 * this in vmxnet3_init_shared_data(). 630 */ 631static void 632vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 633{ 634 struct vmxnet3_txqueue *txq; 635 struct vmxnet3_txq_shared *txs; 636 struct vmxnet3_rxqueue *rxq; 637 struct vmxnet3_rxq_shared *rxs; 638 int i; 639 640 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 641 642 for (i = 0; i < sc->vmx_ntxqueues; i++) { 643 txq = &sc->vmx_txq[i]; 644 txs = txq->vxtxq_ts; 645 txs->intr_idx = txq->vxtxq_intr_idx; 646 } 647 648 for (i = 0; i < sc->vmx_nrxqueues; i++) { 649 rxq = &sc->vmx_rxq[i]; 650 rxs = rxq->vxrxq_rs; 651 rxs->intr_idx = rxq->vxrxq_intr_idx; 652 } 653} 654 655static int 656vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 657{ 658 int error; 659 660 error = vmxnet3_alloc_intr_resources(sc); 661 if (error) 662 return (error); 663 664 switch (sc->vmx_intr_type) { 665 case VMXNET3_IT_MSIX: 666 error = vmxnet3_setup_msix_interrupts(sc); 667 break; 668 case VMXNET3_IT_MSI: 669 case VMXNET3_IT_LEGACY: 670 error = vmxnet3_setup_legacy_interrupt(sc); 671 break; 672 default: 673 panic("%s: invalid interrupt type %d", __func__, 674 sc->vmx_intr_type); 675 } 676 677 if (error == 0) 678 vmxnet3_set_interrupt_idx(sc); 679 680 return (error); 681} 682 683static int 684vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 685{ 686 device_t dev; 687 uint32_t config; 688 int error; 689 690 dev = sc->vmx_dev; 691 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 692 693 sc->vmx_intr_type = config & 0x03; 694 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 695 696 switch (sc->vmx_intr_type) { 697 case VMXNET3_IT_AUTO: 698 sc->vmx_intr_type = VMXNET3_IT_MSIX; 699 /* FALLTHROUGH */ 700 case VMXNET3_IT_MSIX: 701 error = vmxnet3_alloc_msix_interrupts(sc); 702 if (error == 0) 703 break; 704 sc->vmx_intr_type = VMXNET3_IT_MSI; 705 /* FALLTHROUGH */ 706 case VMXNET3_IT_MSI: 707 error = vmxnet3_alloc_msi_interrupts(sc); 708 if (error == 0) 709 break; 710 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 711 /* FALLTHROUGH */ 712 case VMXNET3_IT_LEGACY: 713 error = vmxnet3_alloc_legacy_interrupts(sc); 714 if (error == 0) 715 break; 716 /* FALLTHROUGH */ 717 default: 718 sc->vmx_intr_type = -1; 719 device_printf(dev, "cannot allocate any interrupt resources\n"); 720 return (ENXIO); 721 } 722 723 return (error); 724} 725 726static void 727vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 728 struct vmxnet3_interrupt *intr) 729{ 730 device_t dev; 731 732 dev = sc->vmx_dev; 733 734 if (intr->vmxi_handler != NULL) { 735 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 736 intr->vmxi_handler = NULL; 737 } 738 739 if (intr->vmxi_irq != NULL) { 740 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 741 intr->vmxi_irq); 742 intr->vmxi_irq = NULL; 743 intr->vmxi_rid = -1; 744 } 745} 746 747static void 748vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 749{ 750 int i; 751 752 for (i = 0; i < sc->vmx_nintrs; i++) 753 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 754 755 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 756 sc->vmx_intr_type == VMXNET3_IT_MSIX) 757 pci_release_msi(sc->vmx_dev); 758} 759 760static int 761vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 762{ 763 struct vmxnet3_rxqueue *rxq; 764 struct vmxnet3_rxring *rxr; 765 int i; 766 767 rxq = &sc->vmx_rxq[q]; 768 769 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 770 device_get_nameunit(sc->vmx_dev), q); 771 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 772 773 rxq->vxrxq_sc = sc; 774 rxq->vxrxq_id = q; 775 776 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 777 rxr = &rxq->vxrxq_cmd_ring[i]; 778 rxr->vxrxr_rid = i; 779 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 780 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 781 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 782 if (rxr->vxrxr_rxbuf == NULL) 783 return (ENOMEM); 784 785 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 786 } 787 788 return (0); 789} 790 791static int 792vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 793{ 794 struct vmxnet3_txqueue *txq; 795 struct vmxnet3_txring *txr; 796 797 txq = &sc->vmx_txq[q]; 798 txr = &txq->vxtxq_cmd_ring; 799 800 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 801 device_get_nameunit(sc->vmx_dev), q); 802 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 803 804 txq->vxtxq_sc = sc; 805 txq->vxtxq_id = q; 806 807 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 808 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 809 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 810 if (txr->vxtxr_txbuf == NULL) 811 return (ENOMEM); 812 813 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 814 815 return (0); 816} 817 818static int 819vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 820{ 821 int i, error; 822 823 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 824 sc->vmx_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 825 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 826 sc->vmx_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 827 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 828 return (ENOMEM); 829 830 for (i = 0; i < sc->vmx_nrxqueues; i++) { 831 error = vmxnet3_init_rxq(sc, i); 832 if (error) 833 return (error); 834 } 835 836 for (i = 0; i < sc->vmx_ntxqueues; i++) { 837 error = vmxnet3_init_txq(sc, i); 838 if (error) 839 return (error); 840 } 841 842 return (0); 843} 844 845static void 846vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 847{ 848 struct vmxnet3_rxring *rxr; 849 int i; 850 851 rxq->vxrxq_sc = NULL; 852 rxq->vxrxq_id = -1; 853 854 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 855 rxr = &rxq->vxrxq_cmd_ring[i]; 856 857 if (rxr->vxrxr_rxbuf != NULL) { 858 free(rxr->vxrxr_rxbuf, M_DEVBUF); 859 rxr->vxrxr_rxbuf = NULL; 860 } 861 } 862 863 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 864 mtx_destroy(&rxq->vxrxq_mtx); 865} 866 867static void 868vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 869{ 870 struct vmxnet3_txring *txr; 871 872 txr = &txq->vxtxq_cmd_ring; 873 874 txq->vxtxq_sc = NULL; 875 txq->vxtxq_id = -1; 876 877 if (txr->vxtxr_txbuf != NULL) { 878 free(txr->vxtxr_txbuf, M_DEVBUF); 879 txr->vxtxr_txbuf = NULL; 880 } 881 882 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 883 mtx_destroy(&txq->vxtxq_mtx); 884} 885 886static void 887vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 888{ 889 int i; 890 891 if (sc->vmx_rxq != NULL) { 892 for (i = 0; i < sc->vmx_nrxqueues; i++) 893 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 894 free(sc->vmx_rxq, M_DEVBUF); 895 sc->vmx_rxq = NULL; 896 } 897 898 if (sc->vmx_txq != NULL) { 899 for (i = 0; i < sc->vmx_ntxqueues; i++) 900 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 901 free(sc->vmx_txq, M_DEVBUF); 902 sc->vmx_txq = NULL; 903 } 904} 905 906static int 907vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 908{ 909 device_t dev; 910 uint8_t *kva; 911 size_t size; 912 int i, error; 913 914 dev = sc->vmx_dev; 915 916 size = sizeof(struct vmxnet3_driver_shared); 917 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 918 if (error) { 919 device_printf(dev, "cannot alloc shared memory\n"); 920 return (error); 921 } 922 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 923 924 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 925 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 926 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 927 if (error) { 928 device_printf(dev, "cannot alloc queue shared memory\n"); 929 return (error); 930 } 931 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 932 kva = sc->vmx_qs; 933 934 for (i = 0; i < sc->vmx_ntxqueues; i++) { 935 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 936 kva += sizeof(struct vmxnet3_txq_shared); 937 } 938 for (i = 0; i < sc->vmx_nrxqueues; i++) { 939 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 940 kva += sizeof(struct vmxnet3_rxq_shared); 941 } 942 943 return (0); 944} 945 946static void 947vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 948{ 949 950 if (sc->vmx_qs != NULL) { 951 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 952 sc->vmx_qs = NULL; 953 } 954 955 if (sc->vmx_ds != NULL) { 956 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 957 sc->vmx_ds = NULL; 958 } 959} 960 961static int 962vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 963{ 964 device_t dev; 965 struct vmxnet3_txqueue *txq; 966 struct vmxnet3_txring *txr; 967 struct vmxnet3_comp_ring *txc; 968 size_t descsz, compsz; 969 int i, q, error; 970 971 dev = sc->vmx_dev; 972 973 for (q = 0; q < sc->vmx_ntxqueues; q++) { 974 txq = &sc->vmx_txq[q]; 975 txr = &txq->vxtxq_cmd_ring; 976 txc = &txq->vxtxq_comp_ring; 977 978 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 979 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 980 981 error = bus_dma_tag_create(bus_get_dma_tag(dev), 982 1, 0, /* alignment, boundary */ 983 BUS_SPACE_MAXADDR, /* lowaddr */ 984 BUS_SPACE_MAXADDR, /* highaddr */ 985 NULL, NULL, /* filter, filterarg */ 986 VMXNET3_TSO_MAXSIZE, /* maxsize */ 987 VMXNET3_TX_MAXSEGS, /* nsegments */ 988 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 989 0, /* flags */ 990 NULL, NULL, /* lockfunc, lockarg */ 991 &txr->vxtxr_txtag); 992 if (error) { 993 device_printf(dev, 994 "unable to create Tx buffer tag for queue %d\n", q); 995 return (error); 996 } 997 998 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 999 if (error) { 1000 device_printf(dev, "cannot alloc Tx descriptors for " 1001 "queue %d error %d\n", q, error); 1002 return (error); 1003 } 1004 txr->vxtxr_txd = 1005 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1006 1007 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1008 if (error) { 1009 device_printf(dev, "cannot alloc Tx comp descriptors " 1010 "for queue %d error %d\n", q, error); 1011 return (error); 1012 } 1013 txc->vxcr_u.txcd = 1014 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1015 1016 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1017 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1018 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1019 if (error) { 1020 device_printf(dev, "unable to create Tx buf " 1021 "dmamap for queue %d idx %d\n", q, i); 1022 return (error); 1023 } 1024 } 1025 } 1026 1027 return (0); 1028} 1029 1030static void 1031vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1032{ 1033 device_t dev; 1034 struct vmxnet3_txqueue *txq; 1035 struct vmxnet3_txring *txr; 1036 struct vmxnet3_comp_ring *txc; 1037 struct vmxnet3_txbuf *txb; 1038 int i, q; 1039 1040 dev = sc->vmx_dev; 1041 1042 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1043 txq = &sc->vmx_txq[q]; 1044 txr = &txq->vxtxq_cmd_ring; 1045 txc = &txq->vxtxq_comp_ring; 1046 1047 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1048 txb = &txr->vxtxr_txbuf[i]; 1049 if (txb->vtxb_dmamap != NULL) { 1050 bus_dmamap_destroy(txr->vxtxr_txtag, 1051 txb->vtxb_dmamap); 1052 txb->vtxb_dmamap = NULL; 1053 } 1054 } 1055 1056 if (txc->vxcr_u.txcd != NULL) { 1057 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1058 txc->vxcr_u.txcd = NULL; 1059 } 1060 1061 if (txr->vxtxr_txd != NULL) { 1062 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1063 txr->vxtxr_txd = NULL; 1064 } 1065 1066 if (txr->vxtxr_txtag != NULL) { 1067 bus_dma_tag_destroy(txr->vxtxr_txtag); 1068 txr->vxtxr_txtag = NULL; 1069 } 1070 } 1071} 1072 1073static int 1074vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1075{ 1076 device_t dev; 1077 struct vmxnet3_rxqueue *rxq; 1078 struct vmxnet3_rxring *rxr; 1079 struct vmxnet3_comp_ring *rxc; 1080 int descsz, compsz; 1081 int i, j, q, error; 1082 1083 dev = sc->vmx_dev; 1084 1085 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1086 rxq = &sc->vmx_rxq[q]; 1087 rxc = &rxq->vxrxq_comp_ring; 1088 compsz = 0; 1089 1090 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1091 rxr = &rxq->vxrxq_cmd_ring[i]; 1092 1093 descsz = rxr->vxrxr_ndesc * 1094 sizeof(struct vmxnet3_rxdesc); 1095 compsz += rxr->vxrxr_ndesc * 1096 sizeof(struct vmxnet3_rxcompdesc); 1097 1098 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1099 1, 0, /* alignment, boundary */ 1100 BUS_SPACE_MAXADDR, /* lowaddr */ 1101 BUS_SPACE_MAXADDR, /* highaddr */ 1102 NULL, NULL, /* filter, filterarg */ 1103 MJUMPAGESIZE, /* maxsize */ 1104 1, /* nsegments */ 1105 MJUMPAGESIZE, /* maxsegsize */ 1106 0, /* flags */ 1107 NULL, NULL, /* lockfunc, lockarg */ 1108 &rxr->vxrxr_rxtag); 1109 if (error) { 1110 device_printf(dev, 1111 "unable to create Rx buffer tag for " 1112 "queue %d\n", q); 1113 return (error); 1114 } 1115 1116 error = vmxnet3_dma_malloc(sc, descsz, 512, 1117 &rxr->vxrxr_dma); 1118 if (error) { 1119 device_printf(dev, "cannot allocate Rx " 1120 "descriptors for queue %d/%d error %d\n", 1121 i, q, error); 1122 return (error); 1123 } 1124 rxr->vxrxr_rxd = 1125 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1126 } 1127 1128 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1129 if (error) { 1130 device_printf(dev, "cannot alloc Rx comp descriptors " 1131 "for queue %d error %d\n", q, error); 1132 return (error); 1133 } 1134 rxc->vxcr_u.rxcd = 1135 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1136 1137 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1138 rxr = &rxq->vxrxq_cmd_ring[i]; 1139 1140 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1141 &rxr->vxrxr_spare_dmap); 1142 if (error) { 1143 device_printf(dev, "unable to create spare " 1144 "dmamap for queue %d/%d error %d\n", 1145 q, i, error); 1146 return (error); 1147 } 1148 1149 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1150 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1151 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1152 if (error) { 1153 device_printf(dev, "unable to create " 1154 "dmamap for queue %d/%d slot %d " 1155 "error %d\n", 1156 q, i, j, error); 1157 return (error); 1158 } 1159 } 1160 } 1161 } 1162 1163 return (0); 1164} 1165 1166static void 1167vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1168{ 1169 device_t dev; 1170 struct vmxnet3_rxqueue *rxq; 1171 struct vmxnet3_rxring *rxr; 1172 struct vmxnet3_comp_ring *rxc; 1173 struct vmxnet3_rxbuf *rxb; 1174 int i, j, q; 1175 1176 dev = sc->vmx_dev; 1177 1178 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1179 rxq = &sc->vmx_rxq[q]; 1180 rxc = &rxq->vxrxq_comp_ring; 1181 1182 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1183 rxr = &rxq->vxrxq_cmd_ring[i]; 1184 1185 if (rxr->vxrxr_spare_dmap != NULL) { 1186 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1187 rxr->vxrxr_spare_dmap); 1188 rxr->vxrxr_spare_dmap = NULL; 1189 } 1190 1191 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1192 rxb = &rxr->vxrxr_rxbuf[j]; 1193 if (rxb->vrxb_dmamap != NULL) { 1194 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1195 rxb->vrxb_dmamap); 1196 rxb->vrxb_dmamap = NULL; 1197 } 1198 } 1199 } 1200 1201 if (rxc->vxcr_u.rxcd != NULL) { 1202 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1203 rxc->vxcr_u.rxcd = NULL; 1204 } 1205 1206 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1207 rxr = &rxq->vxrxq_cmd_ring[i]; 1208 1209 if (rxr->vxrxr_rxd != NULL) { 1210 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1211 rxr->vxrxr_rxd = NULL; 1212 } 1213 1214 if (rxr->vxrxr_rxtag != NULL) { 1215 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1216 rxr->vxrxr_rxtag = NULL; 1217 } 1218 } 1219 } 1220} 1221 1222static int 1223vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1224{ 1225 int error; 1226 1227 error = vmxnet3_alloc_txq_data(sc); 1228 if (error) 1229 return (error); 1230 1231 error = vmxnet3_alloc_rxq_data(sc); 1232 if (error) 1233 return (error); 1234 1235 return (0); 1236} 1237 1238static void 1239vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1240{ 1241 1242 if (sc->vmx_rxq != NULL) 1243 vmxnet3_free_rxq_data(sc); 1244 1245 if (sc->vmx_txq != NULL) 1246 vmxnet3_free_txq_data(sc); 1247} 1248 1249static int 1250vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1251{ 1252 int error; 1253 1254 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1255 32, &sc->vmx_mcast_dma); 1256 if (error) 1257 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1258 else 1259 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1260 1261 return (error); 1262} 1263 1264static void 1265vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1266{ 1267 1268 if (sc->vmx_mcast != NULL) { 1269 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1270 sc->vmx_mcast = NULL; 1271 } 1272} 1273 1274static void 1275vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1276{ 1277 struct vmxnet3_driver_shared *ds; 1278 struct vmxnet3_txqueue *txq; 1279 struct vmxnet3_txq_shared *txs; 1280 struct vmxnet3_rxqueue *rxq; 1281 struct vmxnet3_rxq_shared *rxs; 1282 int i; 1283 1284 ds = sc->vmx_ds; 1285 1286 /* 1287 * Initialize fields of the shared data that remains the same across 1288 * reinits. Note the shared data is zero'd when allocated. 1289 */ 1290 1291 ds->magic = VMXNET3_REV1_MAGIC; 1292 1293 /* DriverInfo */ 1294 ds->version = VMXNET3_DRIVER_VERSION; 1295 ds->guest = VMXNET3_GOS_FREEBSD | VMXNET3_GUEST_OS_VERSION | 1296#ifdef __LP64__ 1297 VMXNET3_GOS_64BIT; 1298#else 1299 VMXNET3_GOS_32BIT; 1300#endif 1301 ds->vmxnet3_revision = 1; 1302 ds->upt_version = 1; 1303 1304 /* Misc. conf */ 1305 ds->driver_data = vtophys(sc); 1306 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1307 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1308 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1309 ds->nrxsg_max = sc->vmx_max_rxsegs; 1310 1311 /* Interrupt control. */ 1312 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1313 ds->nintr = sc->vmx_nintrs; 1314 ds->evintr = sc->vmx_event_intr_idx; 1315 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1316 1317 for (i = 0; i < sc->vmx_nintrs; i++) 1318 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1319 1320 /* Receive filter. */ 1321 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1322 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1323 1324 /* Tx queues */ 1325 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1326 txq = &sc->vmx_txq[i]; 1327 txs = txq->vxtxq_ts; 1328 1329 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1330 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1331 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1332 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1333 txs->driver_data = vtophys(txq); 1334 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1335 } 1336 1337 /* Rx queues */ 1338 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1339 rxq = &sc->vmx_rxq[i]; 1340 rxs = rxq->vxrxq_rs; 1341 1342 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1343 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1344 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1345 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1346 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1347 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1348 rxs->driver_data = vtophys(rxq); 1349 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1350 } 1351} 1352 1353static void 1354vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1355{ 1356 struct ifnet *ifp; 1357 1358 ifp = sc->vmx_ifp; 1359 1360 /* Use the current MAC address. */ 1361 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1362 vmxnet3_set_lladdr(sc); 1363 1364 ifp->if_hwassist = 0; 1365 if (ifp->if_capenable & IFCAP_TXCSUM) 1366 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1367 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1368 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1369 if (ifp->if_capenable & IFCAP_TSO4) 1370 ifp->if_hwassist |= CSUM_TSO; 1371 if (ifp->if_capenable & IFCAP_TSO6) 1372 ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */ 1373} 1374 1375static void 1376vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1377{ 1378 struct ifnet *ifp; 1379 struct vmxnet3_driver_shared *ds; 1380 1381 ifp = sc->vmx_ifp; 1382 ds = sc->vmx_ds; 1383 1384 ds->upt_features = 0; 1385 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1386 ds->upt_features |= UPT1_F_VLAN; 1387 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1388 ds->upt_features |= UPT1_F_CSUM; 1389 if (ifp->if_capenable & IFCAP_LRO) 1390 ds->upt_features |= UPT1_F_LRO; 1391 1392 ds->mtu = ifp->if_mtu; 1393 ds->ntxqueue = sc->vmx_ntxqueues; 1394 ds->nrxqueue = sc->vmx_nrxqueues; 1395 1396 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1397 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1398 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1399} 1400 1401static int 1402vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1403{ 1404 int error; 1405 1406 error = vmxnet3_alloc_shared_data(sc); 1407 if (error) 1408 return (error); 1409 1410 error = vmxnet3_alloc_queue_data(sc); 1411 if (error) 1412 return (error); 1413 1414 error = vmxnet3_alloc_mcast_table(sc); 1415 if (error) 1416 return (error); 1417 1418 vmxnet3_init_shared_data(sc); 1419 1420 return (0); 1421} 1422 1423static void 1424vmxnet3_free_data(struct vmxnet3_softc *sc) 1425{ 1426 1427 vmxnet3_free_mcast_table(sc); 1428 vmxnet3_free_queue_data(sc); 1429 vmxnet3_free_shared_data(sc); 1430} 1431 1432static int 1433vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1434{ 1435 device_t dev; 1436 struct ifnet *ifp; 1437 1438 dev = sc->vmx_dev; 1439 1440 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1441 if (ifp == NULL) { 1442 device_printf(dev, "cannot allocate ifnet structure\n"); 1443 return (ENOSPC); 1444 } 1445 1446 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1447#if __FreeBSD_version < 1000025 1448 ifp->if_baudrate = 1000000000; 1449#else 1450 if_initbaudrate(ifp, IF_Gbps(10)); 1451#endif 1452 ifp->if_softc = sc; 1453 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1454 ifp->if_init = vmxnet3_init; 1455 ifp->if_ioctl = vmxnet3_ioctl; 1456 ifp->if_start = vmxnet3_start; 1457 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1458 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1459 IFQ_SET_READY(&ifp->if_snd); 1460 1461 vmxnet3_get_lladdr(sc); 1462 ether_ifattach(ifp, sc->vmx_lladdr); 1463 1464 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1465 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1466 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1467 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; 1468 ifp->if_hwassist |= VMXNET3_CSUM_ALL_OFFLOAD; 1469 1470 ifp->if_capenable = ifp->if_capabilities; 1471 1472 /* 1473 * Capabilities after here are not enabled by default. 1474 */ 1475 1476 ifp->if_capabilities |= IFCAP_LRO; 1477 1478 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 1479 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1480 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1481 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1482 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1483 1484 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1485 vmxnet3_media_status); 1486 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1487 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1488 1489 return (0); 1490} 1491 1492static void 1493vmxnet3_evintr(struct vmxnet3_softc *sc) 1494{ 1495 device_t dev; 1496 struct ifnet *ifp; 1497 struct vmxnet3_txq_shared *ts; 1498 struct vmxnet3_rxq_shared *rs; 1499 uint32_t event; 1500 int reset; 1501 1502 dev = sc->vmx_dev; 1503 ifp = sc->vmx_ifp; 1504 reset = 0; 1505 1506 VMXNET3_CORE_LOCK(sc); 1507 1508 /* Clear events. */ 1509 event = sc->vmx_ds->event; 1510 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1511 1512 if (event & VMXNET3_EVENT_LINK) 1513 vmxnet3_link_status(sc); 1514 1515 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1516 reset = 1; 1517 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1518 ts = sc->vmx_txq[0].vxtxq_ts; 1519 if (ts->stopped != 0) 1520 device_printf(dev, "Tx queue error %#x\n", ts->error); 1521 rs = sc->vmx_rxq[0].vxrxq_rs; 1522 if (rs->stopped != 0) 1523 device_printf(dev, "Rx queue error %#x\n", rs->error); 1524 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1525 } 1526 1527 if (event & VMXNET3_EVENT_DIC) 1528 device_printf(dev, "device implementation change event\n"); 1529 if (event & VMXNET3_EVENT_DEBUG) 1530 device_printf(dev, "debug event\n"); 1531 1532 if (reset != 0) { 1533 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1534 vmxnet3_init_locked(sc); 1535 } 1536 1537 VMXNET3_CORE_UNLOCK(sc); 1538} 1539 1540static void 1541vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1542{ 1543 struct vmxnet3_softc *sc; 1544 struct ifnet *ifp; 1545 struct vmxnet3_txring *txr; 1546 struct vmxnet3_comp_ring *txc; 1547 struct vmxnet3_txcompdesc *txcd; 1548 struct vmxnet3_txbuf *txb; 1549 u_int sop; 1550 1551 sc = txq->vxtxq_sc; 1552 ifp = sc->vmx_ifp; 1553 txr = &txq->vxtxq_cmd_ring; 1554 txc = &txq->vxtxq_comp_ring; 1555 1556 VMXNET3_TXQ_LOCK_ASSERT(txq); 1557 1558 for (;;) { 1559 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1560 if (txcd->gen != txc->vxcr_gen) 1561 break; 1562 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1563 1564 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1565 txc->vxcr_next = 0; 1566 txc->vxcr_gen ^= 1; 1567 } 1568 1569 sop = txr->vxtxr_next; 1570 txb = &txr->vxtxr_txbuf[sop]; 1571 1572 if (txb->vtxb_m != NULL) { 1573 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1574 BUS_DMASYNC_POSTWRITE); 1575 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1576 1577 m_freem(txb->vtxb_m); 1578 txb->vtxb_m = NULL; 1579 1580 ifp->if_opackets++; 1581 } 1582 1583 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1584 } 1585 1586 if (txr->vxtxr_head == txr->vxtxr_next) 1587 txq->vxtxq_watchdog = 0; 1588} 1589 1590static int 1591vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1592{ 1593 struct ifnet *ifp; 1594 struct mbuf *m; 1595 struct vmxnet3_rxdesc *rxd; 1596 struct vmxnet3_rxbuf *rxb; 1597 bus_dma_tag_t tag; 1598 bus_dmamap_t dmap; 1599 bus_dma_segment_t segs[1]; 1600 int idx, clsize, btype, flags, nsegs, error; 1601 1602 ifp = sc->vmx_ifp; 1603 tag = rxr->vxrxr_rxtag; 1604 dmap = rxr->vxrxr_spare_dmap; 1605 idx = rxr->vxrxr_fill; 1606 rxd = &rxr->vxrxr_rxd[idx]; 1607 rxb = &rxr->vxrxr_rxbuf[idx]; 1608 1609#ifdef VMXNET3_FAILPOINTS 1610 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1611 if (rxr->vxrxr_rid != 0) 1612 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1613#endif 1614 1615 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1616 flags = M_PKTHDR; 1617 clsize = MCLBYTES; 1618 btype = VMXNET3_BTYPE_HEAD; 1619 } else { 1620#if __FreeBSD_version < 902001 1621 /* 1622 * These mbufs will never be used for the start of a frame. 1623 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1624 * required the mbuf to always be a packet header. Avoid 1625 * unnecessary mbuf initialization in newer versions where 1626 * that is not the case. 1627 */ 1628 flags = M_PKTHDR; 1629#else 1630 flags = 0; 1631#endif 1632 clsize = MJUMPAGESIZE; 1633 btype = VMXNET3_BTYPE_BODY; 1634 } 1635 1636 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1637 if (m == NULL) { 1638 sc->vmx_stats.vmst_mgetcl_failed++; 1639 return (ENOBUFS); 1640 } 1641 1642 if (btype == VMXNET3_BTYPE_HEAD) { 1643 m->m_len = m->m_pkthdr.len = clsize; 1644 m_adj(m, ETHER_ALIGN); 1645 } else 1646 m->m_len = clsize; 1647 1648 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1649 BUS_DMA_NOWAIT); 1650 if (error) { 1651 m_freem(m); 1652 sc->vmx_stats.vmst_mbuf_load_failed++; 1653 return (error); 1654 } 1655 KASSERT(nsegs == 1, 1656 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1657#if __FreeBSD_version < 902001 1658 if (btype == VMXNET3_BTYPE_BODY) 1659 m->m_flags &= ~M_PKTHDR; 1660#endif 1661 1662 if (rxb->vrxb_m != NULL) { 1663 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1664 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1665 } 1666 1667 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1668 rxb->vrxb_dmamap = dmap; 1669 rxb->vrxb_m = m; 1670 1671 rxd->addr = segs[0].ds_addr; 1672 rxd->len = segs[0].ds_len; 1673 rxd->btype = btype; 1674 rxd->gen = rxr->vxrxr_gen; 1675 1676 vmxnet3_rxr_increment_fill(rxr); 1677 return (0); 1678} 1679 1680static void 1681vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1682 struct vmxnet3_rxring *rxr, int idx) 1683{ 1684 struct vmxnet3_rxdesc *rxd; 1685 1686 rxd = &rxr->vxrxr_rxd[idx]; 1687 rxd->gen = rxr->vxrxr_gen; 1688 vmxnet3_rxr_increment_fill(rxr); 1689} 1690 1691static void 1692vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1693{ 1694 struct vmxnet3_softc *sc; 1695 struct vmxnet3_rxring *rxr; 1696 struct vmxnet3_comp_ring *rxc; 1697 struct vmxnet3_rxcompdesc *rxcd; 1698 int idx, eof; 1699 1700 sc = rxq->vxrxq_sc; 1701 rxc = &rxq->vxrxq_comp_ring; 1702 1703 do { 1704 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1705 if (rxcd->gen != rxc->vxcr_gen) 1706 break; /* Not expected. */ 1707 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1708 1709 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1710 rxc->vxcr_next = 0; 1711 rxc->vxcr_gen ^= 1; 1712 } 1713 1714 idx = rxcd->rxd_idx; 1715 eof = rxcd->eop; 1716 if (rxcd->qid < sc->vmx_nrxqueues) 1717 rxr = &rxq->vxrxq_cmd_ring[0]; 1718 else 1719 rxr = &rxq->vxrxq_cmd_ring[1]; 1720 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1721 } while (!eof); 1722} 1723 1724static void 1725vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1726{ 1727 1728 if (rxcd->ipv4) { 1729 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 1730 if (rxcd->ipcsum_ok) 1731 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1732 } 1733 1734 if (!rxcd->fragment) { 1735 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 1736 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 1737 CSUM_PSEUDO_HDR; 1738 m->m_pkthdr.csum_data = 0xFFFF; 1739 } 1740 } 1741} 1742 1743static void 1744vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 1745 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1746{ 1747 struct vmxnet3_softc *sc; 1748 struct ifnet *ifp; 1749 1750 sc = rxq->vxrxq_sc; 1751 ifp = sc->vmx_ifp; 1752 1753 if (rxcd->error) { 1754 ifp->if_ierrors++; 1755 m_freem(m); 1756 return; 1757 } 1758 1759 if (!rxcd->no_csum) 1760 vmxnet3_rx_csum(rxcd, m); 1761 if (rxcd->vlan) { 1762 m->m_flags |= M_VLANTAG; 1763 m->m_pkthdr.ether_vtag = rxcd->vtag; 1764 } 1765 1766 ifp->if_ipackets++; 1767 VMXNET3_RXQ_UNLOCK(rxq); 1768 (*ifp->if_input)(ifp, m); 1769 VMXNET3_RXQ_LOCK(rxq); 1770} 1771 1772static void 1773vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 1774{ 1775 struct vmxnet3_softc *sc; 1776 struct ifnet *ifp; 1777 struct vmxnet3_rxring *rxr; 1778 struct vmxnet3_comp_ring *rxc; 1779 struct vmxnet3_rxdesc *rxd; 1780 struct vmxnet3_rxcompdesc *rxcd; 1781 struct mbuf *m, *m_head, *m_tail; 1782 int idx, length; 1783 1784 sc = rxq->vxrxq_sc; 1785 ifp = sc->vmx_ifp; 1786 rxc = &rxq->vxrxq_comp_ring; 1787 m_head = m_tail = NULL; 1788 1789 VMXNET3_RXQ_LOCK_ASSERT(rxq); 1790 1791 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1792 return; 1793 1794 for (;;) { 1795 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1796 if (rxcd->gen != rxc->vxcr_gen) 1797 break; 1798 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1799 1800 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1801 rxc->vxcr_next = 0; 1802 rxc->vxcr_gen ^= 1; 1803 } 1804 1805 idx = rxcd->rxd_idx; 1806 length = rxcd->len; 1807 if (rxcd->qid < sc->vmx_nrxqueues) 1808 rxr = &rxq->vxrxq_cmd_ring[0]; 1809 else 1810 rxr = &rxq->vxrxq_cmd_ring[1]; 1811 rxd = &rxr->vxrxr_rxd[idx]; 1812 1813 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 1814 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 1815 __func__, rxcd->qid, idx)); 1816 1817 /* 1818 * The host may skip descriptors. We detect this when this 1819 * descriptor does not match the previous fill index. Catch 1820 * up with the host now. 1821 */ 1822 if (__predict_false(rxr->vxrxr_fill != idx)) { 1823 while (rxr->vxrxr_fill != idx) { 1824 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 1825 rxr->vxrxr_gen; 1826 vmxnet3_rxr_increment_fill(rxr); 1827 } 1828 } 1829 1830 if (rxcd->sop) { 1831 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 1832 ("%s: start of frame w/o head buffer", __func__)); 1833 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 1834 ("%s: start of frame not in ring 0", __func__)); 1835 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 1836 ("%s: start of frame at unexcepted index %d (%d)", 1837 __func__, idx, sc->vmx_rx_max_chain)); 1838 KASSERT(m_head == NULL, 1839 ("%s: duplicate start of frame?", __func__)); 1840 1841 if (length == 0) { 1842 /* Just ignore this descriptor. */ 1843 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1844 goto nextp; 1845 } 1846 1847 if (vmxnet3_newbuf(sc, rxr) != 0) { 1848 ifp->if_iqdrops++; 1849 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1850 if (!rxcd->eop) 1851 vmxnet3_rxq_discard_chain(rxq); 1852 goto nextp; 1853 } 1854 1855 m->m_pkthdr.rcvif = ifp; 1856 m->m_pkthdr.len = m->m_len = length; 1857 m->m_pkthdr.csum_flags = 0; 1858 m_head = m_tail = m; 1859 1860 } else { 1861 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 1862 ("%s: non start of frame w/o body buffer", __func__)); 1863 KASSERT(m_head != NULL, 1864 ("%s: frame not started?", __func__)); 1865 1866 if (vmxnet3_newbuf(sc, rxr) != 0) { 1867 ifp->if_iqdrops++; 1868 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1869 if (!rxcd->eop) 1870 vmxnet3_rxq_discard_chain(rxq); 1871 m_freem(m_head); 1872 m_head = m_tail = NULL; 1873 goto nextp; 1874 } 1875 1876 m->m_len = length; 1877 m_head->m_pkthdr.len += length; 1878 m_tail->m_next = m; 1879 m_tail = m; 1880 } 1881 1882 if (rxcd->eop) { 1883 vmxnet3_rxq_input(rxq, rxcd, m_head); 1884 m_head = m_tail = NULL; 1885 1886 /* Must recheck after dropping the Rx lock. */ 1887 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1888 break; 1889 } 1890 1891nextp: 1892 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 1893 int qid = rxcd->qid; 1894 bus_size_t r; 1895 1896 idx = (idx + 1) % rxr->vxrxr_ndesc; 1897 if (qid >= sc->vmx_nrxqueues) { 1898 qid -= sc->vmx_nrxqueues; 1899 r = VMXNET3_BAR0_RXH2(qid); 1900 } else 1901 r = VMXNET3_BAR0_RXH1(qid); 1902 vmxnet3_write_bar0(sc, r, idx); 1903 } 1904 } 1905} 1906 1907static void 1908vmxnet3_legacy_intr(void *xsc) 1909{ 1910 struct vmxnet3_softc *sc; 1911 struct vmxnet3_rxqueue *rxq; 1912 struct vmxnet3_txqueue *txq; 1913 struct ifnet *ifp; 1914 1915 sc = xsc; 1916 rxq = &sc->vmx_rxq[0]; 1917 txq = &sc->vmx_txq[0]; 1918 ifp = sc->vmx_ifp; 1919 1920 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 1921 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 1922 return; 1923 } 1924 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1925 vmxnet3_disable_all_intrs(sc); 1926 1927 if (sc->vmx_ds->event != 0) 1928 vmxnet3_evintr(sc); 1929 1930 VMXNET3_RXQ_LOCK(rxq); 1931 vmxnet3_rxq_eof(rxq); 1932 VMXNET3_RXQ_UNLOCK(rxq); 1933 1934 VMXNET3_TXQ_LOCK(txq); 1935 vmxnet3_txq_eof(txq); 1936 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1937 vmxnet3_start_locked(ifp); 1938 VMXNET3_TXQ_UNLOCK(txq); 1939 1940 vmxnet3_enable_all_intrs(sc); 1941} 1942 1943static void 1944vmxnet3_txq_intr(void *xtxq) 1945{ 1946 struct vmxnet3_softc *sc; 1947 struct vmxnet3_txqueue *txq; 1948 struct ifnet *ifp; 1949 1950 txq = xtxq; 1951 sc = txq->vxtxq_sc; 1952 ifp = sc->vmx_ifp; 1953 1954 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1955 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 1956 1957 VMXNET3_TXQ_LOCK(txq); 1958 vmxnet3_txq_eof(txq); 1959 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1960 vmxnet3_start_locked(ifp); 1961 VMXNET3_TXQ_UNLOCK(txq); 1962 1963 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 1964} 1965 1966static void 1967vmxnet3_rxq_intr(void *xrxq) 1968{ 1969 struct vmxnet3_softc *sc; 1970 struct vmxnet3_rxqueue *rxq; 1971 1972 rxq = xrxq; 1973 sc = rxq->vxrxq_sc; 1974 1975 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1976 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 1977 1978 VMXNET3_RXQ_LOCK(rxq); 1979 vmxnet3_rxq_eof(rxq); 1980 VMXNET3_RXQ_UNLOCK(rxq); 1981 1982 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 1983} 1984 1985static void 1986vmxnet3_event_intr(void *xsc) 1987{ 1988 struct vmxnet3_softc *sc; 1989 1990 sc = xsc; 1991 1992 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1993 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 1994 1995 if (sc->vmx_ds->event != 0) 1996 vmxnet3_evintr(sc); 1997 1998 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 1999} 2000 2001static void 2002vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2003{ 2004 struct vmxnet3_txring *txr; 2005 struct vmxnet3_txbuf *txb; 2006 int i; 2007 2008 txr = &txq->vxtxq_cmd_ring; 2009 2010 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2011 txb = &txr->vxtxr_txbuf[i]; 2012 2013 if (txb->vtxb_m == NULL) 2014 continue; 2015 2016 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2017 BUS_DMASYNC_POSTWRITE); 2018 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2019 m_freem(txb->vtxb_m); 2020 txb->vtxb_m = NULL; 2021 } 2022} 2023 2024static void 2025vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2026{ 2027 struct vmxnet3_rxring *rxr; 2028 struct vmxnet3_rxbuf *rxb; 2029 int i, j; 2030 2031 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2032 rxr = &rxq->vxrxq_cmd_ring[i]; 2033 2034 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2035 rxb = &rxr->vxrxr_rxbuf[j]; 2036 2037 if (rxb->vrxb_m == NULL) 2038 continue; 2039 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2040 BUS_DMASYNC_POSTREAD); 2041 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2042 m_freem(rxb->vrxb_m); 2043 rxb->vrxb_m = NULL; 2044 } 2045 } 2046} 2047 2048static void 2049vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2050{ 2051 struct vmxnet3_rxqueue *rxq; 2052 struct vmxnet3_txqueue *txq; 2053 int i; 2054 2055 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2056 rxq = &sc->vmx_rxq[i]; 2057 VMXNET3_RXQ_LOCK(rxq); 2058 VMXNET3_RXQ_UNLOCK(rxq); 2059 } 2060 2061 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2062 txq = &sc->vmx_txq[i]; 2063 VMXNET3_TXQ_LOCK(txq); 2064 VMXNET3_TXQ_UNLOCK(txq); 2065 } 2066} 2067 2068static void 2069vmxnet3_stop(struct vmxnet3_softc *sc) 2070{ 2071 struct ifnet *ifp; 2072 int q; 2073 2074 ifp = sc->vmx_ifp; 2075 VMXNET3_CORE_LOCK_ASSERT(sc); 2076 2077 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2078 sc->vmx_link_active = 0; 2079 callout_stop(&sc->vmx_tick); 2080 2081 /* Disable interrupts. */ 2082 vmxnet3_disable_all_intrs(sc); 2083 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2084 2085 vmxnet3_stop_rendezvous(sc); 2086 2087 for (q = 0; q < sc->vmx_ntxqueues; q++) 2088 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2089 for (q = 0; q < sc->vmx_nrxqueues; q++) 2090 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2091 2092 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2093} 2094 2095static void 2096vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2097{ 2098 struct vmxnet3_txring *txr; 2099 struct vmxnet3_comp_ring *txc; 2100 2101 txr = &txq->vxtxq_cmd_ring; 2102 txr->vxtxr_head = 0; 2103 txr->vxtxr_next = 0; 2104 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2105 bzero(txr->vxtxr_txd, 2106 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2107 2108 txc = &txq->vxtxq_comp_ring; 2109 txc->vxcr_next = 0; 2110 txc->vxcr_gen = VMXNET3_INIT_GEN; 2111 bzero(txc->vxcr_u.txcd, 2112 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2113} 2114 2115static int 2116vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2117{ 2118 struct ifnet *ifp; 2119 struct vmxnet3_rxring *rxr; 2120 struct vmxnet3_comp_ring *rxc; 2121 int i, populate, idx, frame_size, error; 2122 2123 ifp = sc->vmx_ifp; 2124 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2125 ifp->if_mtu; 2126 2127 /* 2128 * If the MTU causes us to exceed what a regular sized cluster can 2129 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2130 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2131 * 2132 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2133 * our life easier. We do not support changing the ring size after 2134 * the attach. 2135 */ 2136 if (frame_size <= MCLBYTES) 2137 sc->vmx_rx_max_chain = 1; 2138 else 2139 sc->vmx_rx_max_chain = 2; 2140 2141 /* 2142 * Only populate ring 1 if the configuration will take advantage 2143 * of it. That is either when LRO is enabled or the frame size 2144 * exceeds what ring 0 can contain. 2145 */ 2146 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2147 frame_size <= MCLBYTES + MJUMPAGESIZE) 2148 populate = 1; 2149 else 2150 populate = VMXNET3_RXRINGS_PERQ; 2151 2152 for (i = 0; i < populate; i++) { 2153 rxr = &rxq->vxrxq_cmd_ring[i]; 2154 rxr->vxrxr_fill = 0; 2155 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2156 bzero(rxr->vxrxr_rxd, 2157 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2158 2159 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2160 error = vmxnet3_newbuf(sc, rxr); 2161 if (error) 2162 return (error); 2163 } 2164 } 2165 2166 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2167 rxr = &rxq->vxrxq_cmd_ring[i]; 2168 rxr->vxrxr_fill = 0; 2169 rxr->vxrxr_gen = 0; 2170 bzero(rxr->vxrxr_rxd, 2171 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2172 } 2173 2174 rxc = &rxq->vxrxq_comp_ring; 2175 rxc->vxcr_next = 0; 2176 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2177 bzero(rxc->vxcr_u.rxcd, 2178 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2179 2180 return (0); 2181} 2182 2183static int 2184vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2185{ 2186 device_t dev; 2187 int q, error; 2188 2189 dev = sc->vmx_dev; 2190 2191 for (q = 0; q < sc->vmx_ntxqueues; q++) 2192 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2193 2194 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2195 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2196 if (error) { 2197 device_printf(dev, "cannot populate Rx queue %d\n", q); 2198 return (error); 2199 } 2200 } 2201 2202 return (0); 2203} 2204 2205static int 2206vmxnet3_enable_device(struct vmxnet3_softc *sc) 2207{ 2208 int q; 2209 2210 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2211 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2212 return (1); 2213 } 2214 2215 /* Reset the Rx queue heads. */ 2216 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2217 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2218 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2219 } 2220 2221 return (0); 2222} 2223 2224static void 2225vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2226{ 2227 struct ifnet *ifp; 2228 2229 ifp = sc->vmx_ifp; 2230 2231 vmxnet3_set_rxfilter(sc); 2232 2233 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2234 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2235 sizeof(sc->vmx_ds->vlan_filter)); 2236 else 2237 bzero(sc->vmx_ds->vlan_filter, 2238 sizeof(sc->vmx_ds->vlan_filter)); 2239 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2240} 2241 2242static int 2243vmxnet3_reinit(struct vmxnet3_softc *sc) 2244{ 2245 2246 vmxnet3_reinit_interface(sc); 2247 vmxnet3_reinit_shared_data(sc); 2248 2249 if (vmxnet3_reinit_queues(sc) != 0) 2250 return (ENXIO); 2251 2252 if (vmxnet3_enable_device(sc) != 0) 2253 return (ENXIO); 2254 2255 vmxnet3_reinit_rxfilters(sc); 2256 2257 return (0); 2258} 2259 2260static void 2261vmxnet3_init_locked(struct vmxnet3_softc *sc) 2262{ 2263 struct ifnet *ifp; 2264 2265 ifp = sc->vmx_ifp; 2266 2267 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2268 return; 2269 2270 vmxnet3_stop(sc); 2271 2272 if (vmxnet3_reinit(sc) != 0) { 2273 vmxnet3_stop(sc); 2274 return; 2275 } 2276 2277 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2278 vmxnet3_link_status(sc); 2279 2280 vmxnet3_enable_all_intrs(sc); 2281 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2282} 2283 2284static void 2285vmxnet3_init(void *xsc) 2286{ 2287 struct vmxnet3_softc *sc; 2288 2289 sc = xsc; 2290 2291 VMXNET3_CORE_LOCK(sc); 2292 vmxnet3_init_locked(sc); 2293 VMXNET3_CORE_UNLOCK(sc); 2294} 2295 2296/* 2297 * BMV: Much of this can go away once we finally have offsets in 2298 * the mbuf packet header. Bug andre@. 2299 */ 2300static int 2301vmxnet3_txq_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start) 2302{ 2303 struct ether_vlan_header *evh; 2304 int offset; 2305 2306 evh = mtod(m, struct ether_vlan_header *); 2307 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2308 /* BMV: We should handle nested VLAN tags too. */ 2309 *etype = ntohs(evh->evl_proto); 2310 offset = sizeof(struct ether_vlan_header); 2311 } else { 2312 *etype = ntohs(evh->evl_encap_proto); 2313 offset = sizeof(struct ether_header); 2314 } 2315 2316 switch (*etype) { 2317#if defined(INET) 2318 case ETHERTYPE_IP: { 2319 struct ip *ip, iphdr; 2320 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2321 m_copydata(m, offset, sizeof(struct ip), 2322 (caddr_t) &iphdr); 2323 ip = &iphdr; 2324 } else 2325 ip = (struct ip *)(m->m_data + offset); 2326 *proto = ip->ip_p; 2327 *start = offset + (ip->ip_hl << 2); 2328 break; 2329 } 2330#endif 2331#if defined(INET6) 2332 case ETHERTYPE_IPV6: 2333 *proto = -1; 2334 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2335 /* Assert the network stack sent us a valid packet. */ 2336 KASSERT(*start > offset, 2337 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2338 *start, offset, *proto)); 2339 break; 2340#endif 2341 default: 2342 return (EINVAL); 2343 } 2344 2345 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2346 struct tcphdr *tcp, tcphdr; 2347 2348 if (__predict_false(*proto != IPPROTO_TCP)) { 2349 /* Likely failed to correctly parse the mbuf. */ 2350 return (EINVAL); 2351 } 2352 2353 if (m->m_len < *start + sizeof(struct tcphdr)) { 2354 m_copydata(m, offset, sizeof(struct tcphdr), 2355 (caddr_t) &tcphdr); 2356 tcp = &tcphdr; 2357 } else 2358 tcp = (struct tcphdr *)(m->m_data + *start); 2359 2360 /* 2361 * For TSO, the size of the protocol header is also 2362 * included in the descriptor header size. 2363 */ 2364 *start += (tcp->th_off << 2); 2365 } 2366 2367 return (0); 2368} 2369 2370static int 2371vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2372 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2373{ 2374 struct vmxnet3_txring *txr; 2375 struct mbuf *m; 2376 bus_dma_tag_t tag; 2377 int maxsegs, error; 2378 2379 txr = &txq->vxtxq_cmd_ring; 2380 m = *m0; 2381 tag = txr->vxtxr_txtag; 2382 maxsegs = VMXNET3_TX_MAXSEGS; 2383 2384 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2385 if (error == 0 || error != EFBIG) 2386 return (error); 2387 2388 m = m_collapse(m, M_NOWAIT, maxsegs); 2389 if (m != NULL) { 2390 *m0 = m; 2391 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2392 } else 2393 error = ENOBUFS; 2394 2395 if (error) { 2396 m_freem(*m0); 2397 *m0 = NULL; 2398 } else 2399 txq->vxtxq_sc->vmx_stats.vmst_collapsed++; 2400 2401 return (error); 2402} 2403 2404static void 2405vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2406{ 2407 struct vmxnet3_txring *txr; 2408 2409 txr = &txq->vxtxq_cmd_ring; 2410 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2411} 2412 2413static int 2414vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2415{ 2416 struct vmxnet3_softc *sc; 2417 struct ifnet *ifp; 2418 struct vmxnet3_txring *txr; 2419 struct vmxnet3_txdesc *txd, *sop; 2420 struct mbuf *m; 2421 bus_dmamap_t dmap; 2422 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2423 int i, gen, nsegs, etype, proto, start, error; 2424 2425 sc = txq->vxtxq_sc; 2426 ifp = sc->vmx_ifp; 2427 start = 0; 2428 txd = NULL; 2429 txr = &txq->vxtxq_cmd_ring; 2430 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2431 2432 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2433 if (error) 2434 return (error); 2435 2436 m = *m0; 2437 M_ASSERTPKTHDR(m); 2438 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2439 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2440 2441 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2442 txq->vxtxq_stats.vtxrs_full++; 2443 vmxnet3_txq_unload_mbuf(txq, dmap); 2444 return (ENOSPC); 2445 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2446 error = vmxnet3_txq_offload_ctx(m, &etype, &proto, &start); 2447 if (error) { 2448 txq->vxtxq_stats.vtxrs_offload_failed++; 2449 vmxnet3_txq_unload_mbuf(txq, dmap); 2450 m_freem(m); 2451 *m0 = NULL; 2452 return (error); 2453 } 2454 } 2455 2456 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m = *m0; 2457 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2458 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2459 2460 for (i = 0; i < nsegs; i++) { 2461 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2462 2463 txd->addr = segs[i].ds_addr; 2464 txd->len = segs[i].ds_len; 2465 txd->gen = gen; 2466 txd->dtype = 0; 2467 txd->offload_mode = VMXNET3_OM_NONE; 2468 txd->offload_pos = 0; 2469 txd->hlen = 0; 2470 txd->eop = 0; 2471 txd->compreq = 0; 2472 txd->vtag_mode = 0; 2473 txd->vtag = 0; 2474 2475 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2476 txr->vxtxr_head = 0; 2477 txr->vxtxr_gen ^= 1; 2478 } 2479 gen = txr->vxtxr_gen; 2480 } 2481 txd->eop = 1; 2482 txd->compreq = 1; 2483 2484 if (m->m_flags & M_VLANTAG) { 2485 sop->vtag_mode = 1; 2486 sop->vtag = m->m_pkthdr.ether_vtag; 2487 } 2488 2489 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2490 sop->offload_mode = VMXNET3_OM_TSO; 2491 sop->hlen = start; 2492 sop->offload_pos = m->m_pkthdr.tso_segsz; 2493 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2494 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2495 sop->offload_mode = VMXNET3_OM_CSUM; 2496 sop->hlen = start; 2497 sop->offload_pos = start + m->m_pkthdr.csum_data; 2498 } 2499 2500 /* Finally, change the ownership. */ 2501 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2502 sop->gen ^= 1; 2503 2504 if (++txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2505 txq->vxtxq_ts->npending = 0; 2506 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2507 txr->vxtxr_head); 2508 } 2509 2510 return (0); 2511} 2512 2513static void 2514vmxnet3_start_locked(struct ifnet *ifp) 2515{ 2516 struct vmxnet3_softc *sc; 2517 struct vmxnet3_txqueue *txq; 2518 struct vmxnet3_txring *txr; 2519 struct mbuf *m_head; 2520 int tx; 2521 2522 sc = ifp->if_softc; 2523 txq = &sc->vmx_txq[0]; 2524 txr = &txq->vxtxq_cmd_ring; 2525 tx = 0; 2526 2527 VMXNET3_TXQ_LOCK_ASSERT(txq); 2528 2529 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2530 sc->vmx_link_active == 0) 2531 return; 2532 2533 while (VMXNET3_TXRING_AVAIL(txr) > 0) { 2534 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2535 if (m_head == NULL) 2536 break; 2537 2538 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2539 if (m_head != NULL) 2540 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2541 break; 2542 } 2543 2544 tx++; 2545 ETHER_BPF_MTAP(ifp, m_head); 2546 } 2547 2548 if (tx > 0) { 2549 if (txq->vxtxq_ts->npending > 0) { 2550 txq->vxtxq_ts->npending = 0; 2551 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2552 txr->vxtxr_head); 2553 } 2554 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2555 } 2556} 2557 2558static void 2559vmxnet3_start(struct ifnet *ifp) 2560{ 2561 struct vmxnet3_softc *sc; 2562 struct vmxnet3_txqueue *txq; 2563 2564 sc = ifp->if_softc; 2565 txq = &sc->vmx_txq[0]; 2566 2567 VMXNET3_TXQ_LOCK(txq); 2568 vmxnet3_start_locked(ifp); 2569 VMXNET3_TXQ_UNLOCK(txq); 2570} 2571 2572static void 2573vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 2574{ 2575 struct ifnet *ifp; 2576 int idx, bit; 2577 2578 ifp = sc->vmx_ifp; 2579 idx = (tag >> 5) & 0x7F; 2580 bit = tag & 0x1F; 2581 2582 if (tag == 0 || tag > 4095) 2583 return; 2584 2585 VMXNET3_CORE_LOCK(sc); 2586 2587 /* Update our private VLAN bitvector. */ 2588 if (add) 2589 sc->vmx_vlan_filter[idx] |= (1 << bit); 2590 else 2591 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 2592 2593 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 2594 if (add) 2595 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 2596 else 2597 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 2598 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2599 } 2600 2601 VMXNET3_CORE_UNLOCK(sc); 2602} 2603 2604static void 2605vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2606{ 2607 2608 if (ifp->if_softc == arg) 2609 vmxnet3_update_vlan_filter(arg, 1, tag); 2610} 2611 2612static void 2613vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2614{ 2615 2616 if (ifp->if_softc == arg) 2617 vmxnet3_update_vlan_filter(arg, 0, tag); 2618} 2619 2620static void 2621vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 2622{ 2623 struct ifnet *ifp; 2624 struct vmxnet3_driver_shared *ds; 2625 struct ifmultiaddr *ifma; 2626 u_int mode; 2627 2628 ifp = sc->vmx_ifp; 2629 ds = sc->vmx_ds; 2630 2631 mode = VMXNET3_RXMODE_UCAST; 2632 if (ifp->if_flags & IFF_BROADCAST) 2633 mode |= VMXNET3_RXMODE_BCAST; 2634 if (ifp->if_flags & IFF_PROMISC) 2635 mode |= VMXNET3_RXMODE_PROMISC; 2636 if (ifp->if_flags & IFF_ALLMULTI) 2637 mode |= VMXNET3_RXMODE_ALLMULTI; 2638 else { 2639 int cnt = 0, overflow = 0; 2640 2641 if_maddr_rlock(ifp); 2642 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2643 if (ifma->ifma_addr->sa_family != AF_LINK) 2644 continue; 2645 else if (cnt == VMXNET3_MULTICAST_MAX) { 2646 overflow = 1; 2647 break; 2648 } 2649 2650 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2651 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 2652 cnt++; 2653 } 2654 if_maddr_runlock(ifp); 2655 2656 if (overflow != 0) { 2657 cnt = 0; 2658 mode |= VMXNET3_RXMODE_ALLMULTI; 2659 } else if (cnt > 0) 2660 mode |= VMXNET3_RXMODE_MCAST; 2661 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 2662 } 2663 2664 ds->rxmode = mode; 2665 2666 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 2667 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 2668} 2669 2670static int 2671vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 2672{ 2673 struct ifnet *ifp; 2674 2675 ifp = sc->vmx_ifp; 2676 2677 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 2678 return (EINVAL); 2679 2680 ifp->if_mtu = mtu; 2681 2682 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2683 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2684 vmxnet3_init_locked(sc); 2685 } 2686 2687 return (0); 2688} 2689 2690static int 2691vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 2692{ 2693 struct vmxnet3_softc *sc; 2694 struct ifreq *ifr; 2695 int reinit, mask, error; 2696 2697 sc = ifp->if_softc; 2698 ifr = (struct ifreq *) data; 2699 error = 0; 2700 2701 switch (cmd) { 2702 case SIOCSIFMTU: 2703 if (ifp->if_mtu != ifr->ifr_mtu) { 2704 VMXNET3_CORE_LOCK(sc); 2705 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 2706 VMXNET3_CORE_UNLOCK(sc); 2707 } 2708 break; 2709 2710 case SIOCSIFFLAGS: 2711 VMXNET3_CORE_LOCK(sc); 2712 if (ifp->if_flags & IFF_UP) { 2713 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2714 if ((ifp->if_flags ^ sc->vmx_if_flags) & 2715 (IFF_PROMISC | IFF_ALLMULTI)) { 2716 vmxnet3_set_rxfilter(sc); 2717 } 2718 } else 2719 vmxnet3_init_locked(sc); 2720 } else { 2721 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2722 vmxnet3_stop(sc); 2723 } 2724 sc->vmx_if_flags = ifp->if_flags; 2725 VMXNET3_CORE_UNLOCK(sc); 2726 break; 2727 2728 case SIOCADDMULTI: 2729 case SIOCDELMULTI: 2730 VMXNET3_CORE_LOCK(sc); 2731 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2732 vmxnet3_set_rxfilter(sc); 2733 VMXNET3_CORE_UNLOCK(sc); 2734 break; 2735 2736 case SIOCSIFMEDIA: 2737 case SIOCGIFMEDIA: 2738 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 2739 break; 2740 2741 case SIOCSIFCAP: 2742 VMXNET3_CORE_LOCK(sc); 2743 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 2744 2745 if (mask & IFCAP_TXCSUM) 2746 ifp->if_capenable ^= IFCAP_TXCSUM; 2747 if (mask & IFCAP_TXCSUM_IPV6) 2748 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 2749 if (mask & IFCAP_TSO4) 2750 ifp->if_capenable ^= IFCAP_TSO4; 2751 if (mask & IFCAP_TSO6) 2752 ifp->if_capenable ^= IFCAP_TSO6; 2753 2754 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 2755 IFCAP_VLAN_HWFILTER)) { 2756 /* These Rx features require us to renegotiate. */ 2757 reinit = 1; 2758 2759 if (mask & IFCAP_RXCSUM) 2760 ifp->if_capenable ^= IFCAP_RXCSUM; 2761 if (mask & IFCAP_RXCSUM_IPV6) 2762 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 2763 if (mask & IFCAP_LRO) 2764 ifp->if_capenable ^= IFCAP_LRO; 2765 if (mask & IFCAP_VLAN_HWFILTER) 2766 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 2767 } else 2768 reinit = 0; 2769 2770 if (mask & IFCAP_VLAN_HWTSO) 2771 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2772 if (mask & IFCAP_VLAN_HWTAGGING) 2773 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2774 2775 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2776 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2777 vmxnet3_init_locked(sc); 2778 } 2779 2780 VMXNET3_CORE_UNLOCK(sc); 2781 VLAN_CAPABILITIES(ifp); 2782 break; 2783 2784 default: 2785 error = ether_ioctl(ifp, cmd, data); 2786 break; 2787 } 2788 2789 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 2790 2791 return (error); 2792} 2793 2794static int 2795vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 2796{ 2797 struct vmxnet3_softc *sc; 2798 2799 sc = txq->vxtxq_sc; 2800 2801 VMXNET3_TXQ_LOCK(txq); 2802 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 2803 VMXNET3_TXQ_UNLOCK(txq); 2804 return (0); 2805 } 2806 VMXNET3_TXQ_UNLOCK(txq); 2807 2808 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 2809 txq->vxtxq_id); 2810 return (1); 2811} 2812 2813static void 2814vmxnet3_refresh_stats(struct vmxnet3_softc *sc) 2815{ 2816 2817 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 2818} 2819 2820static void 2821vmxnet3_tick(void *xsc) 2822{ 2823 struct vmxnet3_softc *sc; 2824 struct ifnet *ifp; 2825 int i, timedout; 2826 2827 sc = xsc; 2828 ifp = sc->vmx_ifp; 2829 timedout = 0; 2830 2831 VMXNET3_CORE_LOCK_ASSERT(sc); 2832 vmxnet3_refresh_stats(sc); 2833 2834 for (i = 0; i < sc->vmx_ntxqueues; i++) 2835 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 2836 2837 if (timedout != 0) { 2838 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2839 vmxnet3_init_locked(sc); 2840 } else 2841 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2842} 2843 2844static int 2845vmxnet3_link_is_up(struct vmxnet3_softc *sc) 2846{ 2847 uint32_t status; 2848 2849 /* Also update the link speed while here. */ 2850 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 2851 sc->vmx_link_speed = status >> 16; 2852 return !!(status & 0x1); 2853} 2854 2855static void 2856vmxnet3_link_status(struct vmxnet3_softc *sc) 2857{ 2858 struct ifnet *ifp; 2859 int link; 2860 2861 ifp = sc->vmx_ifp; 2862 link = vmxnet3_link_is_up(sc); 2863 2864 if (link != 0 && sc->vmx_link_active == 0) { 2865 sc->vmx_link_active = 1; 2866 if_link_state_change(ifp, LINK_STATE_UP); 2867 } else if (link == 0 && sc->vmx_link_active != 0) { 2868 sc->vmx_link_active = 0; 2869 if_link_state_change(ifp, LINK_STATE_DOWN); 2870 } 2871} 2872 2873static void 2874vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2875{ 2876 struct vmxnet3_softc *sc; 2877 2878 sc = ifp->if_softc; 2879 2880 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 2881 ifmr->ifm_status = IFM_AVALID; 2882 2883 VMXNET3_CORE_LOCK(sc); 2884 if (vmxnet3_link_is_up(sc) != 0) 2885 ifmr->ifm_status |= IFM_ACTIVE; 2886 else 2887 ifmr->ifm_status |= IFM_NONE; 2888 VMXNET3_CORE_UNLOCK(sc); 2889} 2890 2891static int 2892vmxnet3_media_change(struct ifnet *ifp) 2893{ 2894 2895 /* Ignore. */ 2896 return (0); 2897} 2898 2899static void 2900vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 2901{ 2902 uint32_t ml, mh; 2903 2904 ml = sc->vmx_lladdr[0]; 2905 ml |= sc->vmx_lladdr[1] << 8; 2906 ml |= sc->vmx_lladdr[2] << 16; 2907 ml |= sc->vmx_lladdr[3] << 24; 2908 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 2909 2910 mh = sc->vmx_lladdr[4]; 2911 mh |= sc->vmx_lladdr[5] << 8; 2912 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 2913} 2914 2915static void 2916vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 2917{ 2918 uint32_t ml, mh; 2919 2920 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 2921 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 2922 2923 sc->vmx_lladdr[0] = ml; 2924 sc->vmx_lladdr[1] = ml >> 8; 2925 sc->vmx_lladdr[2] = ml >> 16; 2926 sc->vmx_lladdr[3] = ml >> 24; 2927 sc->vmx_lladdr[4] = mh; 2928 sc->vmx_lladdr[5] = mh >> 8; 2929} 2930 2931static void 2932vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 2933 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2934{ 2935 struct sysctl_oid *node, *txsnode; 2936 struct sysctl_oid_list *list, *txslist; 2937 struct vmxnet3_txq_stats *stats; 2938 struct UPT1_TxStats *txstats; 2939 char namebuf[16]; 2940 2941 stats = &txq->vxtxq_stats; 2942 txstats = &txq->vxtxq_ts->stats; 2943 2944 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 2945 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 2946 NULL, "Transmit Queue"); 2947 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 2948 2949 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 2950 &stats->vtxrs_full, "Tx ring full"); 2951 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 2952 &stats->vtxrs_offload_failed, "Tx checksum offload failed"); 2953 2954 /* 2955 * Add statistics reported by the host. These are updated once 2956 * per second. 2957 */ 2958 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 2959 NULL, "Host Statistics"); 2960 txslist = SYSCTL_CHILDREN(txsnode); 2961 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 2962 &txstats->TSO_packets, "TSO packets"); 2963 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 2964 &txstats->TSO_bytes, "TSO bytes"); 2965 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 2966 &txstats->ucast_packets, "Unicast packets"); 2967 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 2968 &txstats->ucast_bytes, "Unicast bytes"); 2969 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 2970 &txstats->mcast_packets, "Multicast packets"); 2971 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 2972 &txstats->mcast_bytes, "Multicast bytes"); 2973 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 2974 &txstats->error, "Errors"); 2975 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 2976 &txstats->discard, "Discards"); 2977} 2978 2979static void 2980vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 2981 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2982{ 2983 struct sysctl_oid *node, *rxsnode; 2984 struct sysctl_oid_list *list, *rxslist; 2985 struct vmxnet3_rxq_stats *stats; 2986 struct UPT1_RxStats *rxstats; 2987 char namebuf[16]; 2988 2989 stats = &rxq->vxrxq_stats; 2990 rxstats = &rxq->vxrxq_rs->stats; 2991 2992 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 2993 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 2994 NULL, "Receive Queue"); 2995 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 2996 2997 /* 2998 * Add statistics reported by the host. These are updated once 2999 * per second. 3000 */ 3001 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3002 NULL, "Host Statistics"); 3003 rxslist = SYSCTL_CHILDREN(rxsnode); 3004 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3005 &rxstats->LRO_packets, "LRO packets"); 3006 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3007 &rxstats->LRO_bytes, "LRO bytes"); 3008 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3009 &rxstats->ucast_packets, "Unicast packets"); 3010 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3011 &rxstats->ucast_bytes, "Unicast bytes"); 3012 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3013 &rxstats->mcast_packets, "Multicast packets"); 3014 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3015 &rxstats->mcast_bytes, "Multicast bytes"); 3016 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3017 &rxstats->bcast_packets, "Broadcast packets"); 3018 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3019 &rxstats->bcast_bytes, "Broadcast bytes"); 3020 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3021 &rxstats->nobuffer, "No buffer"); 3022 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3023 &rxstats->error, "Errors"); 3024} 3025 3026#ifdef VMXNET3_DEBUG_SYSCTL 3027static void 3028vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3029 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3030{ 3031 struct sysctl_oid *node; 3032 struct sysctl_oid_list *list; 3033 int i; 3034 3035 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3036 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3037 3038 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3039 "debug", CTLFLAG_RD, NULL, ""); 3040 list = SYSCTL_CHILDREN(node); 3041 3042 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3043 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3044 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3045 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3046 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3047 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3048 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3049 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3050 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3051 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3052 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3053 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3054 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3055 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3056 } 3057 3058 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3059 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3060 3061 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3062 "debug", CTLFLAG_RD, NULL, ""); 3063 list = SYSCTL_CHILDREN(node); 3064 3065 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3066 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3067 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3068 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3069 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3070 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3071 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3072 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3073 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3074 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3075 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3076 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3077 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3078 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3079 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3080 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3081 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3082 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3083 } 3084} 3085#endif 3086 3087static void 3088vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3089 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3090{ 3091 int i; 3092 3093 for (i = 0; i < sc->vmx_ntxqueues; i++) 3094 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3095 for (i = 0; i < sc->vmx_nrxqueues; i++) 3096 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3097 3098#ifdef VMXNET3_DEBUG_SYSCTL 3099 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3100#endif 3101} 3102 3103static void 3104vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3105{ 3106 device_t dev; 3107 struct vmxnet3_statistics *stats; 3108 struct sysctl_ctx_list *ctx; 3109 struct sysctl_oid *tree; 3110 struct sysctl_oid_list *child; 3111 3112 dev = sc->vmx_dev; 3113 ctx = device_get_sysctl_ctx(dev); 3114 tree = device_get_sysctl_tree(dev); 3115 child = SYSCTL_CHILDREN(tree); 3116 3117 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3118 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3119 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3120 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3121 3122 stats = &sc->vmx_stats; 3123 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "collapsed", CTLFLAG_RD, 3124 &stats->vmst_collapsed, 0, "Tx mbuf chains collapsed"); 3125 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3126 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3127 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3128 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3129 3130 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3131} 3132 3133static void 3134vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3135{ 3136 3137 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3138} 3139 3140static uint32_t 3141vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3142{ 3143 3144 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3145} 3146 3147static void 3148vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3149{ 3150 3151 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3152} 3153 3154static void 3155vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3156{ 3157 3158 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3159} 3160 3161static uint32_t 3162vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3163{ 3164 3165 vmxnet3_write_cmd(sc, cmd); 3166 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3167 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3168 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3169} 3170 3171static void 3172vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3173{ 3174 3175 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3176} 3177 3178static void 3179vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3180{ 3181 3182 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3183} 3184 3185static void 3186vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3187{ 3188 int i; 3189 3190 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3191 for (i = 0; i < sc->vmx_nintrs; i++) 3192 vmxnet3_enable_intr(sc, i); 3193} 3194 3195static void 3196vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3197{ 3198 int i; 3199 3200 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3201 for (i = 0; i < sc->vmx_nintrs; i++) 3202 vmxnet3_disable_intr(sc, i); 3203} 3204 3205static void 3206vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3207{ 3208 bus_addr_t *baddr = arg; 3209 3210 if (error == 0) 3211 *baddr = segs->ds_addr; 3212} 3213 3214static int 3215vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3216 struct vmxnet3_dma_alloc *dma) 3217{ 3218 device_t dev; 3219 int error; 3220 3221 dev = sc->vmx_dev; 3222 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3223 3224 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3225 align, 0, /* alignment, bounds */ 3226 BUS_SPACE_MAXADDR, /* lowaddr */ 3227 BUS_SPACE_MAXADDR, /* highaddr */ 3228 NULL, NULL, /* filter, filterarg */ 3229 size, /* maxsize */ 3230 1, /* nsegments */ 3231 size, /* maxsegsize */ 3232 BUS_DMA_ALLOCNOW, /* flags */ 3233 NULL, /* lockfunc */ 3234 NULL, /* lockfuncarg */ 3235 &dma->dma_tag); 3236 if (error) { 3237 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3238 goto fail; 3239 } 3240 3241 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3242 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3243 if (error) { 3244 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3245 goto fail; 3246 } 3247 3248 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3249 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3250 if (error) { 3251 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3252 goto fail; 3253 } 3254 3255 dma->dma_size = size; 3256 3257fail: 3258 if (error) 3259 vmxnet3_dma_free(sc, dma); 3260 3261 return (error); 3262} 3263 3264static void 3265vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3266{ 3267 3268 if (dma->dma_tag != NULL) { 3269 if (dma->dma_map != NULL) { 3270 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3271 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3272 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3273 } 3274 3275 if (dma->dma_vaddr != NULL) { 3276 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3277 dma->dma_map); 3278 } 3279 3280 bus_dma_tag_destroy(dma->dma_tag); 3281 } 3282 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3283} 3284 3285/* 3286 * Since this is a purely paravirtualized device, we do not have 3287 * to worry about DMA coherency. But at times, we must make sure 3288 * both the compiler and CPU do not reorder memory operations. 3289 */ 3290static inline void 3291vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3292{ 3293 3294 switch (type) { 3295 case VMXNET3_BARRIER_RD: 3296 rmb(); 3297 break; 3298 case VMXNET3_BARRIER_WR: 3299 wmb(); 3300 break; 3301 case VMXNET3_BARRIER_RDWR: 3302 mb(); 3303 break; 3304 default: 3305 panic("%s: bad barrier type %d", __func__, type); 3306 } 3307} 3308