if_vtnet.c revision 304067
1148330Snetchild/*- 2148330Snetchild * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3148330Snetchild * All rights reserved. 4148330Snetchild * 5148330Snetchild * Redistribution and use in source and binary forms, with or without 6148330Snetchild * modification, are permitted provided that the following conditions 7148330Snetchild * are met: 8148330Snetchild * 1. Redistributions of source code must retain the above copyright 9148330Snetchild * notice unmodified, this list of conditions, and the following 10148330Snetchild * disclaimer. 11148330Snetchild * 2. Redistributions in binary form must reproduce the above copyright 12148330Snetchild * notice, this list of conditions and the following disclaimer in the 13148330Snetchild * documentation and/or other materials provided with the distribution. 14148543Snetchild * 15148543Snetchild * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16148330Snetchild * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17180257Sjhb * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18180257Sjhb * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19180257Sjhb * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20180257Sjhb * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21180257Sjhb * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22180257Sjhb * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23180248Smarcel * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24180248Smarcel * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25180248Smarcel */ 26180248Smarcel 27180248Smarcel/* Driver for VirtIO network devices. */ 28180230Smarcel 29180230Smarcel#include <sys/cdefs.h> 30180230Smarcel__FBSDID("$FreeBSD: stable/11/sys/dev/virtio/network/if_vtnet.c 304067 2016-08-14 00:40:17Z smh $"); 31180230Smarcel 32180230Smarcel#include <sys/param.h> 33180230Smarcel#include <sys/eventhandler.h> 34180230Smarcel#include <sys/systm.h> 35180230Smarcel#include <sys/kernel.h> 36180159Sdanger#include <sys/sockio.h> 37180159Sdanger#include <sys/mbuf.h> 38180159Sdanger#include <sys/malloc.h> 39179784Sed#include <sys/module.h> 40179784Sed#include <sys/socket.h> 41179784Sed#include <sys/sysctl.h> 42179784Sed#include <sys/random.h> 43179784Sed#include <sys/sglist.h> 44179692Smarcel#include <sys/lock.h> 45179692Smarcel#include <sys/mutex.h> 46179692Smarcel#include <sys/taskqueue.h> 47179315Sbz#include <sys/smp.h> 48179315Sbz#include <machine/smp.h> 49179315Sbz 50179315Sbz#include <vm/uma.h> 51179315Sbz 52179315Sbz#include <net/ethernet.h> 53179315Sbz#include <net/if.h> 54179315Sbz#include <net/if_var.h> 55179315Sbz#include <net/if_arp.h> 56179315Sbz#include <net/if_dl.h> 57179315Sbz#include <net/if_types.h> 58179315Sbz#include <net/if_media.h> 59179315Sbz#include <net/if_vlan_var.h> 60179315Sbz 61179315Sbz#include <net/bpf.h> 62179315Sbz 63179315Sbz#include <netinet/in_systm.h> 64179315Sbz#include <netinet/in.h> 65179315Sbz#include <netinet/ip.h> 66179315Sbz#include <netinet/ip6.h> 67179315Sbz#include <netinet6/ip6_var.h> 68179315Sbz#include <netinet/udp.h> 69179315Sbz#include <netinet/tcp.h> 70179315Sbz#include <netinet/sctp.h> 71179315Sbz 72179315Sbz#include <machine/bus.h> 73179315Sbz#include <machine/resource.h> 74179315Sbz#include <sys/bus.h> 75179315Sbz#include <sys/rman.h> 76179315Sbz 77179315Sbz#include <dev/virtio/virtio.h> 78179315Sbz#include <dev/virtio/virtqueue.h> 79179315Sbz#include <dev/virtio/network/virtio_net.h> 80179315Sbz#include <dev/virtio/network/if_vtnetvar.h> 81179315Sbz 82179315Sbz#include "virtio_if.h" 83179315Sbz 84179315Sbz#include "opt_inet.h" 85179315Sbz#include "opt_inet6.h" 86179315Sbz 87179315Sbzstatic int vtnet_modevent(module_t, int, void *); 88179315Sbz 89179315Sbzstatic int vtnet_probe(device_t); 90179315Sbzstatic int vtnet_attach(device_t); 91179315Sbzstatic int vtnet_detach(device_t); 92179315Sbzstatic int vtnet_suspend(device_t); 93179315Sbzstatic int vtnet_resume(device_t); 94179315Sbzstatic int vtnet_shutdown(device_t); 95179315Sbzstatic int vtnet_attach_completed(device_t); 96179315Sbzstatic int vtnet_config_change(device_t); 97179315Sbz 98179315Sbzstatic void vtnet_negotiate_features(struct vtnet_softc *); 99179315Sbzstatic void vtnet_setup_features(struct vtnet_softc *); 100179315Sbzstatic int vtnet_init_rxq(struct vtnet_softc *, int); 101179315Sbzstatic int vtnet_init_txq(struct vtnet_softc *, int); 102179315Sbzstatic int vtnet_alloc_rxtx_queues(struct vtnet_softc *); 103179315Sbzstatic void vtnet_free_rxtx_queues(struct vtnet_softc *); 104179315Sbzstatic int vtnet_alloc_rx_filters(struct vtnet_softc *); 105179315Sbzstatic void vtnet_free_rx_filters(struct vtnet_softc *); 106179315Sbzstatic int vtnet_alloc_virtqueues(struct vtnet_softc *); 107179315Sbzstatic int vtnet_setup_interface(struct vtnet_softc *); 108179315Sbzstatic int vtnet_change_mtu(struct vtnet_softc *, int); 109179315Sbzstatic int vtnet_ioctl(struct ifnet *, u_long, caddr_t); 110179315Sbzstatic uint64_t vtnet_get_counter(struct ifnet *, ift_counter); 111179315Sbz 112179315Sbzstatic int vtnet_rxq_populate(struct vtnet_rxq *); 113179315Sbzstatic void vtnet_rxq_free_mbufs(struct vtnet_rxq *); 114179315Sbzstatic struct mbuf * 115179315Sbz vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **); 116179315Sbzstatic int vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *, 117179315Sbz struct mbuf *, int); 118179315Sbzstatic int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); 119179315Sbzstatic int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); 120179315Sbzstatic int vtnet_rxq_new_buf(struct vtnet_rxq *); 121179315Sbzstatic int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, 122179315Sbz struct virtio_net_hdr *); 123179315Sbzstatic void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); 124179315Sbzstatic void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); 125179315Sbzstatic int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); 126179315Sbzstatic void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *, 127179315Sbz struct virtio_net_hdr *); 128179315Sbzstatic int vtnet_rxq_eof(struct vtnet_rxq *); 129179315Sbzstatic void vtnet_rx_vq_intr(void *); 130179315Sbzstatic void vtnet_rxq_tq_intr(void *, int); 131179315Sbz 132179315Sbzstatic int vtnet_txq_below_threshold(struct vtnet_txq *); 133179315Sbzstatic int vtnet_txq_notify(struct vtnet_txq *); 134179315Sbzstatic void vtnet_txq_free_mbufs(struct vtnet_txq *); 135179315Sbzstatic int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, 136179315Sbz int *, int *, int *); 137179315Sbzstatic int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, 138179315Sbz int, struct virtio_net_hdr *); 139179315Sbzstatic struct mbuf * 140179315Sbz vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, 141179315Sbz struct virtio_net_hdr *); 142179315Sbzstatic int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, 143179315Sbz struct vtnet_tx_header *); 144179315Sbzstatic int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **); 145179315Sbz#ifdef VTNET_LEGACY_TX 146179315Sbzstatic void vtnet_start_locked(struct vtnet_txq *, struct ifnet *); 147179315Sbzstatic void vtnet_start(struct ifnet *); 148179315Sbz#else 149179315Sbzstatic int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *); 150179315Sbzstatic int vtnet_txq_mq_start(struct ifnet *, struct mbuf *); 151179315Sbzstatic void vtnet_txq_tq_deferred(void *, int); 152179315Sbz#endif 153179315Sbzstatic void vtnet_txq_start(struct vtnet_txq *); 154179315Sbzstatic void vtnet_txq_tq_intr(void *, int); 155179315Sbzstatic int vtnet_txq_eof(struct vtnet_txq *); 156179315Sbzstatic void vtnet_tx_vq_intr(void *); 157179315Sbzstatic void vtnet_tx_start_all(struct vtnet_softc *); 158179315Sbz 159179315Sbz#ifndef VTNET_LEGACY_TX 160179315Sbzstatic void vtnet_qflush(struct ifnet *); 161179361Santoine#endif 162179361Santoine 163179361Santoinestatic int vtnet_watchdog(struct vtnet_txq *); 164179361Santoinestatic void vtnet_accum_stats(struct vtnet_softc *, 165179361Santoine struct vtnet_rxq_stats *, struct vtnet_txq_stats *); 166179361Santoinestatic void vtnet_tick(void *); 167179361Santoine 168179361Santoinestatic void vtnet_start_taskqueues(struct vtnet_softc *); 169179361Santoinestatic void vtnet_free_taskqueues(struct vtnet_softc *); 170179361Santoinestatic void vtnet_drain_taskqueues(struct vtnet_softc *); 171179361Santoine 172179361Santoinestatic void vtnet_drain_rxtx_queues(struct vtnet_softc *); 173179361Santoinestatic void vtnet_stop_rendezvous(struct vtnet_softc *); 174179361Santoinestatic void vtnet_stop(struct vtnet_softc *); 175179361Santoinestatic int vtnet_virtio_reinit(struct vtnet_softc *); 176179361Santoinestatic void vtnet_init_rx_filters(struct vtnet_softc *); 177179361Santoinestatic int vtnet_init_rx_queues(struct vtnet_softc *); 178179361Santoinestatic int vtnet_init_tx_queues(struct vtnet_softc *); 179179361Santoinestatic int vtnet_init_rxtx_queues(struct vtnet_softc *); 180179361Santoinestatic void vtnet_set_active_vq_pairs(struct vtnet_softc *); 181178924Santoinestatic int vtnet_reinit(struct vtnet_softc *); 182178924Santoinestatic void vtnet_init_locked(struct vtnet_softc *); 183178924Santoinestatic void vtnet_init(void *); 184178924Santoine 185178924Santoinestatic void vtnet_free_ctrl_vq(struct vtnet_softc *); 186177831Sflzstatic void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *, 187177831Sflz struct sglist *, int, int); 188177831Sflzstatic int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *); 189177831Sflzstatic int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t); 190177831Sflzstatic int vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int); 191178331Santoinestatic int vtnet_set_promisc(struct vtnet_softc *, int); 192178331Santoinestatic int vtnet_set_allmulti(struct vtnet_softc *, int); 193178331Santoinestatic void vtnet_attach_disable_promisc(struct vtnet_softc *); 194178331Santoinestatic void vtnet_rx_filter(struct vtnet_softc *); 195178331Santoinestatic void vtnet_rx_filter_mac(struct vtnet_softc *); 196178331Santoinestatic int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); 197178331Santoinestatic void vtnet_rx_filter_vlan(struct vtnet_softc *); 198178331Santoinestatic void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t); 199178331Santoinestatic void vtnet_register_vlan(void *, struct ifnet *, uint16_t); 200178331Santoinestatic void vtnet_unregister_vlan(void *, struct ifnet *, uint16_t); 201178331Santoine 202178331Santoinestatic int vtnet_is_link_up(struct vtnet_softc *); 203178331Santoinestatic void vtnet_update_link_status(struct vtnet_softc *); 204178331Santoinestatic int vtnet_ifmedia_upd(struct ifnet *); 205178331Santoinestatic void vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *); 206178331Santoinestatic void vtnet_get_hwaddr(struct vtnet_softc *); 207178924Santoinestatic void vtnet_set_hwaddr(struct vtnet_softc *); 208178924Santoinestatic void vtnet_vlan_tag_remove(struct mbuf *); 209178924Santoinestatic void vtnet_set_rx_process_limit(struct vtnet_softc *); 210178924Santoinestatic void vtnet_set_tx_intr_threshold(struct vtnet_softc *); 211176422Sthompsa 212176422Sthompsastatic void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, 213175690Sbrueffer struct sysctl_oid_list *, struct vtnet_rxq *); 214175690Sbruefferstatic void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, 215175690Sbrueffer struct sysctl_oid_list *, struct vtnet_txq *); 216175576Sattiliostatic void vtnet_setup_queue_sysctl(struct vtnet_softc *); 217175576Sattiliostatic void vtnet_setup_sysctl(struct vtnet_softc *); 218175227Sjhb 219175227Sjhbstatic int vtnet_rxq_enable_intr(struct vtnet_rxq *); 220175227Sjhbstatic void vtnet_rxq_disable_intr(struct vtnet_rxq *); 221174426Sdougbstatic int vtnet_txq_enable_intr(struct vtnet_txq *); 222174426Sdougbstatic void vtnet_txq_disable_intr(struct vtnet_txq *); 223174426Sdougbstatic void vtnet_enable_rx_interrupts(struct vtnet_softc *); 224177153Sbruefferstatic void vtnet_enable_tx_interrupts(struct vtnet_softc *); 225177153Sbruefferstatic void vtnet_enable_interrupts(struct vtnet_softc *); 226174092Sbrooksstatic void vtnet_disable_rx_interrupts(struct vtnet_softc *); 227174092Sbrooksstatic void vtnet_disable_tx_interrupts(struct vtnet_softc *); 228174092Sbrooksstatic void vtnet_disable_interrupts(struct vtnet_softc *); 229174092Sbrooks 230176956Santoinestatic int vtnet_tunable_int(struct vtnet_softc *, const char *, int); 231176956Santoine 232176956Santoine/* Tunables. */ 233176956Santoinestatic SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD, 0, "VNET driver parameters"); 234174092Sbrooksstatic int vtnet_csum_disable = 0; 235174061SjbTUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable); 236174061SjbSYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN, 237175227Sjhb &vtnet_csum_disable, 0, "Disables receive and send checksum offload"); 238175227Sjhbstatic int vtnet_tso_disable = 0; 239173466SimpTUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable); 240173466SimpSYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, &vtnet_tso_disable, 241173662Smarcel 0, "Disables TCP Segmentation Offload"); 242173662Smarcelstatic int vtnet_lro_disable = 0; 243173662SmarcelTUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable); 244173662SmarcelSYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, &vtnet_lro_disable, 245173662Smarcel 0, "Disables TCP Large Receive Offload"); 246173662Smarcelstatic int vtnet_mq_disable = 0; 247175227SjhbTUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable); 248175227SjhbSYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, &vtnet_mq_disable, 249172983Smtm 0, "Disables Multi Queue support"); 250172983Smtmstatic int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS; 251172390SbushmanTUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs); 252173176SbushmanSYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN, 253172390Sbushman &vtnet_mq_max_pairs, 0, "Sets the maximum number of Multi Queue pairs"); 254172390Sbushmanstatic int vtnet_rx_process_limit = 512; 255172570SruTUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); 256172570SruSYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 257171786Smarcel &vtnet_rx_process_limit, 0, 258171786Smarcel "Limits the number RX segments processed in a single pass"); 259171786Smarcel 260171786Smarcelstatic uma_zone_t vtnet_tx_header_zone; 261171696Sbz 262171696Sbzstatic struct virtio_feature_desc vtnet_feature_desc[] = { 263179368Sbz { VIRTIO_NET_F_CSUM, "TxChecksum" }, 264171461Srwatson { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, 265171461Srwatson { VIRTIO_NET_F_MAC, "MacAddress" }, 266171461Srwatson { VIRTIO_NET_F_GSO, "TxAllGSO" }, 267171461Srwatson { VIRTIO_NET_F_GUEST_TSO4, "RxTSOv4" }, 268171461Srwatson { VIRTIO_NET_F_GUEST_TSO6, "RxTSOv6" }, 269171461Srwatson { VIRTIO_NET_F_GUEST_ECN, "RxECN" }, 270171461Srwatson { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, 271171461Srwatson { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, 272171461Srwatson { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, 273171461Srwatson { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, 274171461Srwatson { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, 275171461Srwatson { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, 276171461Srwatson { VIRTIO_NET_F_STATUS, "Status" }, 277171461Srwatson { VIRTIO_NET_F_CTRL_VQ, "ControlVq" }, 278171461Srwatson { VIRTIO_NET_F_CTRL_RX, "RxMode" }, 279171461Srwatson { VIRTIO_NET_F_CTRL_VLAN, "VLanFilter" }, 280171461Srwatson { VIRTIO_NET_F_CTRL_RX_EXTRA, "RxModeExtra" }, 281171461Srwatson { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, 282171461Srwatson { VIRTIO_NET_F_MQ, "Multiqueue" }, 283171461Srwatson { VIRTIO_NET_F_CTRL_MAC_ADDR, "SetMacAddress" }, 284171461Srwatson 285171461Srwatson { 0, NULL } 286171461Srwatson}; 287171461Srwatson 288171461Srwatsonstatic device_method_t vtnet_methods[] = { 289171461Srwatson /* Device methods. */ 290171461Srwatson DEVMETHOD(device_probe, vtnet_probe), 291171461Srwatson DEVMETHOD(device_attach, vtnet_attach), 292171461Srwatson DEVMETHOD(device_detach, vtnet_detach), 293171461Srwatson DEVMETHOD(device_suspend, vtnet_suspend), 294171461Srwatson DEVMETHOD(device_resume, vtnet_resume), 295171461Srwatson DEVMETHOD(device_shutdown, vtnet_shutdown), 296171461Srwatson 297171461Srwatson /* VirtIO methods. */ 298171461Srwatson DEVMETHOD(virtio_attach_completed, vtnet_attach_completed), 299171461Srwatson DEVMETHOD(virtio_config_change, vtnet_config_change), 300171461Srwatson 301171461Srwatson DEVMETHOD_END 302171461Srwatson}; 303171461Srwatson 304171461Srwatson#ifdef DEV_NETMAP 305171461Srwatson#include <dev/netmap/if_vtnet_netmap.h> 306171461Srwatson#endif /* DEV_NETMAP */ 307171461Srwatson 308171461Srwatsonstatic driver_t vtnet_driver = { 309171461Srwatson "vtnet", 310171461Srwatson vtnet_methods, 311171461Srwatson sizeof(struct vtnet_softc) 312171461Srwatson}; 313171461Srwatsonstatic devclass_t vtnet_devclass; 314171461Srwatson 315171461SrwatsonDRIVER_MODULE(vtnet, virtio_mmio, vtnet_driver, vtnet_devclass, 316171461Srwatson vtnet_modevent, 0); 317171461SrwatsonDRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass, 318171461Srwatson vtnet_modevent, 0); 319171461SrwatsonMODULE_VERSION(vtnet, 1); 320171461SrwatsonMODULE_DEPEND(vtnet, virtio, 1, 1, 1); 321171461Srwatson#ifdef DEV_NETMAP 322171461SrwatsonMODULE_DEPEND(vtnet, netmap, 1, 1, 1); 323171461Srwatson#endif /* DEV_NETMAP */ 324171461Srwatson 325171461Srwatsonstatic int 326171461Srwatsonvtnet_modevent(module_t mod, int type, void *unused) 327171461Srwatson{ 328171461Srwatson int error = 0; 329171461Srwatson static int loaded = 0; 330171461Srwatson 331171461Srwatson switch (type) { 332171461Srwatson case MOD_LOAD: 333171461Srwatson if (loaded++ == 0) 334171461Srwatson vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr", 335171461Srwatson sizeof(struct vtnet_tx_header), 336176956Santoine NULL, NULL, NULL, NULL, 0, 0); 337176956Santoine break; 338176956Santoine case MOD_QUIESCE: 339176956Santoine if (uma_zone_get_cur(vtnet_tx_header_zone) > 0) 340176956Santoine error = EBUSY; 341176956Santoine break; 342171274Sbz case MOD_UNLOAD: 343171274Sbz if (--loaded == 0) { 344171274Sbz uma_zdestroy(vtnet_tx_header_zone); 345171274Sbz vtnet_tx_header_zone = NULL; 346171274Sbz } 347171274Sbz break; 348171274Sbz case MOD_SHUTDOWN: 349171274Sbz break; 350171274Sbz default: 351179368Sbz error = EOPNOTSUPP; 352171205Sbz break; 353171205Sbz } 354171205Sbz 355171205Sbz return (error); 356171205Sbz} 357171175Smlaier 358171175Smlaierstatic int 359171137Sbzvtnet_probe(device_t dev) 360171137Sbz{ 361171137Sbz 362171137Sbz if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK) 363171137Sbz return (ENXIO); 364171137Sbz 365171137Sbz device_set_desc(dev, "VirtIO Networking Adapter"); 366171137Sbz 367171137Sbz return (BUS_PROBE_DEFAULT); 368171137Sbz} 369171137Sbz 370171137Sbzstatic int 371171137Sbzvtnet_attach(device_t dev) 372171137Sbz{ 373171137Sbz struct vtnet_softc *sc; 374171137Sbz int error; 375171137Sbz 376171137Sbz sc = device_get_softc(dev); 377171137Sbz sc->vtnet_dev = dev; 378171131Sthompsa 379171131Sthompsa /* Register our feature descriptions. */ 380171143Sthompsa virtio_set_feature_desc(dev, vtnet_feature_desc); 381171023Srafan 382171023Srafan VTNET_CORE_LOCK_INIT(sc); 383171023Srafan callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); 384171023Srafan 385171023Srafan vtnet_setup_sysctl(sc); 386171023Srafan vtnet_setup_features(sc); 387171388Sdougb 388171388Sdougb error = vtnet_alloc_rx_filters(sc); 389171388Sdougb if (error) { 390171388Sdougb device_printf(dev, "cannot allocate Rx filters\n"); 391170926Srafan goto fail; 392170926Srafan } 393170926Srafan 394170926Srafan error = vtnet_alloc_rxtx_queues(sc); 395170926Srafan if (error) { 396170926Srafan device_printf(dev, "cannot allocate queues\n"); 397170926Srafan goto fail; 398170926Srafan } 399170926Srafan 400170926Srafan error = vtnet_alloc_virtqueues(sc); 401170926Srafan if (error) { 402170926Srafan device_printf(dev, "cannot allocate virtqueues\n"); 403170926Srafan goto fail; 404170926Srafan } 405170926Srafan 406170926Srafan error = vtnet_setup_interface(sc); 407170926Srafan if (error) { 408170926Srafan device_printf(dev, "cannot setup interface\n"); 409170926Srafan goto fail; 410170926Srafan } 411170926Srafan 412170926Srafan error = virtio_setup_intr(dev, INTR_TYPE_NET); 413170926Srafan if (error) { 414170926Srafan device_printf(dev, "cannot setup virtqueue interrupts\n"); 415170926Srafan /* BMV: This will crash if during boot! */ 416170926Srafan ether_ifdetach(sc->vtnet_ifp); 417170926Srafan goto fail; 418170926Srafan } 419170926Srafan 420170926Srafan#ifdef DEV_NETMAP 421170926Srafan vtnet_netmap_attach(sc); 422170926Srafan#endif /* DEV_NETMAP */ 423170926Srafan 424170926Srafan vtnet_start_taskqueues(sc); 425170926Srafan 426170926Srafanfail: 427170926Srafan if (error) 428170926Srafan vtnet_detach(dev); 429170926Srafan 430170926Srafan return (error); 431170926Srafan} 432176956Santoine 433176956Santoinestatic int 434176956Santoinevtnet_detach(device_t dev) 435176956Santoine{ 436176956Santoine struct vtnet_softc *sc; 437176956Santoine struct ifnet *ifp; 438176956Santoine 439176956Santoine sc = device_get_softc(dev); 440176956Santoine ifp = sc->vtnet_ifp; 441176956Santoine 442176956Santoine if (device_is_attached(dev)) { 443176956Santoine VTNET_CORE_LOCK(sc); 444176956Santoine vtnet_stop(sc); 445176956Santoine VTNET_CORE_UNLOCK(sc); 446176956Santoine 447176956Santoine callout_drain(&sc->vtnet_tick_ch); 448176956Santoine vtnet_drain_taskqueues(sc); 449176956Santoine 450176956Santoine ether_ifdetach(ifp); 451176956Santoine } 452176956Santoine 453176956Santoine#ifdef DEV_NETMAP 454176956Santoine netmap_detach(ifp); 455176956Santoine#endif /* DEV_NETMAP */ 456176956Santoine 457176956Santoine vtnet_free_taskqueues(sc); 458176956Santoine 459176956Santoine if (sc->vtnet_vlan_attach != NULL) { 460176956Santoine EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); 461176956Santoine sc->vtnet_vlan_attach = NULL; 462176956Santoine } 463176956Santoine if (sc->vtnet_vlan_detach != NULL) { 464176956Santoine EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach); 465176956Santoine sc->vtnet_vlan_detach = NULL; 466176956Santoine } 467176956Santoine 468171476Sdelphij ifmedia_removeall(&sc->vtnet_media); 469171476Sdelphij 470170312Sdelphij if (ifp != NULL) { 471170312Sdelphij if_free(ifp); 472170926Srafan sc->vtnet_ifp = NULL; 473173816Sru } 474169815Sdelphij 475169815Sdelphij vtnet_free_rxtx_queues(sc); 476169815Sdelphij vtnet_free_rx_filters(sc); 477169815Sdelphij 478169815Sdelphij if (sc->vtnet_ctrl_vq != NULL) 479169815Sdelphij vtnet_free_ctrl_vq(sc); 480169815Sdelphij 481169815Sdelphij VTNET_CORE_LOCK_DESTROY(sc); 482169815Sdelphij 483169815Sdelphij return (0); 484169815Sdelphij} 485169815Sdelphij 486170204Srustatic int 487169815Sdelphijvtnet_suspend(device_t dev) 488169815Sdelphij{ 489169815Sdelphij struct vtnet_softc *sc; 490169815Sdelphij 491170204Sru sc = device_get_softc(dev); 492169815Sdelphij 493169815Sdelphij VTNET_CORE_LOCK(sc); 494169815Sdelphij vtnet_stop(sc); 495169815Sdelphij sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; 496169815Sdelphij VTNET_CORE_UNLOCK(sc); 497169815Sdelphij 498169815Sdelphij return (0); 499169815Sdelphij} 500169815Sdelphij 501169815Sdelphijstatic int 502169815Sdelphijvtnet_resume(device_t dev) 503169815Sdelphij{ 504169815Sdelphij struct vtnet_softc *sc; 505169815Sdelphij struct ifnet *ifp; 506169815Sdelphij 507169815Sdelphij sc = device_get_softc(dev); 508169815Sdelphij ifp = sc->vtnet_ifp; 509169815Sdelphij 510169815Sdelphij VTNET_CORE_LOCK(sc); 511169815Sdelphij if (ifp->if_flags & IFF_UP) 512169815Sdelphij vtnet_init_locked(sc); 513169815Sdelphij sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; 514169815Sdelphij VTNET_CORE_UNLOCK(sc); 515169815Sdelphij 516169815Sdelphij return (0); 517169815Sdelphij} 518169815Sdelphij 519169815Sdelphijstatic int 520169815Sdelphijvtnet_shutdown(device_t dev) 521169815Sdelphij{ 522169815Sdelphij 523169815Sdelphij /* 524169815Sdelphij * Suspend already does all of what we need to 525169815Sdelphij * do here; we just never expect to be resumed. 526169815Sdelphij */ 527169815Sdelphij return (vtnet_suspend(dev)); 528170204Sru} 529169815Sdelphij 530169815Sdelphijstatic int 531169815Sdelphijvtnet_attach_completed(device_t dev) 532170917Srafan{ 533169815Sdelphij 534169815Sdelphij vtnet_attach_disable_promisc(device_get_softc(dev)); 535169815Sdelphij 536169815Sdelphij return (0); 537169815Sdelphij} 538169815Sdelphij 539169815Sdelphijstatic int 540169815Sdelphijvtnet_config_change(device_t dev) 541169815Sdelphij{ 542169815Sdelphij struct vtnet_softc *sc; 543169815Sdelphij 544169815Sdelphij sc = device_get_softc(dev); 545169815Sdelphij 546169815Sdelphij VTNET_CORE_LOCK(sc); 547169815Sdelphij vtnet_update_link_status(sc); 548169815Sdelphij if (sc->vtnet_link_active != 0) 549169815Sdelphij vtnet_tx_start_all(sc); 550169815Sdelphij VTNET_CORE_UNLOCK(sc); 551169815Sdelphij 552169815Sdelphij return (0); 553169815Sdelphij} 554169815Sdelphij 555169815Sdelphijstatic void 556169815Sdelphijvtnet_negotiate_features(struct vtnet_softc *sc) 557169815Sdelphij{ 558169815Sdelphij device_t dev; 559169815Sdelphij uint64_t mask, features; 560169815Sdelphij 561169815Sdelphij dev = sc->vtnet_dev; 562169815Sdelphij mask = 0; 563169815Sdelphij 564169815Sdelphij /* 565169815Sdelphij * TSO and LRO are only available when their corresponding checksum 566169815Sdelphij * offload feature is also negotiated. 567169815Sdelphij */ 568169815Sdelphij if (vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable)) { 569169815Sdelphij mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM; 570169815Sdelphij mask |= VTNET_TSO_FEATURES | VTNET_LRO_FEATURES; 571169815Sdelphij } 572169815Sdelphij if (vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable)) 573169815Sdelphij mask |= VTNET_TSO_FEATURES; 574169815Sdelphij if (vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable)) 575169815Sdelphij mask |= VTNET_LRO_FEATURES; 576169815Sdelphij#ifndef VTNET_LEGACY_TX 577169815Sdelphij if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable)) 578169815Sdelphij mask |= VIRTIO_NET_F_MQ; 579169815Sdelphij#else 580169815Sdelphij mask |= VIRTIO_NET_F_MQ; 581169815Sdelphij#endif 582170204Sru 583169815Sdelphij features = VTNET_FEATURES & ~mask; 584169815Sdelphij sc->vtnet_features = virtio_negotiate_features(dev, features); 585169815Sdelphij 586169815Sdelphij if (virtio_with_feature(dev, VTNET_LRO_FEATURES) && 587169815Sdelphij virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) { 588170190Sru /* 589169815Sdelphij * LRO without mergeable buffers requires special care. This 590169815Sdelphij * is not ideal because every receive buffer must be large 591169815Sdelphij * enough to hold the maximum TCP packet, the Ethernet header, 592169815Sdelphij * and the header. This requires up to 34 descriptors with 593169815Sdelphij * MCLBYTES clusters. If we do not have indirect descriptors, 594169815Sdelphij * LRO is disabled since the virtqueue will not contain very 595169815Sdelphij * many receive buffers. 596169815Sdelphij */ 597169815Sdelphij if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { 598169815Sdelphij device_printf(dev, 599169815Sdelphij "LRO disabled due to both mergeable buffers and " 600169815Sdelphij "indirect descriptors not negotiated\n"); 601170190Sru 602170190Sru features &= ~VTNET_LRO_FEATURES; 603170190Sru sc->vtnet_features = 604170190Sru virtio_negotiate_features(dev, features); 605170190Sru } else 606170190Sru sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; 607170190Sru } 608171476Sdelphij} 609171476Sdelphij 610171476Sdelphijstatic void 611171476Sdelphijvtnet_setup_features(struct vtnet_softc *sc) 612171476Sdelphij{ 613171476Sdelphij device_t dev; 614171476Sdelphij 615171476Sdelphij dev = sc->vtnet_dev; 616171476Sdelphij 617171476Sdelphij vtnet_negotiate_features(sc); 618171476Sdelphij 619171476Sdelphij if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 620171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_INDIRECT; 621171476Sdelphij if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX)) 622171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; 623171476Sdelphij 624171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { 625171476Sdelphij /* This feature should always be negotiated. */ 626171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_MAC; 627171476Sdelphij } 628171476Sdelphij 629171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { 630171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; 631171476Sdelphij sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 632171476Sdelphij } else 633171476Sdelphij sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 634171476Sdelphij 635171476Sdelphij if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) 636171476Sdelphij sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS; 637171476Sdelphij else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 638171476Sdelphij sc->vtnet_rx_nsegs = VTNET_MAX_RX_SEGS; 639171476Sdelphij else 640171476Sdelphij sc->vtnet_rx_nsegs = VTNET_MIN_RX_SEGS; 641171476Sdelphij 642171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || 643171476Sdelphij virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || 644171476Sdelphij virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 645171476Sdelphij sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS; 646171476Sdelphij else 647171476Sdelphij sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS; 648171476Sdelphij 649171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { 650171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; 651171476Sdelphij 652171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) 653171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; 654171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) 655171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; 656171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) 657171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; 658171476Sdelphij } 659171476Sdelphij 660171476Sdelphij if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) && 661171476Sdelphij sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 662171476Sdelphij sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev, 663171476Sdelphij offsetof(struct virtio_net_config, max_virtqueue_pairs)); 664171476Sdelphij } else 665171476Sdelphij sc->vtnet_max_vq_pairs = 1; 666171476Sdelphij 667171476Sdelphij if (sc->vtnet_max_vq_pairs > 1) { 668171476Sdelphij /* 669171476Sdelphij * Limit the maximum number of queue pairs to the lower of 670171476Sdelphij * the number of CPUs and the configured maximum. 671171476Sdelphij * The actual number of queues that get used may be less. 672171476Sdelphij */ 673171476Sdelphij int max; 674171476Sdelphij 675171476Sdelphij max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); 676171476Sdelphij if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN) { 677171476Sdelphij if (max > mp_ncpus) 678171476Sdelphij max = mp_ncpus; 679171476Sdelphij if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 680171476Sdelphij max = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX; 681171476Sdelphij if (max > 1) { 682171476Sdelphij sc->vtnet_requested_vq_pairs = max; 683171476Sdelphij sc->vtnet_flags |= VTNET_FLAG_MULTIQ; 684171476Sdelphij } 685171476Sdelphij } 686171476Sdelphij } 687171476Sdelphij} 688171476Sdelphij 689171476Sdelphijstatic int 690171476Sdelphijvtnet_init_rxq(struct vtnet_softc *sc, int id) 691171476Sdelphij{ 692171476Sdelphij struct vtnet_rxq *rxq; 693171476Sdelphij 694171476Sdelphij rxq = &sc->vtnet_rxqs[id]; 695171476Sdelphij 696171476Sdelphij snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", 697171476Sdelphij device_get_nameunit(sc->vtnet_dev), id); 698171476Sdelphij mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); 699171476Sdelphij 700171476Sdelphij rxq->vtnrx_sc = sc; 701171476Sdelphij rxq->vtnrx_id = id; 702171476Sdelphij 703171476Sdelphij rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); 704171476Sdelphij if (rxq->vtnrx_sg == NULL) 705171476Sdelphij return (ENOMEM); 706171476Sdelphij 707171476Sdelphij TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); 708171476Sdelphij rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, 709171476Sdelphij taskqueue_thread_enqueue, &rxq->vtnrx_tq); 710171476Sdelphij 711171476Sdelphij return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); 712171476Sdelphij} 713171476Sdelphij 714171476Sdelphijstatic int 715171476Sdelphijvtnet_init_txq(struct vtnet_softc *sc, int id) 716171476Sdelphij{ 717171476Sdelphij struct vtnet_txq *txq; 718171476Sdelphij 719171476Sdelphij txq = &sc->vtnet_txqs[id]; 720171476Sdelphij 721171476Sdelphij snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", 722171476Sdelphij device_get_nameunit(sc->vtnet_dev), id); 723171476Sdelphij mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); 724171476Sdelphij 725171476Sdelphij txq->vtntx_sc = sc; 726171476Sdelphij txq->vtntx_id = id; 727171476Sdelphij 728171476Sdelphij txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); 729171476Sdelphij if (txq->vtntx_sg == NULL) 730171476Sdelphij return (ENOMEM); 731171476Sdelphij 732171476Sdelphij#ifndef VTNET_LEGACY_TX 733171476Sdelphij txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, 734171476Sdelphij M_NOWAIT, &txq->vtntx_mtx); 735171476Sdelphij if (txq->vtntx_br == NULL) 736171476Sdelphij return (ENOMEM); 737171476Sdelphij 738171476Sdelphij TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); 739171476Sdelphij#endif 740171476Sdelphij TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); 741171476Sdelphij txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, 742171476Sdelphij taskqueue_thread_enqueue, &txq->vtntx_tq); 743171476Sdelphij if (txq->vtntx_tq == NULL) 744171476Sdelphij return (ENOMEM); 745171476Sdelphij 746171476Sdelphij return (0); 747171476Sdelphij} 748171476Sdelphij 749171476Sdelphijstatic int 750171476Sdelphijvtnet_alloc_rxtx_queues(struct vtnet_softc *sc) 751171476Sdelphij{ 752171476Sdelphij int i, npairs, error; 753171476Sdelphij 754171476Sdelphij npairs = sc->vtnet_max_vq_pairs; 755171476Sdelphij 756171476Sdelphij sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, 757171476Sdelphij M_NOWAIT | M_ZERO); 758171476Sdelphij sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, 759171476Sdelphij M_NOWAIT | M_ZERO); 760171476Sdelphij if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) 761171476Sdelphij return (ENOMEM); 762171476Sdelphij 763171476Sdelphij for (i = 0; i < npairs; i++) { 764171476Sdelphij error = vtnet_init_rxq(sc, i); 765171476Sdelphij if (error) 766171476Sdelphij return (error); 767171476Sdelphij error = vtnet_init_txq(sc, i); 768171476Sdelphij if (error) 769171476Sdelphij return (error); 770171476Sdelphij } 771171476Sdelphij 772171476Sdelphij vtnet_setup_queue_sysctl(sc); 773171476Sdelphij 774171476Sdelphij return (0); 775171476Sdelphij} 776171476Sdelphij 777171476Sdelphijstatic void 778171476Sdelphijvtnet_destroy_rxq(struct vtnet_rxq *rxq) 779171476Sdelphij{ 780171476Sdelphij 781171476Sdelphij rxq->vtnrx_sc = NULL; 782171476Sdelphij rxq->vtnrx_id = -1; 783171476Sdelphij 784171476Sdelphij if (rxq->vtnrx_sg != NULL) { 785171476Sdelphij sglist_free(rxq->vtnrx_sg); 786171476Sdelphij rxq->vtnrx_sg = NULL; 787171476Sdelphij } 788171476Sdelphij 789171476Sdelphij if (mtx_initialized(&rxq->vtnrx_mtx) != 0) 790171476Sdelphij mtx_destroy(&rxq->vtnrx_mtx); 791171476Sdelphij} 792171476Sdelphij 793171476Sdelphijstatic void 794171476Sdelphijvtnet_destroy_txq(struct vtnet_txq *txq) 795171476Sdelphij{ 796171476Sdelphij 797171476Sdelphij txq->vtntx_sc = NULL; 798171476Sdelphij txq->vtntx_id = -1; 799171476Sdelphij 800171476Sdelphij if (txq->vtntx_sg != NULL) { 801171476Sdelphij sglist_free(txq->vtntx_sg); 802171476Sdelphij txq->vtntx_sg = NULL; 803171476Sdelphij } 804171476Sdelphij 805171476Sdelphij#ifndef VTNET_LEGACY_TX 806171476Sdelphij if (txq->vtntx_br != NULL) { 807171476Sdelphij buf_ring_free(txq->vtntx_br, M_DEVBUF); 808171476Sdelphij txq->vtntx_br = NULL; 809171476Sdelphij } 810171476Sdelphij#endif 811171476Sdelphij 812171476Sdelphij if (mtx_initialized(&txq->vtntx_mtx) != 0) 813171476Sdelphij mtx_destroy(&txq->vtntx_mtx); 814171476Sdelphij} 815171476Sdelphij 816171476Sdelphijstatic void 817171476Sdelphijvtnet_free_rxtx_queues(struct vtnet_softc *sc) 818171476Sdelphij{ 819171476Sdelphij int i; 820171476Sdelphij 821171476Sdelphij if (sc->vtnet_rxqs != NULL) { 822171476Sdelphij for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 823171476Sdelphij vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); 824171476Sdelphij free(sc->vtnet_rxqs, M_DEVBUF); 825171476Sdelphij sc->vtnet_rxqs = NULL; 826171476Sdelphij } 827171476Sdelphij 828171476Sdelphij if (sc->vtnet_txqs != NULL) { 829171476Sdelphij for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 830171476Sdelphij vtnet_destroy_txq(&sc->vtnet_txqs[i]); 831171476Sdelphij free(sc->vtnet_txqs, M_DEVBUF); 832169445Sroberto sc->vtnet_txqs = NULL; 833169445Sroberto } 834169445Sroberto} 835169026Semax 836169026Semaxstatic int 837168916Sbrueffervtnet_alloc_rx_filters(struct vtnet_softc *sc) 838168916Sbrueffer{ 839168796Sthompsa 840168796Sthompsa if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 841168544Spjd sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), 842168544Spjd M_DEVBUF, M_NOWAIT | M_ZERO); 843167980Sdelphij if (sc->vtnet_mac_filter == NULL) 844167980Sdelphij return (ENOMEM); 845167699Sdelphij } 846167699Sdelphij 847176956Santoine if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 848176956Santoine sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * 849176956Santoine VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO); 850167137Sbms if (sc->vtnet_vlan_filter == NULL) 851167137Sbms return (ENOMEM); 852170190Sru } 853166981Sru 854166981Sru return (0); 855170192Sru} 856170218Struckman 857166668Sbruefferstatic void 858166668Sbrueffervtnet_free_rx_filters(struct vtnet_softc *sc) 859166389Srafan{ 860166389Srafan 861166389Srafan if (sc->vtnet_mac_filter != NULL) { 862172882Sru free(sc->vtnet_mac_filter, M_DEVBUF); 863172882Sru sc->vtnet_mac_filter = NULL; 864172882Sru } 865172882Sru 866170190Sru if (sc->vtnet_vlan_filter != NULL) { 867170190Sru free(sc->vtnet_vlan_filter, M_DEVBUF); 868170190Sru sc->vtnet_vlan_filter = NULL; 869170190Sru } 870172882Sru} 871172882Sru 872172882Srustatic int 873170190Sruvtnet_alloc_virtqueues(struct vtnet_softc *sc) 874166308Sphk{ 875166308Sphk device_t dev; 876170192Sru struct vq_alloc_info *info; 877170192Sru struct vtnet_rxq *rxq; 878166246Speter struct vtnet_txq *txq; 879166246Speter int i, idx, flags, nvqs, error; 880166246Speter 881166246Speter dev = sc->vtnet_dev; 882166246Speter flags = 0; 883164796Spiso 884164796Spiso nvqs = sc->vtnet_max_vq_pairs * 2; 885164796Spiso if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 886164796Spiso nvqs++; 887164796Spiso 888164796Spiso info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); 889164796Spiso if (info == NULL) 890164796Spiso return (ENOMEM); 891164796Spiso 892166672Sbrueffer for (i = 0, idx = 0; i < sc->vtnet_max_vq_pairs; i++, idx+=2) { 893164796Spiso rxq = &sc->vtnet_rxqs[i]; 894164796Spiso VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, 895164796Spiso vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, 896164796Spiso "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id); 897164796Spiso 898164796Spiso txq = &sc->vtnet_txqs[i]; 899164796Spiso VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs, 900164796Spiso vtnet_tx_vq_intr, txq, &txq->vtntx_vq, 901165726Skientzle "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id); 902165726Skientzle } 903164610Simp 904164610Simp if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 905164537Srodrigc VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL, 906164537Srodrigc &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); 907164537Srodrigc } 908164537Srodrigc 909164537Srodrigc /* 910164537Srodrigc * Enable interrupt binding if this is multiqueue. This only matters 911164537Srodrigc * when per-vq MSIX is available. 912170190Sru */ 913170190Sru if (sc->vtnet_flags & VTNET_FLAG_MULTIQ) 914170190Sru flags |= 0; 915170190Sru 916170190Sru error = virtio_alloc_virtqueues(dev, flags, nvqs, info); 917170190Sru free(info, M_TEMP); 918164537Srodrigc 919164537Srodrigc return (error); 920164537Srodrigc} 921164537Srodrigc 922164537Srodrigcstatic int 923164537Srodrigcvtnet_setup_interface(struct vtnet_softc *sc) 924164344Sbrueffer{ 925164344Sbrueffer device_t dev; 926170220Struckman struct ifnet *ifp; 927170220Struckman 928164088Smarcel dev = sc->vtnet_dev; 929164088Smarcel 930164088Smarcel ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER); 931164088Smarcel if (ifp == NULL) { 932163570Sru device_printf(dev, "cannot allocate ifnet structure\n"); 933163570Sru return (ENOSPC); 934162837Sdelphij } 935162837Sdelphij 936162780Sbms if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 937162780Sbms ifp->if_baudrate = IF_Gbps(10); /* Approx. */ 938162780Sbms ifp->if_softc = sc; 939162780Sbms ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 940162780Sbms ifp->if_init = vtnet_init; 941162780Sbms ifp->if_ioctl = vtnet_ioctl; 942162780Sbms ifp->if_get_counter = vtnet_get_counter; 943162780Sbms#ifndef VTNET_LEGACY_TX 944162780Sbms ifp->if_transmit = vtnet_txq_mq_start; 945162598Ssimon ifp->if_qflush = vtnet_qflush; 946162598Ssimon#else 947162598Ssimon struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; 948162716Sdelphij ifp->if_start = vtnet_start; 949162716Sdelphij IFQ_SET_MAXLEN(&ifp->if_snd, virtqueue_size(vq) - 1); 950161529Sflz ifp->if_snd.ifq_drv_maxlen = virtqueue_size(vq) - 1; 951161529Sflz IFQ_SET_READY(&ifp->if_snd); 952161529Sflz#endif 953170192Sru 954170192Sru ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd, 955170192Sru vtnet_ifmedia_sts); 956170192Sru ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL); 957170192Sru ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE); 958170192Sru 959170192Sru /* Read (or generate) the MAC address for the adapter. */ 960170192Sru vtnet_get_hwaddr(sc); 961170192Sru 962170192Sru ether_ifattach(ifp, sc->vtnet_hwaddr); 963170192Sru 964170192Sru if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) 965170192Sru ifp->if_capabilities |= IFCAP_LINKSTATE; 966160983Sbrooks 967160983Sbrooks /* Tell the upper layer(s) we support long frames. */ 968170255Struckman ifp->if_hdrlen = sizeof(struct ether_vlan_header); 969170255Struckman ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 970170255Struckman 971170255Struckman if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { 972158687Sphk ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6; 973158687Sphk 974158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_GSO)) { 975158687Sphk ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 976158687Sphk sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 977158687Sphk } else { 978158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) 979158687Sphk ifp->if_capabilities |= IFCAP_TSO4; 980158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 981158687Sphk ifp->if_capabilities |= IFCAP_TSO6; 982158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) 983158687Sphk sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 984158687Sphk } 985158687Sphk 986158687Sphk if (ifp->if_capabilities & IFCAP_TSO) 987158687Sphk ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 988158687Sphk } 989158687Sphk 990158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) { 991158687Sphk ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; 992158687Sphk 993158687Sphk if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) || 994158687Sphk virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6)) 995158687Sphk ifp->if_capabilities |= IFCAP_LRO; 996158687Sphk } 997158687Sphk 998158687Sphk if (ifp->if_capabilities & IFCAP_HWCSUM) { 999158687Sphk /* 1000158687Sphk * VirtIO does not support VLAN tagging, but we can fake 1001158687Sphk * it by inserting and removing the 802.1Q header during 1002158687Sphk * transmit and receive. We are then able to do checksum 1003158687Sphk * offloading of VLAN frames. 1004158687Sphk */ 1005158687Sphk ifp->if_capabilities |= 1006158687Sphk IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 1007158687Sphk } 1008158687Sphk 1009158687Sphk ifp->if_capenable = ifp->if_capabilities; 1010158687Sphk 1011158687Sphk /* 1012158687Sphk * Capabilities after here are not enabled by default. 1013158687Sphk */ 1014158687Sphk 1015158687Sphk if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 1016158687Sphk ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 1017158687Sphk 1018158687Sphk sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1019158687Sphk vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1020158687Sphk sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 1021158687Sphk vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1022158687Sphk } 1023158687Sphk 1024158687Sphk vtnet_set_rx_process_limit(sc); 1025158618Smaxim vtnet_set_tx_intr_threshold(sc); 1026158618Smaxim 1027158618Smaxim return (0); 1028158512Smlaier} 1029158512Smlaier 1030158512Smlaierstatic int 1031158512Smlaiervtnet_change_mtu(struct vtnet_softc *sc, int new_mtu) 1032158512Smlaier{ 1033158754Smarcel struct ifnet *ifp; 1034158754Smarcel int frame_size, clsize; 1035157221Ssimon 1036157221Ssimon ifp = sc->vtnet_ifp; 1037156676Sharti 1038156676Sharti if (new_mtu < ETHERMIN || new_mtu > VTNET_MAX_MTU) 1039170190Sru return (EINVAL); 1040170190Sru 1041170190Sru frame_size = sc->vtnet_hdr_size + sizeof(struct ether_vlan_header) + 1042170190Sru new_mtu; 1043170190Sru 1044170190Sru /* 1045170190Sru * Based on the new MTU (and hence frame size) determine which 1046170190Sru * cluster size is most appropriate for the receive queues. 1047153662Sjhb */ 1048153662Sjhb if (frame_size <= MCLBYTES) { 1049153430Siedowse clsize = MCLBYTES; 1050153430Siedowse } else if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1051153430Siedowse /* Avoid going past 9K jumbos. */ 1052153430Siedowse if (frame_size > MJUM9BYTES) 1053153430Siedowse return (EINVAL); 1054151845Syar clsize = MJUM9BYTES; 1055151845Syar } else 1056151271Spjd clsize = MJUMPAGESIZE; 1057151271Spjd 1058162025Smatusita ifp->if_mtu = new_mtu; 1059162025Smatusita sc->vtnet_rx_new_clsize = clsize; 1060162025Smatusita 1061162025Smatusita if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1062150676Smlaier ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1063150677Smlaier vtnet_init_locked(sc); 1064150002Snetchild } 1065150002Snetchild 1066150002Snetchild return (0); 1067150002Snetchild} 1068149454Sglebius 1069148808Srustatic int 1070148808Sruvtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1071148825Snetchild{ 1072148825Snetchild struct vtnet_softc *sc; 1073148356Sdougb struct ifreq *ifr; 1074148356Sdougb int reinit, mask, error; 1075148572Snetchild 1076170220Struckman sc = ifp->if_softc; 1077170220Struckman ifr = (struct ifreq *) data; 1078170220Struckman error = 0; 1079170220Struckman 1080170220Struckman switch (cmd) { 1081148330Snetchild case SIOCSIFMTU: 1082148330Snetchild if (ifp->if_mtu != ifr->ifr_mtu) { 1083148330Snetchild VTNET_CORE_LOCK(sc); 1084148330Snetchild error = vtnet_change_mtu(sc, ifr->ifr_mtu); 1085148572Snetchild VTNET_CORE_UNLOCK(sc); 1086148572Snetchild } 1087149105Simp break; 1088148572Snetchild 1089148572Snetchild case SIOCSIFFLAGS: 1090148572Snetchild VTNET_CORE_LOCK(sc); 1091172026Syar if ((ifp->if_flags & IFF_UP) == 0) { 1092172026Syar if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1093172026Syar vtnet_stop(sc); 1094172026Syar } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1095148572Snetchild if ((ifp->if_flags ^ sc->vtnet_if_flags) & 1096148572Snetchild (IFF_PROMISC | IFF_ALLMULTI)) { 1097148572Snetchild if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) 1098148572Snetchild vtnet_rx_filter(sc); 1099148572Snetchild else { 1100148572Snetchild ifp->if_flags |= IFF_PROMISC; 1101148572Snetchild if ((ifp->if_flags ^ sc->vtnet_if_flags) 1102148572Snetchild & IFF_ALLMULTI) 1103148572Snetchild error = ENOTSUP; 1104148572Snetchild } 1105148572Snetchild } 1106153939Snetchild } else 1107153939Snetchild vtnet_init_locked(sc); 1108153939Snetchild 1109153939Snetchild if (error == 0) 1110148330Snetchild sc->vtnet_if_flags = ifp->if_flags; 1111148330Snetchild VTNET_CORE_UNLOCK(sc); 1112148423Sdougb break; 1113148423Sdougb 1114148330Snetchild case SIOCADDMULTI: 1115148330Snetchild case SIOCDELMULTI: 1116148330Snetchild if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) 1117148330Snetchild break; 1118148572Snetchild VTNET_CORE_LOCK(sc); 1119148572Snetchild if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1120148353Sdougb vtnet_rx_filter_mac(sc); 1121148353Sdougb VTNET_CORE_UNLOCK(sc); 1122149105Simp break; 1123149105Simp 1124148572Snetchild case SIOCSIFMEDIA: 1125148572Snetchild case SIOCGIFMEDIA: 1126172026Syar error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); 1127149105Simp break; 1128148572Snetchild 1129155813Snetchild case SIOCSIFCAP: 1130155813Snetchild VTNET_CORE_LOCK(sc); 1131155813Snetchild mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1132155813Snetchild 1133148330Snetchild if (mask & IFCAP_TXCSUM) 1134148330Snetchild ifp->if_capenable ^= IFCAP_TXCSUM; 1135148330Snetchild if (mask & IFCAP_TXCSUM_IPV6) 1136163991Strhodes ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1137163991Strhodes if (mask & IFCAP_TSO4) 1138163991Strhodes ifp->if_capenable ^= IFCAP_TSO4; 1139163991Strhodes if (mask & IFCAP_TSO6) 1140163991Strhodes ifp->if_capenable ^= IFCAP_TSO6; 1141163991Strhodes 1142163991Strhodes if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 1143163991Strhodes IFCAP_VLAN_HWFILTER)) { 1144163991Strhodes /* These Rx features require us to renegotiate. */ 1145163991Strhodes reinit = 1; 1146163991Strhodes 1147148330Snetchild if (mask & IFCAP_RXCSUM) 1148148330Snetchild ifp->if_capenable ^= IFCAP_RXCSUM; 1149148330Snetchild if (mask & IFCAP_RXCSUM_IPV6) 1150148330Snetchild ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1151155813Snetchild if (mask & IFCAP_LRO) 1152148330Snetchild ifp->if_capenable ^= IFCAP_LRO; 1153148330Snetchild if (mask & IFCAP_VLAN_HWFILTER) 1154148330Snetchild ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1155148330Snetchild } else 1156148330Snetchild reinit = 0; 1157148330Snetchild 1158148543Snetchild if (mask & IFCAP_VLAN_HWTSO) 1159148543Snetchild ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1160148543Snetchild if (mask & IFCAP_VLAN_HWTAGGING) 1161148543Snetchild ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1162148543Snetchild 1163148543Snetchild if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1164148543Snetchild ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1165148543Snetchild vtnet_init_locked(sc); 1166148543Snetchild } 1167148543Snetchild 1168148543Snetchild VTNET_CORE_UNLOCK(sc); 1169148543Snetchild VLAN_CAPABILITIES(ifp); 1170148543Snetchild 1171148543Snetchild break; 1172148543Snetchild 1173148543Snetchild default: 1174148543Snetchild error = ether_ioctl(ifp, cmd, data); 1175148543Snetchild break; 1176148543Snetchild } 1177148543Snetchild 1178148543Snetchild VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc); 1179148572Snetchild 1180148572Snetchild return (error); 1181148572Snetchild} 1182148572Snetchild 1183148572Snetchildstatic int 1184148572Snetchildvtnet_rxq_populate(struct vtnet_rxq *rxq) 1185148572Snetchild{ 1186148572Snetchild struct virtqueue *vq; 1187148572Snetchild int nbufs, error; 1188148572Snetchild 1189148572Snetchild vq = rxq->vtnrx_vq; 1190148572Snetchild error = ENOSPC; 1191148572Snetchild 1192148572Snetchild for (nbufs = 0; !virtqueue_full(vq); nbufs++) { 1193148572Snetchild error = vtnet_rxq_new_buf(rxq); 1194148572Snetchild if (error) 1195148572Snetchild break; 1196148572Snetchild } 1197148572Snetchild 1198148572Snetchild if (nbufs > 0) { 1199148572Snetchild virtqueue_notify(vq); 1200148572Snetchild /* 1201148572Snetchild * EMSGSIZE signifies the virtqueue did not have enough 1202148572Snetchild * entries available to hold the last mbuf. This is not 1203148572Snetchild * an error. 1204148572Snetchild */ 1205148572Snetchild if (error == EMSGSIZE) 1206148572Snetchild error = 0; 1207148572Snetchild } 1208148572Snetchild 1209148572Snetchild return (error); 1210148572Snetchild} 1211148572Snetchild 1212148572Snetchildstatic void 1213148572Snetchildvtnet_rxq_free_mbufs(struct vtnet_rxq *rxq) 1214148572Snetchild{ 1215148572Snetchild struct virtqueue *vq; 1216148572Snetchild struct mbuf *m; 1217148572Snetchild int last; 1218148572Snetchild 1219148572Snetchild vq = rxq->vtnrx_vq; 1220148572Snetchild last = 0; 1221148572Snetchild 1222148572Snetchild while ((m = virtqueue_drain(vq, &last)) != NULL) 1223148572Snetchild m_freem(m); 1224148572Snetchild 1225148572Snetchild KASSERT(virtqueue_empty(vq), 1226148572Snetchild ("%s: mbufs remaining in rx queue %p", __func__, rxq)); 1227148572Snetchild} 1228148572Snetchild 1229148572Snetchildstatic struct mbuf * 1230148572Snetchildvtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) 1231148572Snetchild{ 1232148572Snetchild struct mbuf *m_head, *m_tail, *m; 1233148572Snetchild int i, clsize; 1234148572Snetchild 1235148572Snetchild clsize = sc->vtnet_rx_clsize; 1236148572Snetchild 1237148572Snetchild KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1238148572Snetchild ("%s: chained mbuf %d request without LRO_NOMRG", __func__, nbufs)); 1239148572Snetchild 1240148572Snetchild m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize); 1241148572Snetchild if (m_head == NULL) 1242148572Snetchild goto fail; 1243148572Snetchild 1244148572Snetchild m_head->m_len = clsize; 1245148572Snetchild m_tail = m_head; 1246148572Snetchild 1247148572Snetchild /* Allocate the rest of the chain. */ 1248148572Snetchild for (i = 1; i < nbufs; i++) { 1249148572Snetchild m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize); 1250148572Snetchild if (m == NULL) 1251148572Snetchild goto fail; 1252148572Snetchild 1253148572Snetchild m->m_len = clsize; 1254148572Snetchild m_tail->m_next = m; 1255148572Snetchild m_tail = m; 1256148572Snetchild } 1257148572Snetchild 1258148572Snetchild if (m_tailp != NULL) 1259148572Snetchild *m_tailp = m_tail; 1260148572Snetchild 1261148572Snetchild return (m_head); 1262148572Snetchild 1263148572Snetchildfail: 1264148572Snetchild sc->vtnet_stats.mbuf_alloc_failed++; 1265148572Snetchild m_freem(m_head); 1266148572Snetchild 1267148572Snetchild return (NULL); 1268148572Snetchild} 1269148572Snetchild 1270148572Snetchild/* 1271148572Snetchild * Slow path for when LRO without mergeable buffers is negotiated. 1272148572Snetchild */ 1273148572Snetchildstatic int 1274148572Snetchildvtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0, 1275148572Snetchild int len0) 1276148572Snetchild{ 1277148572Snetchild struct vtnet_softc *sc; 1278148572Snetchild struct mbuf *m, *m_prev; 1279148572Snetchild struct mbuf *m_new, *m_tail; 1280148572Snetchild int len, clsize, nreplace, error; 1281148572Snetchild 1282148572Snetchild sc = rxq->vtnrx_sc; 1283148572Snetchild clsize = sc->vtnet_rx_clsize; 1284148572Snetchild 1285148572Snetchild m_prev = NULL; 1286148572Snetchild m_tail = NULL; 1287148330Snetchild nreplace = 0; 1288148330Snetchild 1289148330Snetchild m = m0; 1290148330Snetchild len = len0; 1291148330Snetchild 1292148572Snetchild /* 1293148572Snetchild * Since these mbuf chains are so large, we avoid allocating an 1294148572Snetchild * entire replacement chain if possible. When the received frame 1295148572Snetchild * did not consume the entire chain, the unused mbufs are moved 1296148572Snetchild * to the replacement chain. 1297148572Snetchild */ 1298177625Sremko while (len > 0) { 1299177625Sremko /* 1300177625Sremko * Something is seriously wrong if we received a frame 1301148572Snetchild * larger than the chain. Drop it. 1302148572Snetchild */ 1303148572Snetchild if (m == NULL) { 1304148572Snetchild sc->vtnet_stats.rx_frame_too_large++; 1305149113Simp return (EMSGSIZE); 1306149113Simp } 1307149113Simp 1308149113Simp /* We always allocate the same cluster size. */ 1309149113Simp KASSERT(m->m_len == clsize, 1310149113Simp ("%s: mbuf size %d is not the cluster size %d", 1311149113Simp __func__, m->m_len, clsize)); 1312148572Snetchild 1313148572Snetchild m->m_len = MIN(m->m_len, len); 1314148572Snetchild len -= m->m_len; 1315148572Snetchild 1316148572Snetchild m_prev = m; 1317149105Simp m = m->m_next; 1318148572Snetchild nreplace++; 1319148572Snetchild } 1320148572Snetchild 1321148572Snetchild KASSERT(nreplace <= sc->vtnet_rx_nmbufs, 1322148572Snetchild ("%s: too many replacement mbufs %d max %d", __func__, nreplace, 1323148572Snetchild sc->vtnet_rx_nmbufs)); 1324148572Snetchild 1325148572Snetchild m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail); 1326170220Struckman if (m_new == NULL) { 1327170220Struckman m_prev->m_len = clsize; 1328156385Syar return (ENOBUFS); 1329156385Syar } 1330148330Snetchild 1331148330Snetchild /* 1332148330Snetchild * Move any unused mbufs from the received chain onto the end 1333148330Snetchild * of the new chain. 1334148330Snetchild */ 1335148330Snetchild if (m_prev->m_next != NULL) { 1336148330Snetchild m_tail->m_next = m_prev->m_next; 1337148330Snetchild m_prev->m_next = NULL; 1338148330Snetchild } 1339148330Snetchild 1340148330Snetchild error = vtnet_rxq_enqueue_buf(rxq, m_new); 1341148330Snetchild if (error) { 1342148330Snetchild /* 1343148330Snetchild * BAD! We could not enqueue the replacement mbuf chain. We 1344148330Snetchild * must restore the m0 chain to the original state if it was 1345148330Snetchild * modified so we can subsequently discard it. 1346148330Snetchild * 1347148330Snetchild * NOTE: The replacement is suppose to be an identical copy 1348148330Snetchild * to the one just dequeued so this is an unexpected error. 1349148330Snetchild */ 1350148330Snetchild sc->vtnet_stats.rx_enq_replacement_failed++; 1351148572Snetchild 1352148572Snetchild if (m_tail->m_next != NULL) { 1353149105Simp m_prev->m_next = m_tail->m_next; 1354148572Snetchild m_tail->m_next = NULL; 1355148572Snetchild } 1356148572Snetchild 1357148572Snetchild m_prev->m_len = clsize; 1358148572Snetchild m_freem(m_new); 1359148572Snetchild } 1360148572Snetchild 1361155279Savatar return (error); 1362155279Savatar} 1363155279Savatar 1364155279Savatarstatic int 1365148572Snetchildvtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len) 1366148572Snetchild{ 1367148572Snetchild struct vtnet_softc *sc; 1368148330Snetchild struct mbuf *m_new; 1369148330Snetchild int error; 1370148330Snetchild 1371148330Snetchild sc = rxq->vtnrx_sc; 1372148330Snetchild 1373148330Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1374148330Snetchild ("%s: chained mbuf without LRO_NOMRG", __func__)); 1375148330Snetchild 1376148330Snetchild if (m->m_next == NULL) { 1377148330Snetchild /* Fast-path for the common case of just one mbuf. */ 1378148330Snetchild if (m->m_len < len) 1379148330Snetchild return (EINVAL); 1380148330Snetchild 1381148330Snetchild m_new = vtnet_rx_alloc_buf(sc, 1, NULL); 1382148330Snetchild if (m_new == NULL) 1383148330Snetchild return (ENOBUFS); 1384149105Simp 1385149105Simp error = vtnet_rxq_enqueue_buf(rxq, m_new); 1386148572Snetchild if (error) { 1387170219Struckman /* 1388148572Snetchild * The new mbuf is suppose to be an identical 1389148572Snetchild * copy of the one just dequeued so this is an 1390148572Snetchild * unexpected error. 1391148330Snetchild */ 1392148330Snetchild m_freem(m_new); 1393148330Snetchild sc->vtnet_stats.rx_enq_replacement_failed++; 1394148330Snetchild } else 1395148330Snetchild m->m_len = len; 1396148330Snetchild } else 1397148330Snetchild error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len); 1398148330Snetchild 1399148330Snetchild return (error); 1400148330Snetchild} 1401148330Snetchild 1402148330Snetchildstatic int 1403148330Snetchildvtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1404148330Snetchild{ 1405148330Snetchild struct vtnet_softc *sc; 1406148330Snetchild struct sglist *sg; 1407148330Snetchild struct vtnet_rx_header *rxhdr; 1408148330Snetchild uint8_t *mdata; 1409148330Snetchild int offset, error; 1410148330Snetchild 1411148330Snetchild sc = rxq->vtnrx_sc; 1412148330Snetchild sg = rxq->vtnrx_sg; 1413148330Snetchild mdata = mtod(m, uint8_t *); 1414148572Snetchild 1415148572Snetchild VTNET_RXQ_LOCK_ASSERT(rxq); 1416148572Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1417148572Snetchild ("%s: chained mbuf without LRO_NOMRG", __func__)); 1418148572Snetchild KASSERT(m->m_len == sc->vtnet_rx_clsize, 1419148572Snetchild ("%s: unexpected cluster size %d/%d", __func__, m->m_len, 1420148572Snetchild sc->vtnet_rx_clsize)); 1421148572Snetchild 1422148572Snetchild sglist_reset(sg); 1423148572Snetchild if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1424148572Snetchild MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); 1425148572Snetchild rxhdr = (struct vtnet_rx_header *) mdata; 1426148572Snetchild sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size); 1427148572Snetchild offset = sizeof(struct vtnet_rx_header); 1428148572Snetchild } else 1429148572Snetchild offset = 0; 1430148572Snetchild 1431148572Snetchild sglist_append(sg, mdata + offset, m->m_len - offset); 1432148572Snetchild if (m->m_next != NULL) { 1433148572Snetchild error = sglist_append_mbuf(sg, m->m_next); 1434148572Snetchild MPASS(error == 0); 1435148572Snetchild } 1436148572Snetchild 1437148572Snetchild error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg); 1438148572Snetchild 1439148572Snetchild return (error); 1440148572Snetchild} 1441148572Snetchild 1442148572Snetchildstatic int 1443148572Snetchildvtnet_rxq_new_buf(struct vtnet_rxq *rxq) 1444148572Snetchild{ 1445148572Snetchild struct vtnet_softc *sc; 1446148572Snetchild struct mbuf *m; 1447148572Snetchild int error; 1448148572Snetchild 1449148572Snetchild sc = rxq->vtnrx_sc; 1450148572Snetchild 1451148572Snetchild m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); 1452148572Snetchild if (m == NULL) 1453148572Snetchild return (ENOBUFS); 1454148572Snetchild 1455148572Snetchild error = vtnet_rxq_enqueue_buf(rxq, m); 1456148572Snetchild if (error) 1457148572Snetchild m_freem(m); 1458148572Snetchild 1459148572Snetchild return (error); 1460148572Snetchild} 1461148572Snetchild 1462148572Snetchild/* 1463148572Snetchild * Use the checksum offset in the VirtIO header to set the 1464148572Snetchild * correct CSUM_* flags. 1465148572Snetchild */ 1466148572Snetchildstatic int 1467148572Snetchildvtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m, 1468148572Snetchild uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1469148572Snetchild{ 1470148572Snetchild struct vtnet_softc *sc; 1471148572Snetchild#if defined(INET) || defined(INET6) 1472148572Snetchild int offset = hdr->csum_start + hdr->csum_offset; 1473148572Snetchild#endif 1474148572Snetchild 1475148572Snetchild sc = rxq->vtnrx_sc; 1476148572Snetchild 1477148572Snetchild /* Only do a basic sanity check on the offset. */ 1478148572Snetchild switch (eth_type) { 1479148572Snetchild#if defined(INET) 1480148572Snetchild case ETHERTYPE_IP: 1481148572Snetchild if (__predict_false(offset < ip_start + sizeof(struct ip))) 1482148572Snetchild return (1); 1483148572Snetchild break; 1484148572Snetchild#endif 1485148572Snetchild#if defined(INET6) 1486148572Snetchild case ETHERTYPE_IPV6: 1487148572Snetchild if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr))) 1488148572Snetchild return (1); 1489148572Snetchild break; 1490148572Snetchild#endif 1491148572Snetchild default: 1492148572Snetchild sc->vtnet_stats.rx_csum_bad_ethtype++; 1493148572Snetchild return (1); 1494148572Snetchild } 1495148572Snetchild 1496148572Snetchild /* 1497148572Snetchild * Use the offset to determine the appropriate CSUM_* flags. This is 1498148572Snetchild * a bit dirty, but we can get by with it since the checksum offsets 1499148572Snetchild * happen to be different. We assume the host host does not do IPv4 1500148572Snetchild * header checksum offloading. 1501148572Snetchild */ 1502148572Snetchild switch (hdr->csum_offset) { 1503148572Snetchild case offsetof(struct udphdr, uh_sum): 1504148572Snetchild case offsetof(struct tcphdr, th_sum): 1505148572Snetchild m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1506148572Snetchild m->m_pkthdr.csum_data = 0xFFFF; 1507148572Snetchild break; 1508148572Snetchild case offsetof(struct sctphdr, checksum): 1509148572Snetchild m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1510148572Snetchild break; 1511148572Snetchild default: 1512148572Snetchild sc->vtnet_stats.rx_csum_bad_offset++; 1513149105Simp return (1); 1514149105Simp } 1515149105Simp 1516149105Simp return (0); 1517148572Snetchild} 1518164619Snetchild 1519164619Snetchildstatic int 1520148572Snetchildvtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m, 1521148572Snetchild uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1522148572Snetchild{ 1523148572Snetchild struct vtnet_softc *sc; 1524148572Snetchild int offset, proto; 1525148572Snetchild 1526148572Snetchild sc = rxq->vtnrx_sc; 1527148572Snetchild 1528148572Snetchild switch (eth_type) { 1529148572Snetchild#if defined(INET) 1530148572Snetchild case ETHERTYPE_IP: { 1531148572Snetchild struct ip *ip; 1532148572Snetchild if (__predict_false(m->m_len < ip_start + sizeof(struct ip))) 1533148572Snetchild return (1); 1534148572Snetchild ip = (struct ip *)(m->m_data + ip_start); 1535148572Snetchild proto = ip->ip_p; 1536148572Snetchild offset = ip_start + (ip->ip_hl << 2); 1537148572Snetchild break; 1538148572Snetchild } 1539148572Snetchild#endif 1540148572Snetchild#if defined(INET6) 1541148572Snetchild case ETHERTYPE_IPV6: 1542148572Snetchild if (__predict_false(m->m_len < ip_start + 1543148572Snetchild sizeof(struct ip6_hdr))) 1544148572Snetchild return (1); 1545148572Snetchild offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto); 1546148572Snetchild if (__predict_false(offset < 0)) 1547148572Snetchild return (1); 1548148572Snetchild break; 1549148572Snetchild#endif 1550148572Snetchild default: 1551148572Snetchild sc->vtnet_stats.rx_csum_bad_ethtype++; 1552148572Snetchild return (1); 1553148572Snetchild } 1554148572Snetchild 1555148572Snetchild switch (proto) { 1556148572Snetchild case IPPROTO_TCP: 1557148572Snetchild if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) 1558148572Snetchild return (1); 1559148572Snetchild m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1560148572Snetchild m->m_pkthdr.csum_data = 0xFFFF; 1561148572Snetchild break; 1562148572Snetchild case IPPROTO_UDP: 1563148572Snetchild if (__predict_false(m->m_len < offset + sizeof(struct udphdr))) 1564148572Snetchild return (1); 1565148572Snetchild m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1566148572Snetchild m->m_pkthdr.csum_data = 0xFFFF; 1567148572Snetchild break; 1568148572Snetchild case IPPROTO_SCTP: 1569148572Snetchild if (__predict_false(m->m_len < offset + sizeof(struct sctphdr))) 1570148572Snetchild return (1); 1571148572Snetchild m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1572148572Snetchild break; 1573148572Snetchild default: 1574148572Snetchild /* 1575148572Snetchild * For the remaining protocols, FreeBSD does not support 1576148572Snetchild * checksum offloading, so the checksum will be recomputed. 1577148572Snetchild */ 1578149105Simp#if 0 1579149105Simp if_printf(sc->vtnet_ifp, "cksum offload of unsupported " 1580149105Simp "protocol eth_type=%#x proto=%d csum_start=%d " 1581149105Simp "csum_offset=%d\n", __func__, eth_type, proto, 1582149105Simp hdr->csum_start, hdr->csum_offset); 1583149105Simp#endif 1584149105Simp break; 1585149105Simp } 1586148572Snetchild 1587148572Snetchild return (0); 1588160165Savatar} 1589160165Savatar 1590148330Snetchild/* 1591148330Snetchild * Set the appropriate CSUM_* flags. Unfortunately, the information 1592148330Snetchild * provided is not directly useful to us. The VirtIO header gives the 1593149105Simp * offset of the checksum, which is all Linux needs, but this is not 1594149105Simp * how FreeBSD does things. We are forced to peek inside the packet 1595163831Sjmg * a bit. 1596149105Simp * 1597163831Sjmg * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD 1598149105Simp * could accept the offsets and let the stack figure it out. 1599149105Simp */ 1600149105Simpstatic int 1601149105Simpvtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, 1602149105Simp struct virtio_net_hdr *hdr) 1603149105Simp{ 1604149105Simp struct ether_header *eh; 1605149105Simp struct ether_vlan_header *evh; 1606149105Simp uint16_t eth_type; 1607149105Simp int offset, error; 1608149105Simp 1609149105Simp eh = mtod(m, struct ether_header *); 1610149105Simp eth_type = ntohs(eh->ether_type); 1611149105Simp if (eth_type == ETHERTYPE_VLAN) { 1612149105Simp /* BMV: We should handle nested VLAN tags too. */ 1613149105Simp evh = mtod(m, struct ether_vlan_header *); 1614149105Simp eth_type = ntohs(evh->evl_proto); 1615149105Simp offset = sizeof(struct ether_vlan_header); 1616149105Simp } else 1617149105Simp offset = sizeof(struct ether_header); 1618149105Simp 1619149105Simp if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1620160165Savatar error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr); 1621160165Savatar else 1622160165Savatar error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr); 1623148330Snetchild 1624148330Snetchild return (error); 1625148330Snetchild} 1626148330Snetchild 1627148330Snetchildstatic void 1628148330Snetchildvtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) 1629148330Snetchild{ 1630148330Snetchild struct mbuf *m; 1631149105Simp 1632149105Simp while (--nbufs > 0) { 1633149105Simp m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); 1634149105Simp if (m == NULL) 1635149105Simp break; 1636148330Snetchild vtnet_rxq_discard_buf(rxq, m); 1637149105Simp } 1638149105Simp} 1639149105Simp 1640149105Simpstatic void 1641149105Simpvtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1642149105Simp{ 1643149105Simp int error; 1644149105Simp 1645149105Simp /* 1646149105Simp * Requeue the discarded mbuf. This should always be successful 1647149105Simp * since it was just dequeued. 1648149105Simp */ 1649149105Simp error = vtnet_rxq_enqueue_buf(rxq, m); 1650164971Savatar KASSERT(error == 0, 1651164971Savatar ("%s: cannot requeue discarded mbuf %d", __func__, error)); 1652164971Savatar} 1653164971Savatar 1654164971Savatarstatic int 1655164971Savatarvtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs) 1656164971Savatar{ 1657164971Savatar struct vtnet_softc *sc; 1658164971Savatar struct virtqueue *vq; 1659164971Savatar struct mbuf *m, *m_tail; 1660164971Savatar int len; 1661164971Savatar 1662164971Savatar sc = rxq->vtnrx_sc; 1663164971Savatar vq = rxq->vtnrx_vq; 1664164971Savatar m_tail = m_head; 1665164971Savatar 1666164971Savatar while (--nbufs > 0) { 1667164971Savatar m = virtqueue_dequeue(vq, &len); 1668164971Savatar if (m == NULL) { 1669164971Savatar rxq->vtnrx_stats.vrxs_ierrors++; 1670164971Savatar goto fail; 1671164971Savatar } 1672164971Savatar 1673164971Savatar if (vtnet_rxq_new_buf(rxq) != 0) { 1674164971Savatar rxq->vtnrx_stats.vrxs_iqdrops++; 1675160165Savatar vtnet_rxq_discard_buf(rxq, m); 1676160165Savatar if (nbufs > 1) 1677160165Savatar vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1678148330Snetchild goto fail; 1679148330Snetchild } 1680148330Snetchild 1681148330Snetchild if (m->m_len < len) 1682149105Simp len = m->m_len; 1683149105Simp 1684149105Simp m->m_len = len; 1685164302Smatteo m->m_flags &= ~M_PKTHDR; 1686164302Smatteo 1687148330Snetchild m_head->m_pkthdr.len += len; 1688148330Snetchild m_tail->m_next = m; 1689148330Snetchild m_tail = m; 1690148330Snetchild } 1691148330Snetchild 1692148330Snetchild return (0); 1693148330Snetchild 1694148330Snetchildfail: 1695148330Snetchild sc->vtnet_stats.rx_mergeable_failed++; 1696148330Snetchild m_freem(m_head); 1697148330Snetchild 1698148330Snetchild return (1); 1699148330Snetchild} 1700148330Snetchild 1701148330Snetchildstatic void 1702148330Snetchildvtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, 1703148543Snetchild struct virtio_net_hdr *hdr) 1704149113Simp{ 1705149113Simp struct vtnet_softc *sc; 1706149113Simp struct ifnet *ifp; 1707149113Simp struct ether_header *eh; 1708149113Simp 1709149113Simp sc = rxq->vtnrx_sc; 1710149113Simp ifp = sc->vtnet_ifp; 1711149113Simp 1712149113Simp if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1713149113Simp eh = mtod(m, struct ether_header *); 1714149113Simp if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1715148330Snetchild vtnet_vlan_tag_remove(m); 1716148330Snetchild /* 1717149113Simp * With the 802.1Q header removed, update the 1718149113Simp * checksum starting location accordingly. 1719149113Simp */ 1720149113Simp if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1721149113Simp hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; 1722149113Simp } 1723149113Simp } 1724149113Simp 1725149113Simp m->m_pkthdr.flowid = rxq->vtnrx_id; 1726149113Simp M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1727149113Simp 1728149113Simp /* 1729149113Simp * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum 1730149113Simp * distinction that Linux does. Need to reevaluate if performing 1731149113Simp * offloading for the NEEDS_CSUM case is really appropriate. 1732149113Simp */ 1733149113Simp if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | 1734149113Simp VIRTIO_NET_HDR_F_DATA_VALID)) { 1735149113Simp if (vtnet_rxq_csum(rxq, m, hdr) == 0) 1736149113Simp rxq->vtnrx_stats.vrxs_csum++; 1737149113Simp else 1738149113Simp rxq->vtnrx_stats.vrxs_csum_failed++; 1739149113Simp } 1740149113Simp 1741149113Simp rxq->vtnrx_stats.vrxs_ipackets++; 1742149113Simp rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; 1743149113Simp 1744149113Simp VTNET_RXQ_UNLOCK(rxq); 1745149113Simp (*ifp->if_input)(ifp, m); 1746149113Simp VTNET_RXQ_LOCK(rxq); 1747149113Simp} 1748149113Simp 1749148330Snetchildstatic int 1750148330Snetchildvtnet_rxq_eof(struct vtnet_rxq *rxq) 1751148330Snetchild{ 1752148330Snetchild struct virtio_net_hdr lhdr, *hdr; 1753148330Snetchild struct vtnet_softc *sc; 1754148330Snetchild struct ifnet *ifp; 1755148330Snetchild struct virtqueue *vq; 1756151378Snetchild struct mbuf *m; 1757151378Snetchild struct virtio_net_hdr_mrg_rxbuf *mhdr; 1758151378Snetchild int len, deq, nbufs, adjsz, count; 1759151378Snetchild 1760151378Snetchild sc = rxq->vtnrx_sc; 1761151378Snetchild vq = rxq->vtnrx_vq; 1762151378Snetchild ifp = sc->vtnet_ifp; 1763151378Snetchild hdr = &lhdr; 1764151378Snetchild deq = 0; 1765151378Snetchild count = sc->vtnet_rx_process_limit; 1766161201Snetchild 1767151378Snetchild VTNET_RXQ_LOCK_ASSERT(rxq); 1768151378Snetchild 1769151378Snetchild#ifdef DEV_NETMAP 1770151378Snetchild if (netmap_rx_irq(ifp, 0, &deq)) { 1771151378Snetchild return (FALSE); 1772151378Snetchild } 1773151378Snetchild#endif /* DEV_NETMAP */ 1774151378Snetchild 1775151378Snetchild while (count-- > 0) { 1776151378Snetchild m = virtqueue_dequeue(vq, &len); 1777151378Snetchild if (m == NULL) 1778151378Snetchild break; 1779151378Snetchild deq++; 1780151378Snetchild 1781151378Snetchild if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { 1782151378Snetchild rxq->vtnrx_stats.vrxs_ierrors++; 1783151378Snetchild vtnet_rxq_discard_buf(rxq, m); 1784151378Snetchild continue; 1785151378Snetchild } 1786151378Snetchild 1787151378Snetchild if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1788151378Snetchild nbufs = 1; 1789151378Snetchild adjsz = sizeof(struct vtnet_rx_header); 1790151378Snetchild /* 1791151378Snetchild * Account for our pad inserted between the header 1792151378Snetchild * and the actual start of the frame. 1793151378Snetchild */ 1794151378Snetchild len += VTNET_RX_HEADER_PAD; 1795151378Snetchild } else { 1796151378Snetchild mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *); 1797151378Snetchild nbufs = mhdr->num_buffers; 1798151378Snetchild adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1799151378Snetchild } 1800151378Snetchild 1801151378Snetchild if (vtnet_rxq_replace_buf(rxq, m, len) != 0) { 1802151378Snetchild rxq->vtnrx_stats.vrxs_iqdrops++; 1803151378Snetchild vtnet_rxq_discard_buf(rxq, m); 1804151378Snetchild if (nbufs > 1) 1805151378Snetchild vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1806151378Snetchild continue; 1807151378Snetchild } 1808151378Snetchild 1809151378Snetchild m->m_pkthdr.len = len; 1810151378Snetchild m->m_pkthdr.rcvif = ifp; 1811151378Snetchild m->m_pkthdr.csum_flags = 0; 1812151378Snetchild 1813151378Snetchild if (nbufs > 1) { 1814151378Snetchild /* Dequeue the rest of chain. */ 1815151378Snetchild if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0) 1816151378Snetchild continue; 1817151378Snetchild } 1818151378Snetchild 1819151378Snetchild /* 1820151378Snetchild * Save copy of header before we strip it. For both mergeable 1821151378Snetchild * and non-mergeable, the header is at the beginning of the 1822151378Snetchild * mbuf data. We no longer need num_buffers, so always use a 1823151378Snetchild * regular header. 1824151378Snetchild * 1825151378Snetchild * BMV: Is this memcpy() expensive? We know the mbuf data is 1826151378Snetchild * still valid even after the m_adj(). 1827151378Snetchild */ 1828151378Snetchild memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr)); 1829151378Snetchild m_adj(m, adjsz); 1830151378Snetchild 1831151378Snetchild vtnet_rxq_input(rxq, m, hdr); 1832151378Snetchild 1833151378Snetchild /* Must recheck after dropping the Rx lock. */ 1834151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1835151378Snetchild break; 1836151378Snetchild } 1837151378Snetchild 1838151378Snetchild if (deq > 0) 1839151378Snetchild virtqueue_notify(vq); 1840151378Snetchild 1841151378Snetchild return (count > 0 ? 0 : EAGAIN); 1842151378Snetchild} 1843151378Snetchild 1844151378Snetchildstatic void 1845151378Snetchildvtnet_rx_vq_intr(void *xrxq) 1846151378Snetchild{ 1847151378Snetchild struct vtnet_softc *sc; 1848151378Snetchild struct vtnet_rxq *rxq; 1849151378Snetchild struct ifnet *ifp; 1850151378Snetchild int tries, more; 1851151378Snetchild 1852151378Snetchild rxq = xrxq; 1853151378Snetchild sc = rxq->vtnrx_sc; 1854151378Snetchild ifp = sc->vtnet_ifp; 1855151378Snetchild tries = 0; 1856151378Snetchild 1857151378Snetchild if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { 1858151378Snetchild /* 1859151378Snetchild * Ignore this interrupt. Either this is a spurious interrupt 1860151378Snetchild * or multiqueue without per-VQ MSIX so every queue needs to 1861151378Snetchild * be polled (a brain dead configuration we could try harder 1862151378Snetchild * to avoid). 1863151378Snetchild */ 1864151378Snetchild vtnet_rxq_disable_intr(rxq); 1865151378Snetchild return; 1866151378Snetchild } 1867151378Snetchild 1868151378Snetchild VTNET_RXQ_LOCK(rxq); 1869151378Snetchild 1870151378Snetchildagain: 1871151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1872151378Snetchild VTNET_RXQ_UNLOCK(rxq); 1873151378Snetchild return; 1874151378Snetchild } 1875151378Snetchild 1876151378Snetchild more = vtnet_rxq_eof(rxq); 1877151378Snetchild if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1878151378Snetchild if (!more) 1879151378Snetchild vtnet_rxq_disable_intr(rxq); 1880151378Snetchild /* 1881151378Snetchild * This is an occasional condition or race (when !more), 1882151378Snetchild * so retry a few times before scheduling the taskqueue. 1883151378Snetchild */ 1884151378Snetchild if (tries++ < VTNET_INTR_DISABLE_RETRIES) 1885151378Snetchild goto again; 1886151378Snetchild 1887151378Snetchild VTNET_RXQ_UNLOCK(rxq); 1888151378Snetchild rxq->vtnrx_stats.vrxs_rescheduled++; 1889151378Snetchild taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1890151378Snetchild } else 1891151378Snetchild VTNET_RXQ_UNLOCK(rxq); 1892151378Snetchild} 1893151378Snetchild 1894151378Snetchildstatic void 1895151378Snetchildvtnet_rxq_tq_intr(void *xrxq, int pending) 1896151378Snetchild{ 1897151378Snetchild struct vtnet_softc *sc; 1898151378Snetchild struct vtnet_rxq *rxq; 1899151378Snetchild struct ifnet *ifp; 1900151378Snetchild int more; 1901151378Snetchild 1902151378Snetchild rxq = xrxq; 1903151378Snetchild sc = rxq->vtnrx_sc; 1904151378Snetchild ifp = sc->vtnet_ifp; 1905151378Snetchild 1906151378Snetchild VTNET_RXQ_LOCK(rxq); 1907151378Snetchild 1908151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1909151378Snetchild VTNET_RXQ_UNLOCK(rxq); 1910151378Snetchild return; 1911151378Snetchild } 1912151378Snetchild 1913151378Snetchild more = vtnet_rxq_eof(rxq); 1914151378Snetchild if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1915151378Snetchild if (!more) 1916151378Snetchild vtnet_rxq_disable_intr(rxq); 1917151378Snetchild rxq->vtnrx_stats.vrxs_rescheduled++; 1918151378Snetchild taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1919151378Snetchild } 1920151378Snetchild 1921151378Snetchild VTNET_RXQ_UNLOCK(rxq); 1922151378Snetchild} 1923151378Snetchild 1924151378Snetchildstatic int 1925151378Snetchildvtnet_txq_below_threshold(struct vtnet_txq *txq) 1926151378Snetchild{ 1927151378Snetchild struct vtnet_softc *sc; 1928151378Snetchild struct virtqueue *vq; 1929151378Snetchild 1930151378Snetchild sc = txq->vtntx_sc; 1931151378Snetchild vq = txq->vtntx_vq; 1932151378Snetchild 1933151378Snetchild return (virtqueue_nfree(vq) <= sc->vtnet_tx_intr_thresh); 1934151378Snetchild} 1935151378Snetchild 1936151378Snetchildstatic int 1937151378Snetchildvtnet_txq_notify(struct vtnet_txq *txq) 1938151378Snetchild{ 1939151378Snetchild struct virtqueue *vq; 1940151378Snetchild 1941151378Snetchild vq = txq->vtntx_vq; 1942151378Snetchild 1943151378Snetchild txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 1944151378Snetchild virtqueue_notify(vq); 1945151378Snetchild 1946151378Snetchild if (vtnet_txq_enable_intr(txq) == 0) 1947151378Snetchild return (0); 1948151378Snetchild 1949151378Snetchild /* 1950151378Snetchild * Drain frames that were completed since last checked. If this 1951151378Snetchild * causes the queue to go above the threshold, the caller should 1952151378Snetchild * continue transmitting. 1953151378Snetchild */ 1954151378Snetchild if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) { 1955151378Snetchild virtqueue_disable_intr(vq); 1956151378Snetchild return (1); 1957151378Snetchild } 1958151378Snetchild 1959151378Snetchild return (0); 1960151378Snetchild} 1961151378Snetchild 1962151378Snetchildstatic void 1963151378Snetchildvtnet_txq_free_mbufs(struct vtnet_txq *txq) 1964151378Snetchild{ 1965151378Snetchild struct virtqueue *vq; 1966151378Snetchild struct vtnet_tx_header *txhdr; 1967151378Snetchild int last; 1968151378Snetchild 1969151378Snetchild vq = txq->vtntx_vq; 1970151378Snetchild last = 0; 1971151378Snetchild 1972151378Snetchild while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 1973151378Snetchild m_freem(txhdr->vth_mbuf); 1974151378Snetchild uma_zfree(vtnet_tx_header_zone, txhdr); 1975151378Snetchild } 1976151378Snetchild 1977151378Snetchild KASSERT(virtqueue_empty(vq), 1978151378Snetchild ("%s: mbufs remaining in tx queue %p", __func__, txq)); 1979151378Snetchild} 1980151378Snetchild 1981151378Snetchild/* 1982151378Snetchild * BMV: Much of this can go away once we finally have offsets in 1983151378Snetchild * the mbuf packet header. Bug andre@. 1984151378Snetchild */ 1985151378Snetchildstatic int 1986151378Snetchildvtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, 1987151378Snetchild int *etype, int *proto, int *start) 1988151378Snetchild{ 1989151378Snetchild struct vtnet_softc *sc; 1990151378Snetchild struct ether_vlan_header *evh; 1991151378Snetchild int offset; 1992151378Snetchild 1993151378Snetchild sc = txq->vtntx_sc; 1994151378Snetchild 1995151378Snetchild evh = mtod(m, struct ether_vlan_header *); 1996151378Snetchild if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1997151378Snetchild /* BMV: We should handle nested VLAN tags too. */ 1998151378Snetchild *etype = ntohs(evh->evl_proto); 1999151378Snetchild offset = sizeof(struct ether_vlan_header); 2000151378Snetchild } else { 2001151378Snetchild *etype = ntohs(evh->evl_encap_proto); 2002151378Snetchild offset = sizeof(struct ether_header); 2003151378Snetchild } 2004151378Snetchild 2005151378Snetchild switch (*etype) { 2006151378Snetchild#if defined(INET) 2007151378Snetchild case ETHERTYPE_IP: { 2008151378Snetchild struct ip *ip, iphdr; 2009151378Snetchild if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2010151378Snetchild m_copydata(m, offset, sizeof(struct ip), 2011151378Snetchild (caddr_t) &iphdr); 2012151378Snetchild ip = &iphdr; 2013151378Snetchild } else 2014151378Snetchild ip = (struct ip *)(m->m_data + offset); 2015151378Snetchild *proto = ip->ip_p; 2016151378Snetchild *start = offset + (ip->ip_hl << 2); 2017151378Snetchild break; 2018151378Snetchild } 2019151378Snetchild#endif 2020151378Snetchild#if defined(INET6) 2021151378Snetchild case ETHERTYPE_IPV6: 2022151378Snetchild *proto = -1; 2023151378Snetchild *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2024151378Snetchild /* Assert the network stack sent us a valid packet. */ 2025151378Snetchild KASSERT(*start > offset, 2026151378Snetchild ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2027151378Snetchild *start, offset, *proto)); 2028151378Snetchild break; 2029151378Snetchild#endif 2030151378Snetchild default: 2031151378Snetchild sc->vtnet_stats.tx_csum_bad_ethtype++; 2032151378Snetchild return (EINVAL); 2033151378Snetchild } 2034151378Snetchild 2035151378Snetchild return (0); 2036151378Snetchild} 2037151378Snetchild 2038151378Snetchildstatic int 2039151378Snetchildvtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, 2040151378Snetchild int offset, struct virtio_net_hdr *hdr) 2041151378Snetchild{ 2042151378Snetchild static struct timeval lastecn; 2043151378Snetchild static int curecn; 2044151378Snetchild struct vtnet_softc *sc; 2045151378Snetchild struct tcphdr *tcp, tcphdr; 2046151378Snetchild 2047151378Snetchild sc = txq->vtntx_sc; 2048151378Snetchild 2049151378Snetchild if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { 2050151378Snetchild m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); 2051151378Snetchild tcp = &tcphdr; 2052151378Snetchild } else 2053151378Snetchild tcp = (struct tcphdr *)(m->m_data + offset); 2054151378Snetchild 2055151378Snetchild hdr->hdr_len = offset + (tcp->th_off << 2); 2056151378Snetchild hdr->gso_size = m->m_pkthdr.tso_segsz; 2057151378Snetchild hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : 2058151378Snetchild VIRTIO_NET_HDR_GSO_TCPV6; 2059151378Snetchild 2060151378Snetchild if (tcp->th_flags & TH_CWR) { 2061151378Snetchild /* 2062151378Snetchild * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD, 2063151378Snetchild * ECN support is not on a per-interface basis, but globally via 2064151378Snetchild * the net.inet.tcp.ecn.enable sysctl knob. The default is off. 2065151378Snetchild */ 2066151378Snetchild if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { 2067151378Snetchild if (ppsratecheck(&lastecn, &curecn, 1)) 2068151378Snetchild if_printf(sc->vtnet_ifp, 2069151378Snetchild "TSO with ECN not negotiated with host\n"); 2070151378Snetchild return (ENOTSUP); 2071151378Snetchild } 2072151378Snetchild hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 2073151378Snetchild } 2074151378Snetchild 2075151378Snetchild txq->vtntx_stats.vtxs_tso++; 2076151378Snetchild 2077151378Snetchild return (0); 2078151378Snetchild} 2079151378Snetchild 2080151378Snetchildstatic struct mbuf * 2081151378Snetchildvtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, 2082151378Snetchild struct virtio_net_hdr *hdr) 2083151378Snetchild{ 2084151378Snetchild struct vtnet_softc *sc; 2085151378Snetchild int flags, etype, csum_start, proto, error; 2086151378Snetchild 2087151378Snetchild sc = txq->vtntx_sc; 2088151378Snetchild flags = m->m_pkthdr.csum_flags; 2089151378Snetchild 2090151378Snetchild error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); 2091151378Snetchild if (error) 2092151378Snetchild goto drop; 2093151378Snetchild 2094151378Snetchild if ((etype == ETHERTYPE_IP && flags & VTNET_CSUM_OFFLOAD) || 2095151378Snetchild (etype == ETHERTYPE_IPV6 && flags & VTNET_CSUM_OFFLOAD_IPV6)) { 2096151378Snetchild /* 2097151378Snetchild * We could compare the IP protocol vs the CSUM_ flag too, 2098151378Snetchild * but that really should not be necessary. 2099151378Snetchild */ 2100151378Snetchild hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; 2101151378Snetchild hdr->csum_start = csum_start; 2102151378Snetchild hdr->csum_offset = m->m_pkthdr.csum_data; 2103151378Snetchild txq->vtntx_stats.vtxs_csum++; 2104151378Snetchild } 2105151378Snetchild 2106151378Snetchild if (flags & CSUM_TSO) { 2107151378Snetchild if (__predict_false(proto != IPPROTO_TCP)) { 2108151378Snetchild /* Likely failed to correctly parse the mbuf. */ 2109151378Snetchild sc->vtnet_stats.tx_tso_not_tcp++; 2110151378Snetchild goto drop; 2111151378Snetchild } 2112151378Snetchild 2113151378Snetchild KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM, 2114151378Snetchild ("%s: mbuf %p TSO without checksum offload %#x", 2115151378Snetchild __func__, m, flags)); 2116151378Snetchild 2117151378Snetchild error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); 2118151378Snetchild if (error) 2119151378Snetchild goto drop; 2120151378Snetchild } 2121151378Snetchild 2122151378Snetchild return (m); 2123151378Snetchild 2124151378Snetchilddrop: 2125151378Snetchild m_freem(m); 2126151378Snetchild return (NULL); 2127151378Snetchild} 2128151378Snetchild 2129151378Snetchildstatic int 2130151378Snetchildvtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, 2131151378Snetchild struct vtnet_tx_header *txhdr) 2132151378Snetchild{ 2133151378Snetchild struct vtnet_softc *sc; 2134151378Snetchild struct virtqueue *vq; 2135151378Snetchild struct sglist *sg; 2136151378Snetchild struct mbuf *m; 2137151378Snetchild int error; 2138151378Snetchild 2139151378Snetchild sc = txq->vtntx_sc; 2140151378Snetchild vq = txq->vtntx_vq; 2141151378Snetchild sg = txq->vtntx_sg; 2142151378Snetchild m = *m_head; 2143151378Snetchild 2144151378Snetchild sglist_reset(sg); 2145151378Snetchild error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); 2146151378Snetchild KASSERT(error == 0 && sg->sg_nseg == 1, 2147151378Snetchild ("%s: error %d adding header to sglist", __func__, error)); 2148151378Snetchild 2149151378Snetchild error = sglist_append_mbuf(sg, m); 2150151378Snetchild if (error) { 2151151378Snetchild m = m_defrag(m, M_NOWAIT); 2152151378Snetchild if (m == NULL) 2153151378Snetchild goto fail; 2154151378Snetchild 2155151378Snetchild *m_head = m; 2156151378Snetchild sc->vtnet_stats.tx_defragged++; 2157151378Snetchild 2158151378Snetchild error = sglist_append_mbuf(sg, m); 2159151378Snetchild if (error) 2160151378Snetchild goto fail; 2161151378Snetchild } 2162151378Snetchild 2163151378Snetchild txhdr->vth_mbuf = m; 2164151378Snetchild error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); 2165151378Snetchild 2166151378Snetchild return (error); 2167151378Snetchild 2168151378Snetchildfail: 2169151378Snetchild sc->vtnet_stats.tx_defrag_failed++; 2170151378Snetchild m_freem(*m_head); 2171151378Snetchild *m_head = NULL; 2172151378Snetchild 2173151378Snetchild return (ENOBUFS); 2174151378Snetchild} 2175151378Snetchild 2176151378Snetchildstatic int 2177151378Snetchildvtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head) 2178151378Snetchild{ 2179151378Snetchild struct vtnet_tx_header *txhdr; 2180151378Snetchild struct virtio_net_hdr *hdr; 2181151378Snetchild struct mbuf *m; 2182151378Snetchild int error; 2183151378Snetchild 2184151378Snetchild m = *m_head; 2185151378Snetchild M_ASSERTPKTHDR(m); 2186151378Snetchild 2187151378Snetchild txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO); 2188151378Snetchild if (txhdr == NULL) { 2189151378Snetchild m_freem(m); 2190151378Snetchild *m_head = NULL; 2191151378Snetchild return (ENOMEM); 2192151378Snetchild } 2193151378Snetchild 2194151378Snetchild /* 2195151378Snetchild * Always use the non-mergeable header, regardless if the feature 2196151378Snetchild * was negotiated. For transmit, num_buffers is always zero. The 2197151378Snetchild * vtnet_hdr_size is used to enqueue the correct header size. 2198151378Snetchild */ 2199151378Snetchild hdr = &txhdr->vth_uhdr.hdr; 2200151378Snetchild 2201151378Snetchild if (m->m_flags & M_VLANTAG) { 2202151378Snetchild m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 2203151378Snetchild if ((*m_head = m) == NULL) { 2204151378Snetchild error = ENOBUFS; 2205151378Snetchild goto fail; 2206151378Snetchild } 2207151378Snetchild m->m_flags &= ~M_VLANTAG; 2208151378Snetchild } 2209151378Snetchild 2210151378Snetchild if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { 2211151378Snetchild m = vtnet_txq_offload(txq, m, hdr); 2212151378Snetchild if ((*m_head = m) == NULL) { 2213151378Snetchild error = ENOBUFS; 2214151378Snetchild goto fail; 2215151378Snetchild } 2216151378Snetchild } 2217151378Snetchild 2218151378Snetchild error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); 2219151378Snetchild if (error == 0) 2220151378Snetchild return (0); 2221151378Snetchild 2222151378Snetchildfail: 2223151378Snetchild uma_zfree(vtnet_tx_header_zone, txhdr); 2224151378Snetchild 2225151378Snetchild return (error); 2226151378Snetchild} 2227151378Snetchild 2228151378Snetchild#ifdef VTNET_LEGACY_TX 2229151378Snetchild 2230151378Snetchildstatic void 2231151378Snetchildvtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp) 2232151378Snetchild{ 2233151378Snetchild struct vtnet_softc *sc; 2234151378Snetchild struct virtqueue *vq; 2235151378Snetchild struct mbuf *m0; 2236151378Snetchild int tries, enq; 2237151378Snetchild 2238151378Snetchild sc = txq->vtntx_sc; 2239151378Snetchild vq = txq->vtntx_vq; 2240151378Snetchild tries = 0; 2241151378Snetchild 2242151378Snetchild VTNET_TXQ_LOCK_ASSERT(txq); 2243151378Snetchild 2244151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2245151378Snetchild sc->vtnet_link_active == 0) 2246151378Snetchild return; 2247151378Snetchild 2248151378Snetchild vtnet_txq_eof(txq); 2249151378Snetchild 2250151378Snetchildagain: 2251151378Snetchild enq = 0; 2252151378Snetchild 2253151378Snetchild while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2254151378Snetchild if (virtqueue_full(vq)) 2255151378Snetchild break; 2256151378Snetchild 2257151378Snetchild IFQ_DRV_DEQUEUE(&ifp->if_snd, m0); 2258151378Snetchild if (m0 == NULL) 2259151378Snetchild break; 2260151378Snetchild 2261151378Snetchild if (vtnet_txq_encap(txq, &m0) != 0) { 2262151378Snetchild if (m0 != NULL) 2263151378Snetchild IFQ_DRV_PREPEND(&ifp->if_snd, m0); 2264151378Snetchild break; 2265151378Snetchild } 2266151378Snetchild 2267151378Snetchild enq++; 2268151378Snetchild ETHER_BPF_MTAP(ifp, m0); 2269151378Snetchild } 2270151378Snetchild 2271151378Snetchild if (enq > 0 && vtnet_txq_notify(txq) != 0) { 2272151378Snetchild if (tries++ < VTNET_NOTIFY_RETRIES) 2273151378Snetchild goto again; 2274151378Snetchild 2275151378Snetchild txq->vtntx_stats.vtxs_rescheduled++; 2276151378Snetchild taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2277151378Snetchild } 2278151378Snetchild} 2279151378Snetchild 2280151378Snetchildstatic void 2281151378Snetchildvtnet_start(struct ifnet *ifp) 2282151378Snetchild{ 2283151378Snetchild struct vtnet_softc *sc; 2284151378Snetchild struct vtnet_txq *txq; 2285151378Snetchild 2286151378Snetchild sc = ifp->if_softc; 2287151378Snetchild txq = &sc->vtnet_txqs[0]; 2288151378Snetchild 2289151378Snetchild VTNET_TXQ_LOCK(txq); 2290151378Snetchild vtnet_start_locked(txq, ifp); 2291151378Snetchild VTNET_TXQ_UNLOCK(txq); 2292151378Snetchild} 2293151378Snetchild 2294151378Snetchild#else /* !VTNET_LEGACY_TX */ 2295151378Snetchild 2296151378Snetchildstatic int 2297151378Snetchildvtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m) 2298151378Snetchild{ 2299151378Snetchild struct vtnet_softc *sc; 2300151378Snetchild struct virtqueue *vq; 2301151378Snetchild struct buf_ring *br; 2302151378Snetchild struct ifnet *ifp; 2303151378Snetchild int enq, tries, error; 2304151378Snetchild 2305151378Snetchild sc = txq->vtntx_sc; 2306151378Snetchild vq = txq->vtntx_vq; 2307151378Snetchild br = txq->vtntx_br; 2308151378Snetchild ifp = sc->vtnet_ifp; 2309151378Snetchild tries = 0; 2310151378Snetchild error = 0; 2311151378Snetchild 2312151378Snetchild VTNET_TXQ_LOCK_ASSERT(txq); 2313151378Snetchild 2314151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2315151378Snetchild sc->vtnet_link_active == 0) { 2316151378Snetchild if (m != NULL) 2317151378Snetchild error = drbr_enqueue(ifp, br, m); 2318151378Snetchild return (error); 2319151378Snetchild } 2320151378Snetchild 2321151378Snetchild if (m != NULL) { 2322151378Snetchild error = drbr_enqueue(ifp, br, m); 2323151378Snetchild if (error) 2324151378Snetchild return (error); 2325151378Snetchild } 2326151378Snetchild 2327151378Snetchild vtnet_txq_eof(txq); 2328151378Snetchild 2329151378Snetchildagain: 2330151378Snetchild enq = 0; 2331151378Snetchild 2332151378Snetchild while ((m = drbr_peek(ifp, br)) != NULL) { 2333151378Snetchild if (virtqueue_full(vq)) { 2334151378Snetchild drbr_putback(ifp, br, m); 2335151378Snetchild break; 2336151378Snetchild } 2337151378Snetchild 2338151378Snetchild if (vtnet_txq_encap(txq, &m) != 0) { 2339151378Snetchild if (m != NULL) 2340151378Snetchild drbr_putback(ifp, br, m); 2341151378Snetchild else 2342151378Snetchild drbr_advance(ifp, br); 2343151378Snetchild break; 2344151378Snetchild } 2345151378Snetchild drbr_advance(ifp, br); 2346151378Snetchild 2347151378Snetchild enq++; 2348151378Snetchild ETHER_BPF_MTAP(ifp, m); 2349151378Snetchild } 2350151378Snetchild 2351151378Snetchild if (enq > 0 && vtnet_txq_notify(txq) != 0) { 2352151378Snetchild if (tries++ < VTNET_NOTIFY_RETRIES) 2353151378Snetchild goto again; 2354151378Snetchild 2355151378Snetchild txq->vtntx_stats.vtxs_rescheduled++; 2356151378Snetchild taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2357151378Snetchild } 2358151378Snetchild 2359151378Snetchild return (0); 2360151378Snetchild} 2361151378Snetchild 2362151378Snetchildstatic int 2363151378Snetchildvtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2364151378Snetchild{ 2365151378Snetchild struct vtnet_softc *sc; 2366151378Snetchild struct vtnet_txq *txq; 2367151378Snetchild int i, npairs, error; 2368151378Snetchild 2369151378Snetchild sc = ifp->if_softc; 2370151378Snetchild npairs = sc->vtnet_act_vq_pairs; 2371151378Snetchild 2372151378Snetchild /* check if flowid is set */ 2373151378Snetchild if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 2374151378Snetchild i = m->m_pkthdr.flowid % npairs; 2375151378Snetchild else 2376151378Snetchild i = curcpu % npairs; 2377151378Snetchild 2378151378Snetchild txq = &sc->vtnet_txqs[i]; 2379151378Snetchild 2380151378Snetchild if (VTNET_TXQ_TRYLOCK(txq) != 0) { 2381151378Snetchild error = vtnet_txq_mq_start_locked(txq, m); 2382151378Snetchild VTNET_TXQ_UNLOCK(txq); 2383151378Snetchild } else { 2384151378Snetchild error = drbr_enqueue(ifp, txq->vtntx_br, m); 2385151378Snetchild taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); 2386151378Snetchild } 2387151378Snetchild 2388151378Snetchild return (error); 2389151378Snetchild} 2390151378Snetchild 2391151378Snetchildstatic void 2392151378Snetchildvtnet_txq_tq_deferred(void *xtxq, int pending) 2393151378Snetchild{ 2394151378Snetchild struct vtnet_softc *sc; 2395151378Snetchild struct vtnet_txq *txq; 2396151378Snetchild 2397151378Snetchild txq = xtxq; 2398151378Snetchild sc = txq->vtntx_sc; 2399151378Snetchild 2400151378Snetchild VTNET_TXQ_LOCK(txq); 2401151378Snetchild if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) 2402151378Snetchild vtnet_txq_mq_start_locked(txq, NULL); 2403151378Snetchild VTNET_TXQ_UNLOCK(txq); 2404151378Snetchild} 2405151378Snetchild 2406151378Snetchild#endif /* VTNET_LEGACY_TX */ 2407151378Snetchild 2408151378Snetchildstatic void 2409151378Snetchildvtnet_txq_start(struct vtnet_txq *txq) 2410151378Snetchild{ 2411151378Snetchild struct vtnet_softc *sc; 2412151378Snetchild struct ifnet *ifp; 2413151378Snetchild 2414151378Snetchild sc = txq->vtntx_sc; 2415151378Snetchild ifp = sc->vtnet_ifp; 2416151378Snetchild 2417151378Snetchild#ifdef VTNET_LEGACY_TX 2418151378Snetchild if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 2419151378Snetchild vtnet_start_locked(txq, ifp); 2420151378Snetchild#else 2421151378Snetchild if (!drbr_empty(ifp, txq->vtntx_br)) 2422151378Snetchild vtnet_txq_mq_start_locked(txq, NULL); 2423151378Snetchild#endif 2424151378Snetchild} 2425151378Snetchild 2426151378Snetchildstatic void 2427151378Snetchildvtnet_txq_tq_intr(void *xtxq, int pending) 2428151378Snetchild{ 2429151378Snetchild struct vtnet_softc *sc; 2430151378Snetchild struct vtnet_txq *txq; 2431151378Snetchild struct ifnet *ifp; 2432151378Snetchild 2433151378Snetchild txq = xtxq; 2434151378Snetchild sc = txq->vtntx_sc; 2435151378Snetchild ifp = sc->vtnet_ifp; 2436151378Snetchild 2437151378Snetchild VTNET_TXQ_LOCK(txq); 2438151378Snetchild 2439151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2440151378Snetchild VTNET_TXQ_UNLOCK(txq); 2441151378Snetchild return; 2442151378Snetchild } 2443151378Snetchild 2444151378Snetchild vtnet_txq_eof(txq); 2445151378Snetchild vtnet_txq_start(txq); 2446151378Snetchild 2447151378Snetchild VTNET_TXQ_UNLOCK(txq); 2448151378Snetchild} 2449151378Snetchild 2450151378Snetchildstatic int 2451151378Snetchildvtnet_txq_eof(struct vtnet_txq *txq) 2452151378Snetchild{ 2453151378Snetchild struct virtqueue *vq; 2454151378Snetchild struct vtnet_tx_header *txhdr; 2455151378Snetchild struct mbuf *m; 2456151378Snetchild int deq; 2457151378Snetchild 2458151378Snetchild vq = txq->vtntx_vq; 2459151378Snetchild deq = 0; 2460151378Snetchild VTNET_TXQ_LOCK_ASSERT(txq); 2461151378Snetchild 2462151378Snetchild#ifdef DEV_NETMAP 2463151378Snetchild if (netmap_tx_irq(txq->vtntx_sc->vtnet_ifp, txq->vtntx_id)) { 2464151378Snetchild virtqueue_disable_intr(vq); // XXX luigi 2465151378Snetchild return 0; // XXX or 1 ? 2466151378Snetchild } 2467151378Snetchild#endif /* DEV_NETMAP */ 2468151378Snetchild 2469151378Snetchild while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { 2470151378Snetchild m = txhdr->vth_mbuf; 2471151378Snetchild deq++; 2472151378Snetchild 2473151378Snetchild txq->vtntx_stats.vtxs_opackets++; 2474151378Snetchild txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; 2475151378Snetchild if (m->m_flags & M_MCAST) 2476151378Snetchild txq->vtntx_stats.vtxs_omcasts++; 2477151378Snetchild 2478151378Snetchild m_freem(m); 2479151378Snetchild uma_zfree(vtnet_tx_header_zone, txhdr); 2480151378Snetchild } 2481151378Snetchild 2482151378Snetchild if (virtqueue_empty(vq)) 2483151378Snetchild txq->vtntx_watchdog = 0; 2484151378Snetchild 2485151378Snetchild return (deq); 2486151378Snetchild} 2487151378Snetchild 2488151378Snetchildstatic void 2489151378Snetchildvtnet_tx_vq_intr(void *xtxq) 2490151378Snetchild{ 2491151378Snetchild struct vtnet_softc *sc; 2492151378Snetchild struct vtnet_txq *txq; 2493151378Snetchild struct ifnet *ifp; 2494151378Snetchild 2495151378Snetchild txq = xtxq; 2496151378Snetchild sc = txq->vtntx_sc; 2497151378Snetchild ifp = sc->vtnet_ifp; 2498151378Snetchild 2499151378Snetchild if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { 2500151378Snetchild /* 2501151378Snetchild * Ignore this interrupt. Either this is a spurious interrupt 2502151378Snetchild * or multiqueue without per-VQ MSIX so every queue needs to 2503151378Snetchild * be polled (a brain dead configuration we could try harder 2504151378Snetchild * to avoid). 2505151378Snetchild */ 2506151378Snetchild vtnet_txq_disable_intr(txq); 2507151378Snetchild return; 2508151378Snetchild } 2509151378Snetchild 2510151378Snetchild VTNET_TXQ_LOCK(txq); 2511151378Snetchild 2512151378Snetchild if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2513151378Snetchild VTNET_TXQ_UNLOCK(txq); 2514151378Snetchild return; 2515151378Snetchild } 2516151378Snetchild 2517151378Snetchild vtnet_txq_eof(txq); 2518151378Snetchild vtnet_txq_start(txq); 2519151378Snetchild 2520151378Snetchild VTNET_TXQ_UNLOCK(txq); 2521151378Snetchild} 2522151378Snetchild 2523151378Snetchildstatic void 2524151378Snetchildvtnet_tx_start_all(struct vtnet_softc *sc) 2525151378Snetchild{ 2526151378Snetchild struct vtnet_txq *txq; 2527151378Snetchild int i; 2528151378Snetchild 2529151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 2530151378Snetchild 2531151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2532151378Snetchild txq = &sc->vtnet_txqs[i]; 2533151378Snetchild 2534151378Snetchild VTNET_TXQ_LOCK(txq); 2535151378Snetchild vtnet_txq_start(txq); 2536151378Snetchild VTNET_TXQ_UNLOCK(txq); 2537151378Snetchild } 2538151378Snetchild} 2539151378Snetchild 2540151378Snetchild#ifndef VTNET_LEGACY_TX 2541151378Snetchildstatic void 2542151378Snetchildvtnet_qflush(struct ifnet *ifp) 2543151378Snetchild{ 2544151378Snetchild struct vtnet_softc *sc; 2545151378Snetchild struct vtnet_txq *txq; 2546151378Snetchild struct mbuf *m; 2547151378Snetchild int i; 2548151378Snetchild 2549151378Snetchild sc = ifp->if_softc; 2550151378Snetchild 2551151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2552151378Snetchild txq = &sc->vtnet_txqs[i]; 2553151378Snetchild 2554151378Snetchild VTNET_TXQ_LOCK(txq); 2555151378Snetchild while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) 2556151378Snetchild m_freem(m); 2557151378Snetchild VTNET_TXQ_UNLOCK(txq); 2558151378Snetchild } 2559151378Snetchild 2560151378Snetchild if_qflush(ifp); 2561151378Snetchild} 2562151378Snetchild#endif 2563151378Snetchild 2564151378Snetchildstatic int 2565151378Snetchildvtnet_watchdog(struct vtnet_txq *txq) 2566151378Snetchild{ 2567151378Snetchild struct ifnet *ifp; 2568151378Snetchild 2569151378Snetchild ifp = txq->vtntx_sc->vtnet_ifp; 2570151378Snetchild 2571151378Snetchild VTNET_TXQ_LOCK(txq); 2572151378Snetchild if (txq->vtntx_watchdog == 1) { 2573151378Snetchild /* 2574151378Snetchild * Only drain completed frames if the watchdog is about to 2575151378Snetchild * expire. If any frames were drained, there may be enough 2576151378Snetchild * free descriptors now available to transmit queued frames. 2577151378Snetchild * In that case, the timer will immediately be decremented 2578151378Snetchild * below, but the timeout is generous enough that should not 2579151378Snetchild * be a problem. 2580151378Snetchild */ 2581151378Snetchild if (vtnet_txq_eof(txq) != 0) 2582151378Snetchild vtnet_txq_start(txq); 2583151378Snetchild } 2584151378Snetchild 2585151378Snetchild if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { 2586151378Snetchild VTNET_TXQ_UNLOCK(txq); 2587151378Snetchild return (0); 2588151378Snetchild } 2589151378Snetchild VTNET_TXQ_UNLOCK(txq); 2590151378Snetchild 2591151378Snetchild if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id); 2592151378Snetchild return (1); 2593151378Snetchild} 2594151378Snetchild 2595151378Snetchildstatic void 2596151378Snetchildvtnet_accum_stats(struct vtnet_softc *sc, struct vtnet_rxq_stats *rxacc, 2597151378Snetchild struct vtnet_txq_stats *txacc) 2598151378Snetchild{ 2599151378Snetchild 2600151378Snetchild bzero(rxacc, sizeof(struct vtnet_rxq_stats)); 2601151378Snetchild bzero(txacc, sizeof(struct vtnet_txq_stats)); 2602151378Snetchild 2603151378Snetchild for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2604151378Snetchild struct vtnet_rxq_stats *rxst; 2605151378Snetchild struct vtnet_txq_stats *txst; 2606151378Snetchild 2607151378Snetchild rxst = &sc->vtnet_rxqs[i].vtnrx_stats; 2608151378Snetchild rxacc->vrxs_ipackets += rxst->vrxs_ipackets; 2609151378Snetchild rxacc->vrxs_ibytes += rxst->vrxs_ibytes; 2610151378Snetchild rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops; 2611151378Snetchild rxacc->vrxs_csum += rxst->vrxs_csum; 2612151378Snetchild rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed; 2613151378Snetchild rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled; 2614151378Snetchild 2615151378Snetchild txst = &sc->vtnet_txqs[i].vtntx_stats; 2616151378Snetchild txacc->vtxs_opackets += txst->vtxs_opackets; 2617151378Snetchild txacc->vtxs_obytes += txst->vtxs_obytes; 2618151378Snetchild txacc->vtxs_csum += txst->vtxs_csum; 2619151378Snetchild txacc->vtxs_tso += txst->vtxs_tso; 2620151378Snetchild txacc->vtxs_rescheduled += txst->vtxs_rescheduled; 2621151378Snetchild } 2622151378Snetchild} 2623151378Snetchild 2624151378Snetchildstatic uint64_t 2625151378Snetchildvtnet_get_counter(if_t ifp, ift_counter cnt) 2626151378Snetchild{ 2627151378Snetchild struct vtnet_softc *sc; 2628151378Snetchild struct vtnet_rxq_stats rxaccum; 2629151378Snetchild struct vtnet_txq_stats txaccum; 2630151378Snetchild 2631151378Snetchild sc = if_getsoftc(ifp); 2632151378Snetchild vtnet_accum_stats(sc, &rxaccum, &txaccum); 2633151378Snetchild 2634151378Snetchild switch (cnt) { 2635151378Snetchild case IFCOUNTER_IPACKETS: 2636151378Snetchild return (rxaccum.vrxs_ipackets); 2637151378Snetchild case IFCOUNTER_IQDROPS: 2638151378Snetchild return (rxaccum.vrxs_iqdrops); 2639151378Snetchild case IFCOUNTER_IERRORS: 2640151378Snetchild return (rxaccum.vrxs_ierrors); 2641151378Snetchild case IFCOUNTER_OPACKETS: 2642151378Snetchild return (txaccum.vtxs_opackets); 2643151378Snetchild#ifndef VTNET_LEGACY_TX 2644151378Snetchild case IFCOUNTER_OBYTES: 2645151378Snetchild return (txaccum.vtxs_obytes); 2646151378Snetchild case IFCOUNTER_OMCASTS: 2647151378Snetchild return (txaccum.vtxs_omcasts); 2648151378Snetchild#endif 2649151378Snetchild default: 2650151378Snetchild return (if_get_counter_default(ifp, cnt)); 2651151378Snetchild } 2652151378Snetchild} 2653151378Snetchild 2654151378Snetchildstatic void 2655151378Snetchildvtnet_tick(void *xsc) 2656151378Snetchild{ 2657151378Snetchild struct vtnet_softc *sc; 2658151378Snetchild struct ifnet *ifp; 2659151378Snetchild int i, timedout; 2660151378Snetchild 2661151378Snetchild sc = xsc; 2662151378Snetchild ifp = sc->vtnet_ifp; 2663151378Snetchild timedout = 0; 2664151378Snetchild 2665151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 2666151378Snetchild 2667151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 2668151378Snetchild timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); 2669151378Snetchild 2670151378Snetchild if (timedout != 0) { 2671151378Snetchild ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2672151378Snetchild vtnet_init_locked(sc); 2673151378Snetchild } else 2674151378Snetchild callout_schedule(&sc->vtnet_tick_ch, hz); 2675151378Snetchild} 2676151378Snetchild 2677151378Snetchildstatic void 2678151378Snetchildvtnet_start_taskqueues(struct vtnet_softc *sc) 2679151378Snetchild{ 2680151378Snetchild device_t dev; 2681151378Snetchild struct vtnet_rxq *rxq; 2682151378Snetchild struct vtnet_txq *txq; 2683151378Snetchild int i, error; 2684151378Snetchild 2685151378Snetchild dev = sc->vtnet_dev; 2686151378Snetchild 2687151378Snetchild /* 2688151378Snetchild * Errors here are very difficult to recover from - we cannot 2689151378Snetchild * easily fail because, if this is during boot, we will hang 2690151378Snetchild * when freeing any successfully started taskqueues because 2691151378Snetchild * the scheduler isn't up yet. 2692151378Snetchild * 2693151378Snetchild * Most drivers just ignore the return value - it only fails 2694151378Snetchild * with ENOMEM so an error is not likely. 2695151378Snetchild */ 2696151378Snetchild for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2697151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2698151378Snetchild error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, 2699151378Snetchild "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); 2700151378Snetchild if (error) { 2701151378Snetchild device_printf(dev, "failed to start rx taskq %d\n", 2702151378Snetchild rxq->vtnrx_id); 2703151378Snetchild } 2704151378Snetchild 2705151378Snetchild txq = &sc->vtnet_txqs[i]; 2706151378Snetchild error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, 2707151378Snetchild "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); 2708151378Snetchild if (error) { 2709151378Snetchild device_printf(dev, "failed to start tx taskq %d\n", 2710151378Snetchild txq->vtntx_id); 2711151378Snetchild } 2712151378Snetchild } 2713151378Snetchild} 2714151378Snetchild 2715151378Snetchildstatic void 2716151378Snetchildvtnet_free_taskqueues(struct vtnet_softc *sc) 2717151378Snetchild{ 2718151378Snetchild struct vtnet_rxq *rxq; 2719151378Snetchild struct vtnet_txq *txq; 2720151378Snetchild int i; 2721151378Snetchild 2722151378Snetchild for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2723151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2724151378Snetchild if (rxq->vtnrx_tq != NULL) { 2725151378Snetchild taskqueue_free(rxq->vtnrx_tq); 2726151378Snetchild rxq->vtnrx_vq = NULL; 2727151378Snetchild } 2728151378Snetchild 2729151378Snetchild txq = &sc->vtnet_txqs[i]; 2730151378Snetchild if (txq->vtntx_tq != NULL) { 2731151378Snetchild taskqueue_free(txq->vtntx_tq); 2732151378Snetchild txq->vtntx_tq = NULL; 2733151378Snetchild } 2734151378Snetchild } 2735151378Snetchild} 2736151378Snetchild 2737151378Snetchildstatic void 2738151378Snetchildvtnet_drain_taskqueues(struct vtnet_softc *sc) 2739151378Snetchild{ 2740151378Snetchild struct vtnet_rxq *rxq; 2741151378Snetchild struct vtnet_txq *txq; 2742151378Snetchild int i; 2743151378Snetchild 2744151378Snetchild for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2745151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2746151378Snetchild if (rxq->vtnrx_tq != NULL) 2747151378Snetchild taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 2748151378Snetchild 2749151378Snetchild txq = &sc->vtnet_txqs[i]; 2750151378Snetchild if (txq->vtntx_tq != NULL) { 2751151378Snetchild taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); 2752151378Snetchild#ifndef VTNET_LEGACY_TX 2753151378Snetchild taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); 2754151378Snetchild#endif 2755151378Snetchild } 2756151378Snetchild } 2757151378Snetchild} 2758151378Snetchild 2759151378Snetchildstatic void 2760151378Snetchildvtnet_drain_rxtx_queues(struct vtnet_softc *sc) 2761151378Snetchild{ 2762151378Snetchild struct vtnet_rxq *rxq; 2763151378Snetchild struct vtnet_txq *txq; 2764151378Snetchild int i; 2765151378Snetchild 2766151378Snetchild#ifdef DEV_NETMAP 2767151378Snetchild if (nm_native_on(NA(sc->vtnet_ifp))) 2768151378Snetchild return; 2769151378Snetchild#endif /* DEV_NETMAP */ 2770151378Snetchild 2771151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2772151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2773151378Snetchild vtnet_rxq_free_mbufs(rxq); 2774151378Snetchild 2775151378Snetchild txq = &sc->vtnet_txqs[i]; 2776151378Snetchild vtnet_txq_free_mbufs(txq); 2777151378Snetchild } 2778151378Snetchild} 2779151378Snetchild 2780151378Snetchildstatic void 2781151378Snetchildvtnet_stop_rendezvous(struct vtnet_softc *sc) 2782151378Snetchild{ 2783151378Snetchild struct vtnet_rxq *rxq; 2784151378Snetchild struct vtnet_txq *txq; 2785151378Snetchild int i; 2786151378Snetchild 2787151378Snetchild /* 2788151378Snetchild * Lock and unlock the per-queue mutex so we known the stop 2789151378Snetchild * state is visible. Doing only the active queues should be 2790151378Snetchild * sufficient, but it does not cost much extra to do all the 2791151378Snetchild * queues. Note we hold the core mutex here too. 2792151378Snetchild */ 2793151378Snetchild for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2794151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2795151378Snetchild VTNET_RXQ_LOCK(rxq); 2796151378Snetchild VTNET_RXQ_UNLOCK(rxq); 2797151378Snetchild 2798151378Snetchild txq = &sc->vtnet_txqs[i]; 2799151378Snetchild VTNET_TXQ_LOCK(txq); 2800151378Snetchild VTNET_TXQ_UNLOCK(txq); 2801151378Snetchild } 2802151378Snetchild} 2803151378Snetchild 2804151378Snetchildstatic void 2805151378Snetchildvtnet_stop(struct vtnet_softc *sc) 2806151378Snetchild{ 2807151378Snetchild device_t dev; 2808151378Snetchild struct ifnet *ifp; 2809151378Snetchild 2810151378Snetchild dev = sc->vtnet_dev; 2811151378Snetchild ifp = sc->vtnet_ifp; 2812151378Snetchild 2813151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 2814151378Snetchild 2815151378Snetchild ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2816151378Snetchild sc->vtnet_link_active = 0; 2817151378Snetchild callout_stop(&sc->vtnet_tick_ch); 2818151378Snetchild 2819151378Snetchild /* Only advisory. */ 2820151378Snetchild vtnet_disable_interrupts(sc); 2821151378Snetchild 2822151378Snetchild /* 2823151378Snetchild * Stop the host adapter. This resets it to the pre-initialized 2824151378Snetchild * state. It will not generate any interrupts until after it is 2825151378Snetchild * reinitialized. 2826151378Snetchild */ 2827151378Snetchild virtio_stop(dev); 2828151378Snetchild vtnet_stop_rendezvous(sc); 2829151378Snetchild 2830151378Snetchild /* Free any mbufs left in the virtqueues. */ 2831151378Snetchild vtnet_drain_rxtx_queues(sc); 2832151378Snetchild} 2833151378Snetchild 2834151378Snetchildstatic int 2835151378Snetchildvtnet_virtio_reinit(struct vtnet_softc *sc) 2836151378Snetchild{ 2837151378Snetchild device_t dev; 2838151378Snetchild struct ifnet *ifp; 2839151378Snetchild uint64_t features; 2840151378Snetchild int mask, error; 2841151378Snetchild 2842151378Snetchild dev = sc->vtnet_dev; 2843151378Snetchild ifp = sc->vtnet_ifp; 2844151378Snetchild features = sc->vtnet_features; 2845151378Snetchild 2846151378Snetchild mask = 0; 2847151378Snetchild#if defined(INET) 2848151378Snetchild mask |= IFCAP_RXCSUM; 2849151378Snetchild#endif 2850151378Snetchild#if defined (INET6) 2851151378Snetchild mask |= IFCAP_RXCSUM_IPV6; 2852151378Snetchild#endif 2853151378Snetchild 2854151378Snetchild /* 2855151378Snetchild * Re-negotiate with the host, removing any disabled receive 2856151378Snetchild * features. Transmit features are disabled only on our side 2857151378Snetchild * via if_capenable and if_hwassist. 2858151378Snetchild */ 2859151378Snetchild 2860151378Snetchild if (ifp->if_capabilities & mask) { 2861151378Snetchild /* 2862151378Snetchild * We require both IPv4 and IPv6 offloading to be enabled 2863151378Snetchild * in order to negotiated it: VirtIO does not distinguish 2864151378Snetchild * between the two. 2865151378Snetchild */ 2866151378Snetchild if ((ifp->if_capenable & mask) != mask) 2867151378Snetchild features &= ~VIRTIO_NET_F_GUEST_CSUM; 2868151378Snetchild } 2869151378Snetchild 2870151378Snetchild if (ifp->if_capabilities & IFCAP_LRO) { 2871151378Snetchild if ((ifp->if_capenable & IFCAP_LRO) == 0) 2872151378Snetchild features &= ~VTNET_LRO_FEATURES; 2873151378Snetchild } 2874151378Snetchild 2875151378Snetchild if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) { 2876151378Snetchild if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 2877151378Snetchild features &= ~VIRTIO_NET_F_CTRL_VLAN; 2878151378Snetchild } 2879151378Snetchild 2880151378Snetchild error = virtio_reinit(dev, features); 2881151378Snetchild if (error) 2882151378Snetchild device_printf(dev, "virtio reinit error %d\n", error); 2883151378Snetchild 2884151378Snetchild return (error); 2885151378Snetchild} 2886151378Snetchild 2887151378Snetchildstatic void 2888151378Snetchildvtnet_init_rx_filters(struct vtnet_softc *sc) 2889151378Snetchild{ 2890151378Snetchild struct ifnet *ifp; 2891151378Snetchild 2892151378Snetchild ifp = sc->vtnet_ifp; 2893151378Snetchild 2894151378Snetchild if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 2895151378Snetchild /* Restore promiscuous and all-multicast modes. */ 2896151378Snetchild vtnet_rx_filter(sc); 2897151378Snetchild /* Restore filtered MAC addresses. */ 2898151378Snetchild vtnet_rx_filter_mac(sc); 2899151378Snetchild } 2900151378Snetchild 2901151378Snetchild if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2902151378Snetchild vtnet_rx_filter_vlan(sc); 2903151378Snetchild} 2904151378Snetchild 2905151378Snetchildstatic int 2906151378Snetchildvtnet_init_rx_queues(struct vtnet_softc *sc) 2907151378Snetchild{ 2908151378Snetchild device_t dev; 2909151378Snetchild struct vtnet_rxq *rxq; 2910151378Snetchild int i, clsize, error; 2911151378Snetchild 2912151378Snetchild dev = sc->vtnet_dev; 2913151378Snetchild 2914151378Snetchild /* 2915151378Snetchild * Use the new cluster size if one has been set (via a MTU 2916151378Snetchild * change). Otherwise, use the standard 2K clusters. 2917151378Snetchild * 2918151378Snetchild * BMV: It might make sense to use page sized clusters as 2919151378Snetchild * the default (depending on the features negotiated). 2920151378Snetchild */ 2921151378Snetchild if (sc->vtnet_rx_new_clsize != 0) { 2922151378Snetchild clsize = sc->vtnet_rx_new_clsize; 2923151378Snetchild sc->vtnet_rx_new_clsize = 0; 2924151378Snetchild } else 2925151378Snetchild clsize = MCLBYTES; 2926151378Snetchild 2927151378Snetchild sc->vtnet_rx_clsize = clsize; 2928151378Snetchild sc->vtnet_rx_nmbufs = VTNET_NEEDED_RX_MBUFS(sc, clsize); 2929151378Snetchild 2930151378Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS || 2931151378Snetchild sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, 2932151378Snetchild ("%s: too many rx mbufs %d for %d segments", __func__, 2933151378Snetchild sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); 2934151378Snetchild 2935151378Snetchild#ifdef DEV_NETMAP 2936151378Snetchild if (vtnet_netmap_init_rx_buffers(sc)) 2937151378Snetchild return 0; 2938151378Snetchild#endif /* DEV_NETMAP */ 2939151378Snetchild 2940151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2941151378Snetchild rxq = &sc->vtnet_rxqs[i]; 2942151378Snetchild 2943151378Snetchild /* Hold the lock to satisfy asserts. */ 2944151378Snetchild VTNET_RXQ_LOCK(rxq); 2945151378Snetchild error = vtnet_rxq_populate(rxq); 2946151378Snetchild VTNET_RXQ_UNLOCK(rxq); 2947151378Snetchild 2948151378Snetchild if (error) { 2949151378Snetchild device_printf(dev, 2950151378Snetchild "cannot allocate mbufs for Rx queue %d\n", i); 2951151378Snetchild return (error); 2952151378Snetchild } 2953151378Snetchild } 2954151378Snetchild 2955151378Snetchild return (0); 2956151378Snetchild} 2957151378Snetchild 2958151378Snetchildstatic int 2959151378Snetchildvtnet_init_tx_queues(struct vtnet_softc *sc) 2960151378Snetchild{ 2961151378Snetchild struct vtnet_txq *txq; 2962151378Snetchild int i; 2963151378Snetchild 2964151378Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2965151378Snetchild txq = &sc->vtnet_txqs[i]; 2966151378Snetchild txq->vtntx_watchdog = 0; 2967151378Snetchild } 2968151378Snetchild 2969151378Snetchild return (0); 2970151378Snetchild} 2971151378Snetchild 2972151378Snetchildstatic int 2973151378Snetchildvtnet_init_rxtx_queues(struct vtnet_softc *sc) 2974151378Snetchild{ 2975151378Snetchild int error; 2976151378Snetchild 2977151378Snetchild error = vtnet_init_rx_queues(sc); 2978151378Snetchild if (error) 2979151378Snetchild return (error); 2980151378Snetchild 2981151378Snetchild error = vtnet_init_tx_queues(sc); 2982151378Snetchild if (error) 2983151378Snetchild return (error); 2984151378Snetchild 2985151378Snetchild return (0); 2986151378Snetchild} 2987151378Snetchild 2988151378Snetchildstatic void 2989151378Snetchildvtnet_set_active_vq_pairs(struct vtnet_softc *sc) 2990151378Snetchild{ 2991151378Snetchild device_t dev; 2992151378Snetchild int npairs; 2993151378Snetchild 2994151378Snetchild dev = sc->vtnet_dev; 2995151378Snetchild 2996151378Snetchild if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) { 2997151378Snetchild sc->vtnet_act_vq_pairs = 1; 2998151378Snetchild return; 2999151378Snetchild } 3000151378Snetchild 3001151378Snetchild npairs = sc->vtnet_requested_vq_pairs; 3002151378Snetchild 3003151378Snetchild if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { 3004151378Snetchild device_printf(dev, 3005151378Snetchild "cannot set active queue pairs to %d\n", npairs); 3006151378Snetchild npairs = 1; 3007151378Snetchild } 3008151378Snetchild 3009151378Snetchild sc->vtnet_act_vq_pairs = npairs; 3010151378Snetchild} 3011151378Snetchild 3012151378Snetchildstatic int 3013151378Snetchildvtnet_reinit(struct vtnet_softc *sc) 3014151378Snetchild{ 3015151378Snetchild struct ifnet *ifp; 3016151378Snetchild int error; 3017151378Snetchild 3018151378Snetchild ifp = sc->vtnet_ifp; 3019151378Snetchild 3020151378Snetchild /* Use the current MAC address. */ 3021151378Snetchild bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); 3022151378Snetchild vtnet_set_hwaddr(sc); 3023151378Snetchild 3024151378Snetchild vtnet_set_active_vq_pairs(sc); 3025151378Snetchild 3026151378Snetchild ifp->if_hwassist = 0; 3027151378Snetchild if (ifp->if_capenable & IFCAP_TXCSUM) 3028151378Snetchild ifp->if_hwassist |= VTNET_CSUM_OFFLOAD; 3029151378Snetchild if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 3030151378Snetchild ifp->if_hwassist |= VTNET_CSUM_OFFLOAD_IPV6; 3031151378Snetchild if (ifp->if_capenable & IFCAP_TSO4) 3032151378Snetchild ifp->if_hwassist |= CSUM_IP_TSO; 3033151378Snetchild if (ifp->if_capenable & IFCAP_TSO6) 3034151378Snetchild ifp->if_hwassist |= CSUM_IP6_TSO; 3035151378Snetchild 3036151378Snetchild if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 3037151378Snetchild vtnet_init_rx_filters(sc); 3038151378Snetchild 3039151378Snetchild error = vtnet_init_rxtx_queues(sc); 3040151378Snetchild if (error) 3041151378Snetchild return (error); 3042151378Snetchild 3043151378Snetchild vtnet_enable_interrupts(sc); 3044151378Snetchild ifp->if_drv_flags |= IFF_DRV_RUNNING; 3045151378Snetchild 3046151378Snetchild return (0); 3047151378Snetchild} 3048151378Snetchild 3049151378Snetchildstatic void 3050151378Snetchildvtnet_init_locked(struct vtnet_softc *sc) 3051151378Snetchild{ 3052151378Snetchild device_t dev; 3053151378Snetchild struct ifnet *ifp; 3054151378Snetchild 3055151378Snetchild dev = sc->vtnet_dev; 3056151378Snetchild ifp = sc->vtnet_ifp; 3057151378Snetchild 3058151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3059151378Snetchild 3060151378Snetchild if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3061151378Snetchild return; 3062151378Snetchild 3063151378Snetchild vtnet_stop(sc); 3064151378Snetchild 3065151378Snetchild /* Reinitialize with the host. */ 3066151378Snetchild if (vtnet_virtio_reinit(sc) != 0) 3067151378Snetchild goto fail; 3068151378Snetchild 3069151378Snetchild if (vtnet_reinit(sc) != 0) 3070151378Snetchild goto fail; 3071151378Snetchild 3072151378Snetchild virtio_reinit_complete(dev); 3073151378Snetchild 3074151378Snetchild vtnet_update_link_status(sc); 3075151378Snetchild callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 3076151378Snetchild 3077151378Snetchild return; 3078151378Snetchild 3079151378Snetchildfail: 3080151378Snetchild vtnet_stop(sc); 3081151378Snetchild} 3082151378Snetchild 3083151378Snetchildstatic void 3084151378Snetchildvtnet_init(void *xsc) 3085151378Snetchild{ 3086151378Snetchild struct vtnet_softc *sc; 3087151378Snetchild 3088151378Snetchild sc = xsc; 3089151378Snetchild 3090151378Snetchild#ifdef DEV_NETMAP 3091151378Snetchild if (!NA(sc->vtnet_ifp)) { 3092151378Snetchild D("try to attach again"); 3093151378Snetchild vtnet_netmap_attach(sc); 3094151378Snetchild } 3095151378Snetchild#endif /* DEV_NETMAP */ 3096151378Snetchild 3097151378Snetchild VTNET_CORE_LOCK(sc); 3098151378Snetchild vtnet_init_locked(sc); 3099151378Snetchild VTNET_CORE_UNLOCK(sc); 3100151378Snetchild} 3101151378Snetchild 3102151378Snetchildstatic void 3103151378Snetchildvtnet_free_ctrl_vq(struct vtnet_softc *sc) 3104151378Snetchild{ 3105151378Snetchild struct virtqueue *vq; 3106151378Snetchild 3107151378Snetchild vq = sc->vtnet_ctrl_vq; 3108151378Snetchild 3109151378Snetchild /* 3110151378Snetchild * The control virtqueue is only polled and therefore it should 3111151378Snetchild * already be empty. 3112151378Snetchild */ 3113151378Snetchild KASSERT(virtqueue_empty(vq), 3114151378Snetchild ("%s: ctrl vq %p not empty", __func__, vq)); 3115151378Snetchild} 3116151378Snetchild 3117151378Snetchildstatic void 3118151378Snetchildvtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie, 3119151378Snetchild struct sglist *sg, int readable, int writable) 3120151378Snetchild{ 3121151378Snetchild struct virtqueue *vq; 3122151378Snetchild 3123151378Snetchild vq = sc->vtnet_ctrl_vq; 3124151378Snetchild 3125151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3126151378Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ, 3127151378Snetchild ("%s: CTRL_VQ feature not negotiated", __func__)); 3128151378Snetchild 3129151378Snetchild if (!virtqueue_empty(vq)) 3130151378Snetchild return; 3131151378Snetchild if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0) 3132151378Snetchild return; 3133151378Snetchild 3134151378Snetchild /* 3135151378Snetchild * Poll for the response, but the command is likely already 3136151378Snetchild * done when we return from the notify. 3137151378Snetchild */ 3138151378Snetchild virtqueue_notify(vq); 3139151378Snetchild virtqueue_poll(vq, NULL); 3140151378Snetchild} 3141151378Snetchild 3142151378Snetchildstatic int 3143151378Snetchildvtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr) 3144151378Snetchild{ 3145151378Snetchild struct virtio_net_ctrl_hdr hdr __aligned(2); 3146151378Snetchild struct sglist_seg segs[3]; 3147151378Snetchild struct sglist sg; 3148151378Snetchild uint8_t ack; 3149151378Snetchild int error; 3150151378Snetchild 3151151378Snetchild hdr.class = VIRTIO_NET_CTRL_MAC; 3152151378Snetchild hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 3153151378Snetchild ack = VIRTIO_NET_ERR; 3154151378Snetchild 3155151378Snetchild sglist_init(&sg, 3, segs); 3156151378Snetchild error = 0; 3157151378Snetchild error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3158151378Snetchild error |= sglist_append(&sg, hwaddr, ETHER_ADDR_LEN); 3159151378Snetchild error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3160151378Snetchild KASSERT(error == 0 && sg.sg_nseg == 3, 3161151378Snetchild ("%s: error %d adding set MAC msg to sglist", __func__, error)); 3162151378Snetchild 3163151378Snetchild vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3164151378Snetchild 3165151378Snetchild return (ack == VIRTIO_NET_OK ? 0 : EIO); 3166151378Snetchild} 3167151378Snetchild 3168151378Snetchildstatic int 3169151378Snetchildvtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs) 3170151378Snetchild{ 3171151378Snetchild struct sglist_seg segs[3]; 3172151378Snetchild struct sglist sg; 3173151378Snetchild struct { 3174151378Snetchild struct virtio_net_ctrl_hdr hdr; 3175151378Snetchild uint8_t pad1; 3176151378Snetchild struct virtio_net_ctrl_mq mq; 3177151378Snetchild uint8_t pad2; 3178151378Snetchild uint8_t ack; 3179151378Snetchild } s __aligned(2); 3180151378Snetchild int error; 3181151378Snetchild 3182151378Snetchild s.hdr.class = VIRTIO_NET_CTRL_MQ; 3183151378Snetchild s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 3184151378Snetchild s.mq.virtqueue_pairs = npairs; 3185151378Snetchild s.ack = VIRTIO_NET_ERR; 3186151378Snetchild 3187151378Snetchild sglist_init(&sg, 3, segs); 3188151378Snetchild error = 0; 3189151378Snetchild error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3190151378Snetchild error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq)); 3191151378Snetchild error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3192151378Snetchild KASSERT(error == 0 && sg.sg_nseg == 3, 3193151378Snetchild ("%s: error %d adding MQ message to sglist", __func__, error)); 3194151378Snetchild 3195151378Snetchild vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3196151378Snetchild 3197151378Snetchild return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3198151378Snetchild} 3199151378Snetchild 3200151378Snetchildstatic int 3201151378Snetchildvtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on) 3202151378Snetchild{ 3203151378Snetchild struct sglist_seg segs[3]; 3204151378Snetchild struct sglist sg; 3205151378Snetchild struct { 3206151378Snetchild struct virtio_net_ctrl_hdr hdr; 3207151378Snetchild uint8_t pad1; 3208151378Snetchild uint8_t onoff; 3209151378Snetchild uint8_t pad2; 3210151378Snetchild uint8_t ack; 3211151378Snetchild } s __aligned(2); 3212151378Snetchild int error; 3213151378Snetchild 3214151378Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3215151378Snetchild ("%s: CTRL_RX feature not negotiated", __func__)); 3216151378Snetchild 3217151378Snetchild s.hdr.class = VIRTIO_NET_CTRL_RX; 3218151378Snetchild s.hdr.cmd = cmd; 3219151378Snetchild s.onoff = !!on; 3220151378Snetchild s.ack = VIRTIO_NET_ERR; 3221151378Snetchild 3222151378Snetchild sglist_init(&sg, 3, segs); 3223151378Snetchild error = 0; 3224151378Snetchild error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3225151378Snetchild error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t)); 3226151378Snetchild error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3227151378Snetchild KASSERT(error == 0 && sg.sg_nseg == 3, 3228151378Snetchild ("%s: error %d adding Rx message to sglist", __func__, error)); 3229151378Snetchild 3230151378Snetchild vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3231151378Snetchild 3232151378Snetchild return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3233151378Snetchild} 3234151378Snetchild 3235151378Snetchildstatic int 3236151378Snetchildvtnet_set_promisc(struct vtnet_softc *sc, int on) 3237151378Snetchild{ 3238151378Snetchild 3239151378Snetchild return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); 3240151378Snetchild} 3241151378Snetchild 3242151378Snetchildstatic int 3243151378Snetchildvtnet_set_allmulti(struct vtnet_softc *sc, int on) 3244151378Snetchild{ 3245151378Snetchild 3246151378Snetchild return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); 3247151378Snetchild} 3248151378Snetchild 3249151378Snetchild/* 3250151378Snetchild * The device defaults to promiscuous mode for backwards compatibility. 3251151378Snetchild * Turn it off at attach time if possible. 3252151378Snetchild */ 3253151378Snetchildstatic void 3254151378Snetchildvtnet_attach_disable_promisc(struct vtnet_softc *sc) 3255151378Snetchild{ 3256151378Snetchild struct ifnet *ifp; 3257151378Snetchild 3258151378Snetchild ifp = sc->vtnet_ifp; 3259151378Snetchild 3260151378Snetchild VTNET_CORE_LOCK(sc); 3261151378Snetchild if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) { 3262151378Snetchild ifp->if_flags |= IFF_PROMISC; 3263151378Snetchild } else if (vtnet_set_promisc(sc, 0) != 0) { 3264151378Snetchild ifp->if_flags |= IFF_PROMISC; 3265151378Snetchild device_printf(sc->vtnet_dev, 3266151378Snetchild "cannot disable default promiscuous mode\n"); 3267151378Snetchild } 3268151378Snetchild VTNET_CORE_UNLOCK(sc); 3269151378Snetchild} 3270151378Snetchild 3271151378Snetchildstatic void 3272151378Snetchildvtnet_rx_filter(struct vtnet_softc *sc) 3273151378Snetchild{ 3274151378Snetchild device_t dev; 3275151378Snetchild struct ifnet *ifp; 3276151378Snetchild 3277151378Snetchild dev = sc->vtnet_dev; 3278151378Snetchild ifp = sc->vtnet_ifp; 3279151378Snetchild 3280151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3281151378Snetchild 3282151378Snetchild if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0) 3283151378Snetchild device_printf(dev, "cannot %s promiscuous mode\n", 3284151378Snetchild ifp->if_flags & IFF_PROMISC ? "enable" : "disable"); 3285151378Snetchild 3286151378Snetchild if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0) 3287151378Snetchild device_printf(dev, "cannot %s all-multicast mode\n", 3288151378Snetchild ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable"); 3289151378Snetchild} 3290151378Snetchild 3291151378Snetchildstatic void 3292151378Snetchildvtnet_rx_filter_mac(struct vtnet_softc *sc) 3293151378Snetchild{ 3294151378Snetchild struct virtio_net_ctrl_hdr hdr __aligned(2); 3295151378Snetchild struct vtnet_mac_filter *filter; 3296151378Snetchild struct sglist_seg segs[4]; 3297151378Snetchild struct sglist sg; 3298151378Snetchild struct ifnet *ifp; 3299151378Snetchild struct ifaddr *ifa; 3300151378Snetchild struct ifmultiaddr *ifma; 3301151378Snetchild int ucnt, mcnt, promisc, allmulti, error; 3302151378Snetchild uint8_t ack; 3303151378Snetchild 3304151378Snetchild ifp = sc->vtnet_ifp; 3305151378Snetchild filter = sc->vtnet_mac_filter; 3306151378Snetchild ucnt = 0; 3307151378Snetchild mcnt = 0; 3308151378Snetchild promisc = 0; 3309151378Snetchild allmulti = 0; 3310151378Snetchild 3311151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3312151378Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3313151378Snetchild ("%s: CTRL_RX feature not negotiated", __func__)); 3314151378Snetchild 3315151378Snetchild /* Unicast MAC addresses: */ 3316151378Snetchild if_addr_rlock(ifp); 3317151378Snetchild TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 3318151378Snetchild if (ifa->ifa_addr->sa_family != AF_LINK) 3319151378Snetchild continue; 3320151378Snetchild else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3321151378Snetchild sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) 3322151378Snetchild continue; 3323151378Snetchild else if (ucnt == VTNET_MAX_MAC_ENTRIES) { 3324151378Snetchild promisc = 1; 3325151378Snetchild break; 3326151378Snetchild } 3327151378Snetchild 3328151378Snetchild bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3329151378Snetchild &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN); 3330151378Snetchild ucnt++; 3331151378Snetchild } 3332151378Snetchild if_addr_runlock(ifp); 3333151378Snetchild 3334151378Snetchild if (promisc != 0) { 3335151378Snetchild filter->vmf_unicast.nentries = 0; 3336151378Snetchild if_printf(ifp, "more than %d MAC addresses assigned, " 3337151378Snetchild "falling back to promiscuous mode\n", 3338151378Snetchild VTNET_MAX_MAC_ENTRIES); 3339151378Snetchild } else 3340151378Snetchild filter->vmf_unicast.nentries = ucnt; 3341151378Snetchild 3342151378Snetchild /* Multicast MAC addresses: */ 3343151378Snetchild if_maddr_rlock(ifp); 3344151378Snetchild TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3345151378Snetchild if (ifma->ifma_addr->sa_family != AF_LINK) 3346151378Snetchild continue; 3347151378Snetchild else if (mcnt == VTNET_MAX_MAC_ENTRIES) { 3348151378Snetchild allmulti = 1; 3349151378Snetchild break; 3350151378Snetchild } 3351151378Snetchild 3352151378Snetchild bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3353151378Snetchild &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN); 3354151378Snetchild mcnt++; 3355151378Snetchild } 3356151378Snetchild if_maddr_runlock(ifp); 3357151378Snetchild 3358151378Snetchild if (allmulti != 0) { 3359151378Snetchild filter->vmf_multicast.nentries = 0; 3360151378Snetchild if_printf(ifp, "more than %d multicast MAC addresses " 3361151378Snetchild "assigned, falling back to all-multicast mode\n", 3362151378Snetchild VTNET_MAX_MAC_ENTRIES); 3363151378Snetchild } else 3364151378Snetchild filter->vmf_multicast.nentries = mcnt; 3365151378Snetchild 3366151378Snetchild if (promisc != 0 && allmulti != 0) 3367151378Snetchild goto out; 3368151378Snetchild 3369151378Snetchild hdr.class = VIRTIO_NET_CTRL_MAC; 3370151378Snetchild hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 3371151378Snetchild ack = VIRTIO_NET_ERR; 3372151378Snetchild 3373151378Snetchild sglist_init(&sg, 4, segs); 3374151378Snetchild error = 0; 3375151378Snetchild error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3376151378Snetchild error |= sglist_append(&sg, &filter->vmf_unicast, 3377151378Snetchild sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN); 3378151378Snetchild error |= sglist_append(&sg, &filter->vmf_multicast, 3379151378Snetchild sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN); 3380151378Snetchild error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3381151378Snetchild KASSERT(error == 0 && sg.sg_nseg == 4, 3382151378Snetchild ("%s: error %d adding MAC filter msg to sglist", __func__, error)); 3383151378Snetchild 3384151378Snetchild vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3385151378Snetchild 3386151378Snetchild if (ack != VIRTIO_NET_OK) 3387151378Snetchild if_printf(ifp, "error setting host MAC filter table\n"); 3388151378Snetchild 3389151378Snetchildout: 3390151378Snetchild if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0) 3391151378Snetchild if_printf(ifp, "cannot enable promiscuous mode\n"); 3392151378Snetchild if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0) 3393151378Snetchild if_printf(ifp, "cannot enable all-multicast mode\n"); 3394151378Snetchild} 3395151378Snetchild 3396151378Snetchildstatic int 3397151378Snetchildvtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3398151378Snetchild{ 3399151378Snetchild struct sglist_seg segs[3]; 3400151378Snetchild struct sglist sg; 3401151378Snetchild struct { 3402151378Snetchild struct virtio_net_ctrl_hdr hdr; 3403151378Snetchild uint8_t pad1; 3404151378Snetchild uint16_t tag; 3405151378Snetchild uint8_t pad2; 3406151378Snetchild uint8_t ack; 3407151378Snetchild } s __aligned(2); 3408151378Snetchild int error; 3409151378Snetchild 3410151378Snetchild s.hdr.class = VIRTIO_NET_CTRL_VLAN; 3411151378Snetchild s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 3412151378Snetchild s.tag = tag; 3413151378Snetchild s.ack = VIRTIO_NET_ERR; 3414151378Snetchild 3415151378Snetchild sglist_init(&sg, 3, segs); 3416151378Snetchild error = 0; 3417151378Snetchild error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3418151378Snetchild error |= sglist_append(&sg, &s.tag, sizeof(uint16_t)); 3419151378Snetchild error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3420151378Snetchild KASSERT(error == 0 && sg.sg_nseg == 3, 3421151378Snetchild ("%s: error %d adding VLAN message to sglist", __func__, error)); 3422151378Snetchild 3423151378Snetchild vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3424151378Snetchild 3425151378Snetchild return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3426151378Snetchild} 3427151378Snetchild 3428151378Snetchildstatic void 3429151378Snetchildvtnet_rx_filter_vlan(struct vtnet_softc *sc) 3430151378Snetchild{ 3431151378Snetchild uint32_t w; 3432151378Snetchild uint16_t tag; 3433151378Snetchild int i, bit; 3434151378Snetchild 3435151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3436151378Snetchild KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER, 3437151378Snetchild ("%s: VLAN_FILTER feature not negotiated", __func__)); 3438151378Snetchild 3439151378Snetchild /* Enable the filter for each configured VLAN. */ 3440151378Snetchild for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) { 3441151378Snetchild w = sc->vtnet_vlan_filter[i]; 3442151378Snetchild 3443151378Snetchild while ((bit = ffs(w) - 1) != -1) { 3444151378Snetchild w &= ~(1 << bit); 3445151378Snetchild tag = sizeof(w) * CHAR_BIT * i + bit; 3446151378Snetchild 3447151378Snetchild if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) { 3448151378Snetchild device_printf(sc->vtnet_dev, 3449153939Snetchild "cannot enable VLAN %d filter\n", tag); 3450153939Snetchild } 3451153939Snetchild } 3452153939Snetchild } 3453153939Snetchild} 3454153939Snetchild 3455153939Snetchildstatic void 3456153939Snetchildvtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3457153939Snetchild{ 3458153939Snetchild struct ifnet *ifp; 3459153939Snetchild int idx, bit; 3460153939Snetchild 3461153939Snetchild ifp = sc->vtnet_ifp; 3462153939Snetchild idx = (tag >> 5) & 0x7F; 3463153939Snetchild bit = tag & 0x1F; 3464153939Snetchild 3465153939Snetchild if (tag == 0 || tag > 4095) 3466153939Snetchild return; 3467153939Snetchild 3468151378Snetchild VTNET_CORE_LOCK(sc); 3469151378Snetchild 3470151378Snetchild if (add) 3471151378Snetchild sc->vtnet_vlan_filter[idx] |= (1 << bit); 3472151378Snetchild else 3473151378Snetchild sc->vtnet_vlan_filter[idx] &= ~(1 << bit); 3474151378Snetchild 3475151378Snetchild if (ifp->if_capenable & IFCAP_VLAN_HWFILTER && 3476151378Snetchild vtnet_exec_vlan_filter(sc, add, tag) != 0) { 3477151378Snetchild device_printf(sc->vtnet_dev, 3478151378Snetchild "cannot %s VLAN %d %s the host filter table\n", 3479151378Snetchild add ? "add" : "remove", tag, add ? "to" : "from"); 3480151378Snetchild } 3481151378Snetchild 3482151378Snetchild VTNET_CORE_UNLOCK(sc); 3483151378Snetchild} 3484151378Snetchild 3485151378Snetchildstatic void 3486151378Snetchildvtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3487151378Snetchild{ 3488151378Snetchild 3489151378Snetchild if (ifp->if_softc != arg) 3490151378Snetchild return; 3491151378Snetchild 3492151378Snetchild vtnet_update_vlan_filter(arg, 1, tag); 3493151378Snetchild} 3494151378Snetchild 3495151378Snetchildstatic void 3496151378Snetchildvtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3497151378Snetchild{ 3498151378Snetchild 3499151378Snetchild if (ifp->if_softc != arg) 3500151378Snetchild return; 3501151378Snetchild 3502151378Snetchild vtnet_update_vlan_filter(arg, 0, tag); 3503151378Snetchild} 3504151378Snetchild 3505151378Snetchildstatic int 3506151378Snetchildvtnet_is_link_up(struct vtnet_softc *sc) 3507151378Snetchild{ 3508151378Snetchild device_t dev; 3509151378Snetchild struct ifnet *ifp; 3510151378Snetchild uint16_t status; 3511151378Snetchild 3512151378Snetchild dev = sc->vtnet_dev; 3513151378Snetchild ifp = sc->vtnet_ifp; 3514151378Snetchild 3515151378Snetchild if ((ifp->if_capabilities & IFCAP_LINKSTATE) == 0) 3516151378Snetchild status = VIRTIO_NET_S_LINK_UP; 3517151378Snetchild else 3518151378Snetchild status = virtio_read_dev_config_2(dev, 3519151378Snetchild offsetof(struct virtio_net_config, status)); 3520151378Snetchild 3521151378Snetchild return ((status & VIRTIO_NET_S_LINK_UP) != 0); 3522151378Snetchild} 3523151378Snetchild 3524151378Snetchildstatic void 3525151378Snetchildvtnet_update_link_status(struct vtnet_softc *sc) 3526151378Snetchild{ 3527151378Snetchild struct ifnet *ifp; 3528151378Snetchild int link; 3529151378Snetchild 3530151378Snetchild ifp = sc->vtnet_ifp; 3531151378Snetchild 3532151378Snetchild VTNET_CORE_LOCK_ASSERT(sc); 3533151378Snetchild link = vtnet_is_link_up(sc); 3534151378Snetchild 3535151378Snetchild /* Notify if the link status has changed. */ 3536151378Snetchild if (link != 0 && sc->vtnet_link_active == 0) { 3537153939Snetchild sc->vtnet_link_active = 1; 3538153939Snetchild if_link_state_change(ifp, LINK_STATE_UP); 3539153939Snetchild } else if (link == 0 && sc->vtnet_link_active != 0) { 3540153939Snetchild sc->vtnet_link_active = 0; 3541153939Snetchild if_link_state_change(ifp, LINK_STATE_DOWN); 3542153939Snetchild } 3543153939Snetchild} 3544153939Snetchild 3545153939Snetchildstatic int 3546153939Snetchildvtnet_ifmedia_upd(struct ifnet *ifp) 3547153939Snetchild{ 3548153939Snetchild struct vtnet_softc *sc; 3549153939Snetchild struct ifmedia *ifm; 3550153939Snetchild 3551153939Snetchild sc = ifp->if_softc; 3552153939Snetchild ifm = &sc->vtnet_media; 3553153939Snetchild 3554153939Snetchild if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3555153939Snetchild return (EINVAL); 3556153939Snetchild 3557153939Snetchild return (0); 3558153939Snetchild} 3559153939Snetchild 3560153939Snetchildstatic void 3561153939Snetchildvtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 3562153939Snetchild{ 3563153939Snetchild struct vtnet_softc *sc; 3564153939Snetchild 3565153939Snetchild sc = ifp->if_softc; 3566153939Snetchild 3567153939Snetchild ifmr->ifm_status = IFM_AVALID; 3568153939Snetchild ifmr->ifm_active = IFM_ETHER; 3569153939Snetchild 3570153939Snetchild VTNET_CORE_LOCK(sc); 3571153939Snetchild if (vtnet_is_link_up(sc) != 0) { 3572153939Snetchild ifmr->ifm_status |= IFM_ACTIVE; 3573153939Snetchild ifmr->ifm_active |= VTNET_MEDIATYPE; 3574153939Snetchild } else 3575153939Snetchild ifmr->ifm_active |= IFM_NONE; 3576153939Snetchild VTNET_CORE_UNLOCK(sc); 3577153939Snetchild} 3578153939Snetchild 3579153939Snetchildstatic void 3580153939Snetchildvtnet_set_hwaddr(struct vtnet_softc *sc) 3581153939Snetchild{ 3582153939Snetchild device_t dev; 3583153939Snetchild int i; 3584153939Snetchild 3585153939Snetchild dev = sc->vtnet_dev; 3586153939Snetchild 3587153939Snetchild if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { 3588153939Snetchild if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0) 3589153939Snetchild device_printf(dev, "unable to set MAC address\n"); 3590153939Snetchild } else if (sc->vtnet_flags & VTNET_FLAG_MAC) { 3591153939Snetchild for (i = 0; i < ETHER_ADDR_LEN; i++) { 3592153939Snetchild virtio_write_dev_config_1(dev, 3593153939Snetchild offsetof(struct virtio_net_config, mac) + i, 3594153939Snetchild sc->vtnet_hwaddr[i]); 3595153939Snetchild } 3596153939Snetchild } 3597153939Snetchild} 3598153939Snetchild 3599153939Snetchildstatic void 3600153939Snetchildvtnet_get_hwaddr(struct vtnet_softc *sc) 3601153939Snetchild{ 3602153939Snetchild device_t dev; 3603153939Snetchild int i; 3604153939Snetchild 3605151378Snetchild dev = sc->vtnet_dev; 3606153939Snetchild 3607153939Snetchild if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) { 3608153939Snetchild /* 3609153939Snetchild * Generate a random locally administered unicast address. 3610153939Snetchild * 3611153939Snetchild * It would be nice to generate the same MAC address across 3612153939Snetchild * reboots, but it seems all the hosts currently available 3613153939Snetchild * support the MAC feature, so this isn't too important. 3614153939Snetchild */ 3615153939Snetchild sc->vtnet_hwaddr[0] = 0xB2; 3616153939Snetchild arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); 3617153939Snetchild vtnet_set_hwaddr(sc); 3618153939Snetchild return; 3619153939Snetchild } 3620153939Snetchild 3621153939Snetchild for (i = 0; i < ETHER_ADDR_LEN; i++) { 3622153939Snetchild sc->vtnet_hwaddr[i] = virtio_read_dev_config_1(dev, 3623153939Snetchild offsetof(struct virtio_net_config, mac) + i); 3624153939Snetchild } 3625153939Snetchild} 3626153939Snetchild 3627153939Snetchildstatic void 3628153939Snetchildvtnet_vlan_tag_remove(struct mbuf *m) 3629153939Snetchild{ 3630153939Snetchild struct ether_vlan_header *evh; 3631153939Snetchild 3632153939Snetchild evh = mtod(m, struct ether_vlan_header *); 3633153939Snetchild m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 3634153939Snetchild m->m_flags |= M_VLANTAG; 3635153939Snetchild 3636153939Snetchild /* Strip the 802.1Q header. */ 3637153939Snetchild bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 3638153939Snetchild ETHER_HDR_LEN - ETHER_TYPE_LEN); 3639153939Snetchild m_adj(m, ETHER_VLAN_ENCAP_LEN); 3640153939Snetchild} 3641153939Snetchild 3642153939Snetchildstatic void 3643153939Snetchildvtnet_set_rx_process_limit(struct vtnet_softc *sc) 3644153939Snetchild{ 3645151378Snetchild int limit; 3646151378Snetchild 3647151378Snetchild limit = vtnet_tunable_int(sc, "rx_process_limit", 3648151378Snetchild vtnet_rx_process_limit); 3649151378Snetchild if (limit < 0) 3650151378Snetchild limit = INT_MAX; 3651151378Snetchild sc->vtnet_rx_process_limit = limit; 3652151378Snetchild} 3653151378Snetchild 3654151378Snetchildstatic void 3655151378Snetchildvtnet_set_tx_intr_threshold(struct vtnet_softc *sc) 3656151378Snetchild{ 3657151378Snetchild int size, thresh; 3658151378Snetchild 3659151378Snetchild size = virtqueue_size(sc->vtnet_txqs[0].vtntx_vq); 3660151378Snetchild 3661151378Snetchild /* 3662151378Snetchild * The Tx interrupt is disabled until the queue free count falls 3663151378Snetchild * below our threshold. Completed frames are drained from the Tx 3664151378Snetchild * virtqueue before transmitting new frames and in the watchdog 3665151378Snetchild * callout, so the frequency of Tx interrupts is greatly reduced, 3666151378Snetchild * at the cost of not freeing mbufs as quickly as they otherwise 3667151378Snetchild * would be. 3668151378Snetchild * 3669151378Snetchild * N.B. We assume all the Tx queues are the same size. 3670151378Snetchild */ 3671151378Snetchild thresh = size / 4; 3672151378Snetchild 3673151378Snetchild /* 3674151378Snetchild * Without indirect descriptors, leave enough room for the most 3675151378Snetchild * segments we handle. 3676151378Snetchild */ 3677151378Snetchild if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 && 3678151378Snetchild thresh < sc->vtnet_tx_nsegs) 3679151378Snetchild thresh = sc->vtnet_tx_nsegs; 3680151378Snetchild 3681151378Snetchild sc->vtnet_tx_intr_thresh = thresh; 3682151378Snetchild} 3683151378Snetchild 3684151378Snetchildstatic void 3685151378Snetchildvtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, 3686151378Snetchild struct sysctl_oid_list *child, struct vtnet_rxq *rxq) 3687151378Snetchild{ 3688151378Snetchild struct sysctl_oid *node; 3689151378Snetchild struct sysctl_oid_list *list; 3690151378Snetchild struct vtnet_rxq_stats *stats; 3691151378Snetchild char namebuf[16]; 3692151378Snetchild 3693151378Snetchild snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); 3694151378Snetchild node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3695151378Snetchild CTLFLAG_RD, NULL, "Receive Queue"); 3696151378Snetchild list = SYSCTL_CHILDREN(node); 3697151378Snetchild 3698151378Snetchild stats = &rxq->vtnrx_stats; 3699151378Snetchild 3700151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3701151378Snetchild &stats->vrxs_ipackets, "Receive packets"); 3702151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3703151378Snetchild &stats->vrxs_ibytes, "Receive bytes"); 3704151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3705151378Snetchild &stats->vrxs_iqdrops, "Receive drops"); 3706151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3707151378Snetchild &stats->vrxs_ierrors, "Receive errors"); 3708151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3709151378Snetchild &stats->vrxs_csum, "Receive checksum offloaded"); 3710151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD, 3711151378Snetchild &stats->vrxs_csum_failed, "Receive checksum offload failed"); 3712151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3713151378Snetchild &stats->vrxs_rescheduled, 3714151378Snetchild "Receive interrupt handler rescheduled"); 3715151378Snetchild} 3716151378Snetchild 3717151378Snetchildstatic void 3718151378Snetchildvtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, 3719151378Snetchild struct sysctl_oid_list *child, struct vtnet_txq *txq) 3720151378Snetchild{ 3721151378Snetchild struct sysctl_oid *node; 3722151378Snetchild struct sysctl_oid_list *list; 3723151378Snetchild struct vtnet_txq_stats *stats; 3724151378Snetchild char namebuf[16]; 3725151378Snetchild 3726151378Snetchild snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); 3727151378Snetchild node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3728151378Snetchild CTLFLAG_RD, NULL, "Transmit Queue"); 3729151378Snetchild list = SYSCTL_CHILDREN(node); 3730151378Snetchild 3731151378Snetchild stats = &txq->vtntx_stats; 3732151378Snetchild 3733151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3734151378Snetchild &stats->vtxs_opackets, "Transmit packets"); 3735151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3736151378Snetchild &stats->vtxs_obytes, "Transmit bytes"); 3737151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3738151378Snetchild &stats->vtxs_omcasts, "Transmit multicasts"); 3739151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3740151378Snetchild &stats->vtxs_csum, "Transmit checksum offloaded"); 3741151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3742151378Snetchild &stats->vtxs_tso, "Transmit segmentation offloaded"); 3743151378Snetchild SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3744151378Snetchild &stats->vtxs_rescheduled, 3745161201Snetchild "Transmit interrupt handler rescheduled"); 3746161201Snetchild} 3747164620Snetchild 3748164620Snetchildstatic void 3749164620Snetchildvtnet_setup_queue_sysctl(struct vtnet_softc *sc) 3750164620Snetchild{ 3751164620Snetchild device_t dev; 3752164620Snetchild struct sysctl_ctx_list *ctx; 3753164620Snetchild struct sysctl_oid *tree; 3754164620Snetchild struct sysctl_oid_list *child; 3755164620Snetchild int i; 3756164620Snetchild 3757164620Snetchild dev = sc->vtnet_dev; 3758164620Snetchild ctx = device_get_sysctl_ctx(dev); 3759164620Snetchild tree = device_get_sysctl_tree(dev); 3760164620Snetchild child = SYSCTL_CHILDREN(tree); 3761164620Snetchild 3762164620Snetchild for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3763164620Snetchild vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); 3764164620Snetchild vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); 3765164620Snetchild } 3766164620Snetchild} 3767164620Snetchild 3768164620Snetchildstatic void 3769164620Snetchildvtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, 3770164620Snetchild struct sysctl_oid_list *child, struct vtnet_softc *sc) 3771164620Snetchild{ 3772164620Snetchild struct vtnet_statistics *stats; 3773164620Snetchild struct vtnet_rxq_stats rxaccum; 3774164620Snetchild struct vtnet_txq_stats txaccum; 3775164620Snetchild 3776164620Snetchild vtnet_accum_stats(sc, &rxaccum, &txaccum); 3777164620Snetchild 3778164620Snetchild stats = &sc->vtnet_stats; 3779164620Snetchild stats->rx_csum_offloaded = rxaccum.vrxs_csum; 3780164620Snetchild stats->rx_csum_failed = rxaccum.vrxs_csum_failed; 3781164620Snetchild stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled; 3782164620Snetchild stats->tx_csum_offloaded = txaccum.vtxs_csum; 3783164620Snetchild stats->tx_tso_offloaded = txaccum.vtxs_tso; 3784164620Snetchild stats->tx_task_rescheduled = txaccum.vtxs_rescheduled; 3785164620Snetchild 3786164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", 3787164620Snetchild CTLFLAG_RD, &stats->mbuf_alloc_failed, 3788164620Snetchild "Mbuf cluster allocation failures"); 3789164620Snetchild 3790164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", 3791164620Snetchild CTLFLAG_RD, &stats->rx_frame_too_large, 3792164620Snetchild "Received frame larger than the mbuf chain"); 3793164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", 3794164620Snetchild CTLFLAG_RD, &stats->rx_enq_replacement_failed, 3795164620Snetchild "Enqueuing the replacement receive mbuf failed"); 3796164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", 3797164620Snetchild CTLFLAG_RD, &stats->rx_mergeable_failed, 3798164620Snetchild "Mergeable buffers receive failures"); 3799164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", 3800164620Snetchild CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 3801164620Snetchild "Received checksum offloaded buffer with unsupported " 3802164620Snetchild "Ethernet type"); 3803164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", 3804164620Snetchild CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 3805164620Snetchild "Received checksum offloaded buffer with incorrect IP protocol"); 3806164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset", 3807164620Snetchild CTLFLAG_RD, &stats->rx_csum_bad_offset, 3808164620Snetchild "Received checksum offloaded buffer with incorrect offset"); 3809164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto", 3810164620Snetchild CTLFLAG_RD, &stats->rx_csum_bad_proto, 3811164620Snetchild "Received checksum offloaded buffer with incorrect protocol"); 3812164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed", 3813164620Snetchild CTLFLAG_RD, &stats->rx_csum_failed, 3814164620Snetchild "Received buffer checksum offload failed"); 3815164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded", 3816164620Snetchild CTLFLAG_RD, &stats->rx_csum_offloaded, 3817164620Snetchild "Received buffer checksum offload succeeded"); 3818164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled", 3819164620Snetchild CTLFLAG_RD, &stats->rx_task_rescheduled, 3820164620Snetchild "Times the receive interrupt task rescheduled itself"); 3821164620Snetchild 3822164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype", 3823164620Snetchild CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 3824164620Snetchild "Aborted transmit of checksum offloaded buffer with unknown " 3825164620Snetchild "Ethernet type"); 3826164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype", 3827164620Snetchild CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 3828164620Snetchild "Aborted transmit of TSO buffer with unknown Ethernet type"); 3829164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", 3830164620Snetchild CTLFLAG_RD, &stats->tx_tso_not_tcp, 3831164620Snetchild "Aborted transmit of TSO buffer with non TCP protocol"); 3832164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", 3833164620Snetchild CTLFLAG_RD, &stats->tx_defragged, 3834164620Snetchild "Transmit mbufs defragged"); 3835164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", 3836164620Snetchild CTLFLAG_RD, &stats->tx_defrag_failed, 3837164620Snetchild "Aborted transmit of buffer because defrag failed"); 3838164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded", 3839164620Snetchild CTLFLAG_RD, &stats->tx_csum_offloaded, 3840164620Snetchild "Offloaded checksum of transmitted buffer"); 3841164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded", 3842164620Snetchild CTLFLAG_RD, &stats->tx_tso_offloaded, 3843164620Snetchild "Segmentation offload of transmitted buffer"); 3844164620Snetchild SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled", 3845164620Snetchild CTLFLAG_RD, &stats->tx_task_rescheduled, 3846164620Snetchild "Times the transmit interrupt task rescheduled itself"); 3847164620Snetchild} 3848164620Snetchild 3849164620Snetchildstatic void 3850164620Snetchildvtnet_setup_sysctl(struct vtnet_softc *sc) 3851164620Snetchild{ 3852164620Snetchild device_t dev; 3853164620Snetchild struct sysctl_ctx_list *ctx; 3854164620Snetchild struct sysctl_oid *tree; 3855164620Snetchild struct sysctl_oid_list *child; 3856164620Snetchild 3857164620Snetchild dev = sc->vtnet_dev; 3858164620Snetchild ctx = device_get_sysctl_ctx(dev); 3859164620Snetchild tree = device_get_sysctl_tree(dev); 3860164620Snetchild child = SYSCTL_CHILDREN(tree); 3861164620Snetchild 3862164620Snetchild SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", 3863164620Snetchild CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, 3864164620Snetchild "Maximum number of supported virtqueue pairs"); 3865164620Snetchild SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs", 3866164620Snetchild CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0, 3867164620Snetchild "Requested number of virtqueue pairs"); 3868164620Snetchild SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", 3869164620Snetchild CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, 3870164620Snetchild "Number of active virtqueue pairs"); 3871164620Snetchild 3872164620Snetchild vtnet_setup_stat_sysctl(ctx, child, sc); 3873164620Snetchild} 3874164620Snetchild 3875164620Snetchildstatic int 3876164620Snetchildvtnet_rxq_enable_intr(struct vtnet_rxq *rxq) 3877164620Snetchild{ 3878164620Snetchild 3879164620Snetchild return (virtqueue_enable_intr(rxq->vtnrx_vq)); 3880164620Snetchild} 3881164620Snetchild 3882164620Snetchildstatic void 3883164620Snetchildvtnet_rxq_disable_intr(struct vtnet_rxq *rxq) 3884164620Snetchild{ 3885164620Snetchild 3886164620Snetchild virtqueue_disable_intr(rxq->vtnrx_vq); 3887164620Snetchild} 3888164620Snetchild 3889164620Snetchildstatic int 3890164620Snetchildvtnet_txq_enable_intr(struct vtnet_txq *txq) 3891164620Snetchild{ 3892164620Snetchild struct virtqueue *vq; 3893164620Snetchild 3894164620Snetchild vq = txq->vtntx_vq; 3895164620Snetchild 3896164620Snetchild if (vtnet_txq_below_threshold(txq) != 0) 3897164620Snetchild return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG)); 3898164620Snetchild 3899164620Snetchild /* 3900164620Snetchild * The free count is above our threshold. Keep the Tx interrupt 3901164620Snetchild * disabled until the queue is fuller. 3902164620Snetchild */ 3903164620Snetchild return (0); 3904164620Snetchild} 3905164620Snetchild 3906164620Snetchildstatic void 3907164620Snetchildvtnet_txq_disable_intr(struct vtnet_txq *txq) 3908164620Snetchild{ 3909164620Snetchild 3910164620Snetchild virtqueue_disable_intr(txq->vtntx_vq); 3911164620Snetchild} 3912164620Snetchild 3913164620Snetchildstatic void 3914164620Snetchildvtnet_enable_rx_interrupts(struct vtnet_softc *sc) 3915164620Snetchild{ 3916164620Snetchild int i; 3917164620Snetchild 3918164620Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3919164620Snetchild vtnet_rxq_enable_intr(&sc->vtnet_rxqs[i]); 3920164620Snetchild} 3921164620Snetchild 3922164620Snetchildstatic void 3923164620Snetchildvtnet_enable_tx_interrupts(struct vtnet_softc *sc) 3924164620Snetchild{ 3925164620Snetchild int i; 3926164620Snetchild 3927164620Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3928164620Snetchild vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); 3929164620Snetchild} 3930164620Snetchild 3931164620Snetchildstatic void 3932164620Snetchildvtnet_enable_interrupts(struct vtnet_softc *sc) 3933164620Snetchild{ 3934164620Snetchild 3935164620Snetchild vtnet_enable_rx_interrupts(sc); 3936164620Snetchild vtnet_enable_tx_interrupts(sc); 3937164620Snetchild} 3938164620Snetchild 3939164620Snetchildstatic void 3940164620Snetchildvtnet_disable_rx_interrupts(struct vtnet_softc *sc) 3941164620Snetchild{ 3942164620Snetchild int i; 3943164620Snetchild 3944164620Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3945164620Snetchild vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); 3946164620Snetchild} 3947164620Snetchild 3948164620Snetchildstatic void 3949164620Snetchildvtnet_disable_tx_interrupts(struct vtnet_softc *sc) 3950164620Snetchild{ 3951164620Snetchild int i; 3952164620Snetchild 3953164620Snetchild for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3954164620Snetchild vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); 3955164620Snetchild} 3956164620Snetchild 3957164620Snetchildstatic void 3958164620Snetchildvtnet_disable_interrupts(struct vtnet_softc *sc) 3959164620Snetchild{ 3960164620Snetchild 3961164620Snetchild vtnet_disable_rx_interrupts(sc); 3962164620Snetchild vtnet_disable_tx_interrupts(sc); 3963164620Snetchild} 3964164620Snetchild 3965164620Snetchildstatic int 3966164620Snetchildvtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def) 3967164620Snetchild{ 3968164620Snetchild char path[64]; 3969164620Snetchild 3970164620Snetchild snprintf(path, sizeof(path), 3971164620Snetchild "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); 3972164620Snetchild TUNABLE_INT_FETCH(path, &def); 3973164620Snetchild 3974164620Snetchild return (def); 3975164620Snetchild} 3976164620Snetchild