1/*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27/* Driver for VirtIO network devices. */ 28 29#include <sys/cdefs.h>
| 1/*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27/* Driver for VirtIO network devices. */ 28 29#include <sys/cdefs.h>
|
30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/network/if_vtnet.c 267279 2014-06-09 15:09:05Z luigi $");
| 30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/network/if_vtnet.c 268010 2014-06-29 00:37:59Z bryanv $");
|
31 32#include <sys/param.h> 33#include <sys/systm.h> 34#include <sys/kernel.h> 35#include <sys/sockio.h> 36#include <sys/mbuf.h> 37#include <sys/malloc.h> 38#include <sys/module.h> 39#include <sys/socket.h> 40#include <sys/sysctl.h> 41#include <sys/random.h> 42#include <sys/sglist.h> 43#include <sys/lock.h> 44#include <sys/mutex.h> 45#include <sys/taskqueue.h> 46#include <sys/smp.h> 47#include <machine/smp.h> 48 49#include <vm/uma.h> 50 51#include <net/ethernet.h> 52#include <net/if.h> 53#include <net/if_arp.h> 54#include <net/if_dl.h> 55#include <net/if_types.h> 56#include <net/if_media.h> 57#include <net/if_vlan_var.h> 58 59#include <net/bpf.h> 60 61#include <netinet/in_systm.h> 62#include <netinet/in.h> 63#include <netinet/ip.h> 64#include <netinet/ip6.h> 65#include <netinet6/ip6_var.h> 66#include <netinet/udp.h> 67#include <netinet/tcp.h> 68#include <netinet/sctp.h> 69 70#include <machine/bus.h> 71#include <machine/resource.h> 72#include <sys/bus.h> 73#include <sys/rman.h> 74 75#include <dev/virtio/virtio.h> 76#include <dev/virtio/virtqueue.h> 77#include <dev/virtio/network/virtio_net.h> 78#include <dev/virtio/network/if_vtnetvar.h> 79 80#include "virtio_if.h" 81 82#include "opt_inet.h" 83#include "opt_inet6.h" 84 85static int vtnet_modevent(module_t, int, void *); 86 87static int vtnet_probe(device_t); 88static int vtnet_attach(device_t); 89static int vtnet_detach(device_t); 90static int vtnet_suspend(device_t); 91static int vtnet_resume(device_t); 92static int vtnet_shutdown(device_t); 93static int vtnet_attach_completed(device_t); 94static int vtnet_config_change(device_t); 95 96static void vtnet_negotiate_features(struct vtnet_softc *); 97static void vtnet_setup_features(struct vtnet_softc *); 98static int vtnet_init_rxq(struct vtnet_softc *, int); 99static int vtnet_init_txq(struct vtnet_softc *, int); 100static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); 101static void vtnet_free_rxtx_queues(struct vtnet_softc *); 102static int vtnet_alloc_rx_filters(struct vtnet_softc *); 103static void vtnet_free_rx_filters(struct vtnet_softc *); 104static int vtnet_alloc_virtqueues(struct vtnet_softc *); 105static int vtnet_setup_interface(struct vtnet_softc *); 106static int vtnet_change_mtu(struct vtnet_softc *, int); 107static int vtnet_ioctl(struct ifnet *, u_long, caddr_t); 108 109static int vtnet_rxq_populate(struct vtnet_rxq *); 110static void vtnet_rxq_free_mbufs(struct vtnet_rxq *); 111static struct mbuf * 112 vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **); 113static int vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *, 114 struct mbuf *, int); 115static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); 116static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); 117static int vtnet_rxq_new_buf(struct vtnet_rxq *); 118static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, 119 struct virtio_net_hdr *); 120static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); 121static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); 122static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); 123static void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *, 124 struct virtio_net_hdr *); 125static int vtnet_rxq_eof(struct vtnet_rxq *); 126static void vtnet_rx_vq_intr(void *); 127static void vtnet_rxq_tq_intr(void *, int); 128 129static void vtnet_txq_free_mbufs(struct vtnet_txq *); 130static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, 131 int *, int *, int *); 132static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, 133 int, struct virtio_net_hdr *); 134static struct mbuf * 135 vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, 136 struct virtio_net_hdr *); 137static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, 138 struct vtnet_tx_header *); 139static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **); 140#ifdef VTNET_LEGACY_TX 141static void vtnet_start_locked(struct vtnet_txq *, struct ifnet *); 142static void vtnet_start(struct ifnet *); 143#else 144static int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *); 145static int vtnet_txq_mq_start(struct ifnet *, struct mbuf *); 146static void vtnet_txq_tq_deferred(void *, int); 147#endif 148static void vtnet_txq_start(struct vtnet_txq *); 149static void vtnet_txq_tq_intr(void *, int); 150static void vtnet_txq_eof(struct vtnet_txq *); 151static void vtnet_tx_vq_intr(void *); 152static void vtnet_tx_start_all(struct vtnet_softc *); 153 154#ifndef VTNET_LEGACY_TX 155static void vtnet_qflush(struct ifnet *); 156#endif 157 158static int vtnet_watchdog(struct vtnet_txq *); 159static void vtnet_rxq_accum_stats(struct vtnet_rxq *, 160 struct vtnet_rxq_stats *); 161static void vtnet_txq_accum_stats(struct vtnet_txq *, 162 struct vtnet_txq_stats *); 163static void vtnet_accumulate_stats(struct vtnet_softc *); 164static void vtnet_tick(void *); 165 166static void vtnet_start_taskqueues(struct vtnet_softc *); 167static void vtnet_free_taskqueues(struct vtnet_softc *); 168static void vtnet_drain_taskqueues(struct vtnet_softc *); 169 170static void vtnet_drain_rxtx_queues(struct vtnet_softc *); 171static void vtnet_stop_rendezvous(struct vtnet_softc *); 172static void vtnet_stop(struct vtnet_softc *); 173static int vtnet_virtio_reinit(struct vtnet_softc *); 174static void vtnet_init_rx_filters(struct vtnet_softc *); 175static int vtnet_init_rx_queues(struct vtnet_softc *); 176static int vtnet_init_tx_queues(struct vtnet_softc *); 177static int vtnet_init_rxtx_queues(struct vtnet_softc *); 178static void vtnet_set_active_vq_pairs(struct vtnet_softc *); 179static int vtnet_reinit(struct vtnet_softc *); 180static void vtnet_init_locked(struct vtnet_softc *); 181static void vtnet_init(void *); 182 183static void vtnet_free_ctrl_vq(struct vtnet_softc *); 184static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *, 185 struct sglist *, int, int); 186static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *); 187static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t); 188static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int); 189static int vtnet_set_promisc(struct vtnet_softc *, int); 190static int vtnet_set_allmulti(struct vtnet_softc *, int); 191static void vtnet_attach_disable_promisc(struct vtnet_softc *); 192static void vtnet_rx_filter(struct vtnet_softc *); 193static void vtnet_rx_filter_mac(struct vtnet_softc *); 194static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); 195static void vtnet_rx_filter_vlan(struct vtnet_softc *); 196static void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t); 197static void vtnet_register_vlan(void *, struct ifnet *, uint16_t); 198static void vtnet_unregister_vlan(void *, struct ifnet *, uint16_t); 199 200static int vtnet_is_link_up(struct vtnet_softc *); 201static void vtnet_update_link_status(struct vtnet_softc *); 202static int vtnet_ifmedia_upd(struct ifnet *); 203static void vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *); 204static void vtnet_get_hwaddr(struct vtnet_softc *); 205static void vtnet_set_hwaddr(struct vtnet_softc *); 206static void vtnet_vlan_tag_remove(struct mbuf *); 207 208static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, 209 struct sysctl_oid_list *, struct vtnet_rxq *); 210static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, 211 struct sysctl_oid_list *, struct vtnet_txq *); 212static void vtnet_setup_queue_sysctl(struct vtnet_softc *); 213static void vtnet_setup_sysctl(struct vtnet_softc *); 214 215static int vtnet_rxq_enable_intr(struct vtnet_rxq *); 216static void vtnet_rxq_disable_intr(struct vtnet_rxq *); 217static int vtnet_txq_enable_intr(struct vtnet_txq *); 218static void vtnet_txq_disable_intr(struct vtnet_txq *); 219static void vtnet_enable_rx_interrupts(struct vtnet_softc *); 220static void vtnet_enable_tx_interrupts(struct vtnet_softc *); 221static void vtnet_enable_interrupts(struct vtnet_softc *); 222static void vtnet_disable_rx_interrupts(struct vtnet_softc *); 223static void vtnet_disable_tx_interrupts(struct vtnet_softc *); 224static void vtnet_disable_interrupts(struct vtnet_softc *); 225 226static int vtnet_tunable_int(struct vtnet_softc *, const char *, int); 227 228/* Tunables. */ 229static int vtnet_csum_disable = 0; 230TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable); 231static int vtnet_tso_disable = 0; 232TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable); 233static int vtnet_lro_disable = 0; 234TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable); 235static int vtnet_mq_disable = 0; 236TUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable); 237static int vtnet_mq_max_pairs = 0; 238TUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs); 239static int vtnet_rx_process_limit = 512; 240TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); 241 242/* 243 * Reducing the number of transmit completed interrupts can improve 244 * performance. To do so, the define below keeps the Tx vq interrupt 245 * disabled and adds calls to vtnet_txeof() in the start and watchdog 246 * paths. The price to pay for this is the m_free'ing of transmitted 247 * mbufs may be delayed until the watchdog fires. 248 * 249 * BMV: Reintroduce this later as a run-time option, if it makes 250 * sense after the EVENT_IDX feature is supported. 251 * 252 * #define VTNET_TX_INTR_MODERATION 253 */ 254 255static uma_zone_t vtnet_tx_header_zone; 256 257static struct virtio_feature_desc vtnet_feature_desc[] = { 258 { VIRTIO_NET_F_CSUM, "TxChecksum" }, 259 { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, 260 { VIRTIO_NET_F_MAC, "MacAddress" }, 261 { VIRTIO_NET_F_GSO, "TxAllGSO" }, 262 { VIRTIO_NET_F_GUEST_TSO4, "RxTSOv4" }, 263 { VIRTIO_NET_F_GUEST_TSO6, "RxTSOv6" }, 264 { VIRTIO_NET_F_GUEST_ECN, "RxECN" }, 265 { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, 266 { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, 267 { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, 268 { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, 269 { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, 270 { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, 271 { VIRTIO_NET_F_STATUS, "Status" }, 272 { VIRTIO_NET_F_CTRL_VQ, "ControlVq" }, 273 { VIRTIO_NET_F_CTRL_RX, "RxMode" }, 274 { VIRTIO_NET_F_CTRL_VLAN, "VLanFilter" }, 275 { VIRTIO_NET_F_CTRL_RX_EXTRA, "RxModeExtra" }, 276 { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, 277 { VIRTIO_NET_F_MQ, "Multiqueue" }, 278 { VIRTIO_NET_F_CTRL_MAC_ADDR, "SetMacAddress" }, 279 280 { 0, NULL } 281}; 282 283static device_method_t vtnet_methods[] = { 284 /* Device methods. */ 285 DEVMETHOD(device_probe, vtnet_probe), 286 DEVMETHOD(device_attach, vtnet_attach), 287 DEVMETHOD(device_detach, vtnet_detach), 288 DEVMETHOD(device_suspend, vtnet_suspend), 289 DEVMETHOD(device_resume, vtnet_resume), 290 DEVMETHOD(device_shutdown, vtnet_shutdown), 291 292 /* VirtIO methods. */ 293 DEVMETHOD(virtio_attach_completed, vtnet_attach_completed), 294 DEVMETHOD(virtio_config_change, vtnet_config_change), 295 296 DEVMETHOD_END 297}; 298 299static driver_t vtnet_driver = { 300 "vtnet", 301 vtnet_methods, 302 sizeof(struct vtnet_softc) 303}; 304static devclass_t vtnet_devclass; 305 306DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass, 307 vtnet_modevent, 0); 308MODULE_VERSION(vtnet, 1); 309MODULE_DEPEND(vtnet, virtio, 1, 1, 1); 310 311static int 312vtnet_modevent(module_t mod, int type, void *unused) 313{ 314 int error; 315 316 error = 0; 317 318 switch (type) { 319 case MOD_LOAD: 320 vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr", 321 sizeof(struct vtnet_tx_header), 322 NULL, NULL, NULL, NULL, 0, 0); 323 break; 324 case MOD_QUIESCE: 325 case MOD_UNLOAD: 326 if (uma_zone_get_cur(vtnet_tx_header_zone) > 0) 327 error = EBUSY; 328 else if (type == MOD_UNLOAD) { 329 uma_zdestroy(vtnet_tx_header_zone); 330 vtnet_tx_header_zone = NULL; 331 } 332 break; 333 case MOD_SHUTDOWN: 334 break; 335 default: 336 error = EOPNOTSUPP; 337 break; 338 } 339 340 return (error); 341} 342 343static int 344vtnet_probe(device_t dev) 345{ 346 347 if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK) 348 return (ENXIO); 349 350 device_set_desc(dev, "VirtIO Networking Adapter"); 351 352 return (BUS_PROBE_DEFAULT); 353} 354 355static int 356vtnet_attach(device_t dev) 357{ 358 struct vtnet_softc *sc; 359 int error; 360 361 sc = device_get_softc(dev); 362 sc->vtnet_dev = dev; 363 364 /* Register our feature descriptions. */ 365 virtio_set_feature_desc(dev, vtnet_feature_desc); 366 367 VTNET_CORE_LOCK_INIT(sc); 368 callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); 369 370 vtnet_setup_sysctl(sc); 371 vtnet_setup_features(sc); 372 373 error = vtnet_alloc_rx_filters(sc); 374 if (error) { 375 device_printf(dev, "cannot allocate Rx filters\n"); 376 goto fail; 377 } 378 379 error = vtnet_alloc_rxtx_queues(sc); 380 if (error) { 381 device_printf(dev, "cannot allocate queues\n"); 382 goto fail; 383 } 384 385 error = vtnet_alloc_virtqueues(sc); 386 if (error) { 387 device_printf(dev, "cannot allocate virtqueues\n"); 388 goto fail; 389 } 390 391 error = vtnet_setup_interface(sc); 392 if (error) { 393 device_printf(dev, "cannot setup interface\n"); 394 goto fail; 395 } 396 397 error = virtio_setup_intr(dev, INTR_TYPE_NET); 398 if (error) { 399 device_printf(dev, "cannot setup virtqueue interrupts\n"); 400 /* BMV: This will crash if during boot! */ 401 ether_ifdetach(sc->vtnet_ifp); 402 goto fail; 403 } 404 405 vtnet_start_taskqueues(sc); 406 407fail: 408 if (error) 409 vtnet_detach(dev); 410 411 return (error); 412} 413 414static int 415vtnet_detach(device_t dev) 416{ 417 struct vtnet_softc *sc; 418 struct ifnet *ifp; 419 420 sc = device_get_softc(dev); 421 ifp = sc->vtnet_ifp; 422 423 if (device_is_attached(dev)) { 424 VTNET_CORE_LOCK(sc); 425 vtnet_stop(sc); 426 VTNET_CORE_UNLOCK(sc); 427 428 callout_drain(&sc->vtnet_tick_ch); 429 vtnet_drain_taskqueues(sc); 430 431 ether_ifdetach(ifp); 432 } 433 434 vtnet_free_taskqueues(sc); 435 436 if (sc->vtnet_vlan_attach != NULL) { 437 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); 438 sc->vtnet_vlan_attach = NULL; 439 } 440 if (sc->vtnet_vlan_detach != NULL) { 441 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach); 442 sc->vtnet_vlan_detach = NULL; 443 } 444 445 ifmedia_removeall(&sc->vtnet_media); 446 447 if (ifp != NULL) { 448 if_free(ifp); 449 sc->vtnet_ifp = NULL; 450 } 451 452 vtnet_free_rxtx_queues(sc); 453 vtnet_free_rx_filters(sc); 454 455 if (sc->vtnet_ctrl_vq != NULL) 456 vtnet_free_ctrl_vq(sc); 457 458 VTNET_CORE_LOCK_DESTROY(sc); 459 460 return (0); 461} 462 463static int 464vtnet_suspend(device_t dev) 465{ 466 struct vtnet_softc *sc; 467 468 sc = device_get_softc(dev); 469 470 VTNET_CORE_LOCK(sc); 471 vtnet_stop(sc); 472 sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; 473 VTNET_CORE_UNLOCK(sc); 474 475 return (0); 476} 477 478static int 479vtnet_resume(device_t dev) 480{ 481 struct vtnet_softc *sc; 482 struct ifnet *ifp; 483 484 sc = device_get_softc(dev); 485 ifp = sc->vtnet_ifp; 486 487 VTNET_CORE_LOCK(sc); 488 if (ifp->if_flags & IFF_UP) 489 vtnet_init_locked(sc); 490 sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; 491 VTNET_CORE_UNLOCK(sc); 492 493 return (0); 494} 495 496static int 497vtnet_shutdown(device_t dev) 498{ 499 500 /* 501 * Suspend already does all of what we need to 502 * do here; we just never expect to be resumed. 503 */ 504 return (vtnet_suspend(dev)); 505} 506 507static int 508vtnet_attach_completed(device_t dev) 509{ 510 511 vtnet_attach_disable_promisc(device_get_softc(dev)); 512 513 return (0); 514} 515 516static int 517vtnet_config_change(device_t dev) 518{ 519 struct vtnet_softc *sc; 520 521 sc = device_get_softc(dev); 522 523 VTNET_CORE_LOCK(sc); 524 vtnet_update_link_status(sc); 525 if (sc->vtnet_link_active != 0) 526 vtnet_tx_start_all(sc); 527 VTNET_CORE_UNLOCK(sc); 528 529 return (0); 530} 531 532static void 533vtnet_negotiate_features(struct vtnet_softc *sc) 534{ 535 device_t dev; 536 uint64_t mask, features; 537 538 dev = sc->vtnet_dev; 539 mask = 0; 540 541 /* 542 * TSO and LRO are only available when their corresponding checksum 543 * offload feature is also negotiated. 544 */ 545 if (vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable)) { 546 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM; 547 mask |= VTNET_TSO_FEATURES | VTNET_LRO_FEATURES; 548 } 549 if (vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable)) 550 mask |= VTNET_TSO_FEATURES; 551 if (vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable)) 552 mask |= VTNET_LRO_FEATURES;
| 31 32#include <sys/param.h> 33#include <sys/systm.h> 34#include <sys/kernel.h> 35#include <sys/sockio.h> 36#include <sys/mbuf.h> 37#include <sys/malloc.h> 38#include <sys/module.h> 39#include <sys/socket.h> 40#include <sys/sysctl.h> 41#include <sys/random.h> 42#include <sys/sglist.h> 43#include <sys/lock.h> 44#include <sys/mutex.h> 45#include <sys/taskqueue.h> 46#include <sys/smp.h> 47#include <machine/smp.h> 48 49#include <vm/uma.h> 50 51#include <net/ethernet.h> 52#include <net/if.h> 53#include <net/if_arp.h> 54#include <net/if_dl.h> 55#include <net/if_types.h> 56#include <net/if_media.h> 57#include <net/if_vlan_var.h> 58 59#include <net/bpf.h> 60 61#include <netinet/in_systm.h> 62#include <netinet/in.h> 63#include <netinet/ip.h> 64#include <netinet/ip6.h> 65#include <netinet6/ip6_var.h> 66#include <netinet/udp.h> 67#include <netinet/tcp.h> 68#include <netinet/sctp.h> 69 70#include <machine/bus.h> 71#include <machine/resource.h> 72#include <sys/bus.h> 73#include <sys/rman.h> 74 75#include <dev/virtio/virtio.h> 76#include <dev/virtio/virtqueue.h> 77#include <dev/virtio/network/virtio_net.h> 78#include <dev/virtio/network/if_vtnetvar.h> 79 80#include "virtio_if.h" 81 82#include "opt_inet.h" 83#include "opt_inet6.h" 84 85static int vtnet_modevent(module_t, int, void *); 86 87static int vtnet_probe(device_t); 88static int vtnet_attach(device_t); 89static int vtnet_detach(device_t); 90static int vtnet_suspend(device_t); 91static int vtnet_resume(device_t); 92static int vtnet_shutdown(device_t); 93static int vtnet_attach_completed(device_t); 94static int vtnet_config_change(device_t); 95 96static void vtnet_negotiate_features(struct vtnet_softc *); 97static void vtnet_setup_features(struct vtnet_softc *); 98static int vtnet_init_rxq(struct vtnet_softc *, int); 99static int vtnet_init_txq(struct vtnet_softc *, int); 100static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); 101static void vtnet_free_rxtx_queues(struct vtnet_softc *); 102static int vtnet_alloc_rx_filters(struct vtnet_softc *); 103static void vtnet_free_rx_filters(struct vtnet_softc *); 104static int vtnet_alloc_virtqueues(struct vtnet_softc *); 105static int vtnet_setup_interface(struct vtnet_softc *); 106static int vtnet_change_mtu(struct vtnet_softc *, int); 107static int vtnet_ioctl(struct ifnet *, u_long, caddr_t); 108 109static int vtnet_rxq_populate(struct vtnet_rxq *); 110static void vtnet_rxq_free_mbufs(struct vtnet_rxq *); 111static struct mbuf * 112 vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **); 113static int vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *, 114 struct mbuf *, int); 115static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); 116static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); 117static int vtnet_rxq_new_buf(struct vtnet_rxq *); 118static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, 119 struct virtio_net_hdr *); 120static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); 121static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); 122static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); 123static void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *, 124 struct virtio_net_hdr *); 125static int vtnet_rxq_eof(struct vtnet_rxq *); 126static void vtnet_rx_vq_intr(void *); 127static void vtnet_rxq_tq_intr(void *, int); 128 129static void vtnet_txq_free_mbufs(struct vtnet_txq *); 130static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, 131 int *, int *, int *); 132static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, 133 int, struct virtio_net_hdr *); 134static struct mbuf * 135 vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, 136 struct virtio_net_hdr *); 137static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, 138 struct vtnet_tx_header *); 139static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **); 140#ifdef VTNET_LEGACY_TX 141static void vtnet_start_locked(struct vtnet_txq *, struct ifnet *); 142static void vtnet_start(struct ifnet *); 143#else 144static int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *); 145static int vtnet_txq_mq_start(struct ifnet *, struct mbuf *); 146static void vtnet_txq_tq_deferred(void *, int); 147#endif 148static void vtnet_txq_start(struct vtnet_txq *); 149static void vtnet_txq_tq_intr(void *, int); 150static void vtnet_txq_eof(struct vtnet_txq *); 151static void vtnet_tx_vq_intr(void *); 152static void vtnet_tx_start_all(struct vtnet_softc *); 153 154#ifndef VTNET_LEGACY_TX 155static void vtnet_qflush(struct ifnet *); 156#endif 157 158static int vtnet_watchdog(struct vtnet_txq *); 159static void vtnet_rxq_accum_stats(struct vtnet_rxq *, 160 struct vtnet_rxq_stats *); 161static void vtnet_txq_accum_stats(struct vtnet_txq *, 162 struct vtnet_txq_stats *); 163static void vtnet_accumulate_stats(struct vtnet_softc *); 164static void vtnet_tick(void *); 165 166static void vtnet_start_taskqueues(struct vtnet_softc *); 167static void vtnet_free_taskqueues(struct vtnet_softc *); 168static void vtnet_drain_taskqueues(struct vtnet_softc *); 169 170static void vtnet_drain_rxtx_queues(struct vtnet_softc *); 171static void vtnet_stop_rendezvous(struct vtnet_softc *); 172static void vtnet_stop(struct vtnet_softc *); 173static int vtnet_virtio_reinit(struct vtnet_softc *); 174static void vtnet_init_rx_filters(struct vtnet_softc *); 175static int vtnet_init_rx_queues(struct vtnet_softc *); 176static int vtnet_init_tx_queues(struct vtnet_softc *); 177static int vtnet_init_rxtx_queues(struct vtnet_softc *); 178static void vtnet_set_active_vq_pairs(struct vtnet_softc *); 179static int vtnet_reinit(struct vtnet_softc *); 180static void vtnet_init_locked(struct vtnet_softc *); 181static void vtnet_init(void *); 182 183static void vtnet_free_ctrl_vq(struct vtnet_softc *); 184static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *, 185 struct sglist *, int, int); 186static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *); 187static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t); 188static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int); 189static int vtnet_set_promisc(struct vtnet_softc *, int); 190static int vtnet_set_allmulti(struct vtnet_softc *, int); 191static void vtnet_attach_disable_promisc(struct vtnet_softc *); 192static void vtnet_rx_filter(struct vtnet_softc *); 193static void vtnet_rx_filter_mac(struct vtnet_softc *); 194static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); 195static void vtnet_rx_filter_vlan(struct vtnet_softc *); 196static void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t); 197static void vtnet_register_vlan(void *, struct ifnet *, uint16_t); 198static void vtnet_unregister_vlan(void *, struct ifnet *, uint16_t); 199 200static int vtnet_is_link_up(struct vtnet_softc *); 201static void vtnet_update_link_status(struct vtnet_softc *); 202static int vtnet_ifmedia_upd(struct ifnet *); 203static void vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *); 204static void vtnet_get_hwaddr(struct vtnet_softc *); 205static void vtnet_set_hwaddr(struct vtnet_softc *); 206static void vtnet_vlan_tag_remove(struct mbuf *); 207 208static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, 209 struct sysctl_oid_list *, struct vtnet_rxq *); 210static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, 211 struct sysctl_oid_list *, struct vtnet_txq *); 212static void vtnet_setup_queue_sysctl(struct vtnet_softc *); 213static void vtnet_setup_sysctl(struct vtnet_softc *); 214 215static int vtnet_rxq_enable_intr(struct vtnet_rxq *); 216static void vtnet_rxq_disable_intr(struct vtnet_rxq *); 217static int vtnet_txq_enable_intr(struct vtnet_txq *); 218static void vtnet_txq_disable_intr(struct vtnet_txq *); 219static void vtnet_enable_rx_interrupts(struct vtnet_softc *); 220static void vtnet_enable_tx_interrupts(struct vtnet_softc *); 221static void vtnet_enable_interrupts(struct vtnet_softc *); 222static void vtnet_disable_rx_interrupts(struct vtnet_softc *); 223static void vtnet_disable_tx_interrupts(struct vtnet_softc *); 224static void vtnet_disable_interrupts(struct vtnet_softc *); 225 226static int vtnet_tunable_int(struct vtnet_softc *, const char *, int); 227 228/* Tunables. */ 229static int vtnet_csum_disable = 0; 230TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable); 231static int vtnet_tso_disable = 0; 232TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable); 233static int vtnet_lro_disable = 0; 234TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable); 235static int vtnet_mq_disable = 0; 236TUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable); 237static int vtnet_mq_max_pairs = 0; 238TUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs); 239static int vtnet_rx_process_limit = 512; 240TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); 241 242/* 243 * Reducing the number of transmit completed interrupts can improve 244 * performance. To do so, the define below keeps the Tx vq interrupt 245 * disabled and adds calls to vtnet_txeof() in the start and watchdog 246 * paths. The price to pay for this is the m_free'ing of transmitted 247 * mbufs may be delayed until the watchdog fires. 248 * 249 * BMV: Reintroduce this later as a run-time option, if it makes 250 * sense after the EVENT_IDX feature is supported. 251 * 252 * #define VTNET_TX_INTR_MODERATION 253 */ 254 255static uma_zone_t vtnet_tx_header_zone; 256 257static struct virtio_feature_desc vtnet_feature_desc[] = { 258 { VIRTIO_NET_F_CSUM, "TxChecksum" }, 259 { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, 260 { VIRTIO_NET_F_MAC, "MacAddress" }, 261 { VIRTIO_NET_F_GSO, "TxAllGSO" }, 262 { VIRTIO_NET_F_GUEST_TSO4, "RxTSOv4" }, 263 { VIRTIO_NET_F_GUEST_TSO6, "RxTSOv6" }, 264 { VIRTIO_NET_F_GUEST_ECN, "RxECN" }, 265 { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, 266 { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, 267 { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, 268 { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, 269 { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, 270 { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, 271 { VIRTIO_NET_F_STATUS, "Status" }, 272 { VIRTIO_NET_F_CTRL_VQ, "ControlVq" }, 273 { VIRTIO_NET_F_CTRL_RX, "RxMode" }, 274 { VIRTIO_NET_F_CTRL_VLAN, "VLanFilter" }, 275 { VIRTIO_NET_F_CTRL_RX_EXTRA, "RxModeExtra" }, 276 { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, 277 { VIRTIO_NET_F_MQ, "Multiqueue" }, 278 { VIRTIO_NET_F_CTRL_MAC_ADDR, "SetMacAddress" }, 279 280 { 0, NULL } 281}; 282 283static device_method_t vtnet_methods[] = { 284 /* Device methods. */ 285 DEVMETHOD(device_probe, vtnet_probe), 286 DEVMETHOD(device_attach, vtnet_attach), 287 DEVMETHOD(device_detach, vtnet_detach), 288 DEVMETHOD(device_suspend, vtnet_suspend), 289 DEVMETHOD(device_resume, vtnet_resume), 290 DEVMETHOD(device_shutdown, vtnet_shutdown), 291 292 /* VirtIO methods. */ 293 DEVMETHOD(virtio_attach_completed, vtnet_attach_completed), 294 DEVMETHOD(virtio_config_change, vtnet_config_change), 295 296 DEVMETHOD_END 297}; 298 299static driver_t vtnet_driver = { 300 "vtnet", 301 vtnet_methods, 302 sizeof(struct vtnet_softc) 303}; 304static devclass_t vtnet_devclass; 305 306DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass, 307 vtnet_modevent, 0); 308MODULE_VERSION(vtnet, 1); 309MODULE_DEPEND(vtnet, virtio, 1, 1, 1); 310 311static int 312vtnet_modevent(module_t mod, int type, void *unused) 313{ 314 int error; 315 316 error = 0; 317 318 switch (type) { 319 case MOD_LOAD: 320 vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr", 321 sizeof(struct vtnet_tx_header), 322 NULL, NULL, NULL, NULL, 0, 0); 323 break; 324 case MOD_QUIESCE: 325 case MOD_UNLOAD: 326 if (uma_zone_get_cur(vtnet_tx_header_zone) > 0) 327 error = EBUSY; 328 else if (type == MOD_UNLOAD) { 329 uma_zdestroy(vtnet_tx_header_zone); 330 vtnet_tx_header_zone = NULL; 331 } 332 break; 333 case MOD_SHUTDOWN: 334 break; 335 default: 336 error = EOPNOTSUPP; 337 break; 338 } 339 340 return (error); 341} 342 343static int 344vtnet_probe(device_t dev) 345{ 346 347 if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK) 348 return (ENXIO); 349 350 device_set_desc(dev, "VirtIO Networking Adapter"); 351 352 return (BUS_PROBE_DEFAULT); 353} 354 355static int 356vtnet_attach(device_t dev) 357{ 358 struct vtnet_softc *sc; 359 int error; 360 361 sc = device_get_softc(dev); 362 sc->vtnet_dev = dev; 363 364 /* Register our feature descriptions. */ 365 virtio_set_feature_desc(dev, vtnet_feature_desc); 366 367 VTNET_CORE_LOCK_INIT(sc); 368 callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); 369 370 vtnet_setup_sysctl(sc); 371 vtnet_setup_features(sc); 372 373 error = vtnet_alloc_rx_filters(sc); 374 if (error) { 375 device_printf(dev, "cannot allocate Rx filters\n"); 376 goto fail; 377 } 378 379 error = vtnet_alloc_rxtx_queues(sc); 380 if (error) { 381 device_printf(dev, "cannot allocate queues\n"); 382 goto fail; 383 } 384 385 error = vtnet_alloc_virtqueues(sc); 386 if (error) { 387 device_printf(dev, "cannot allocate virtqueues\n"); 388 goto fail; 389 } 390 391 error = vtnet_setup_interface(sc); 392 if (error) { 393 device_printf(dev, "cannot setup interface\n"); 394 goto fail; 395 } 396 397 error = virtio_setup_intr(dev, INTR_TYPE_NET); 398 if (error) { 399 device_printf(dev, "cannot setup virtqueue interrupts\n"); 400 /* BMV: This will crash if during boot! */ 401 ether_ifdetach(sc->vtnet_ifp); 402 goto fail; 403 } 404 405 vtnet_start_taskqueues(sc); 406 407fail: 408 if (error) 409 vtnet_detach(dev); 410 411 return (error); 412} 413 414static int 415vtnet_detach(device_t dev) 416{ 417 struct vtnet_softc *sc; 418 struct ifnet *ifp; 419 420 sc = device_get_softc(dev); 421 ifp = sc->vtnet_ifp; 422 423 if (device_is_attached(dev)) { 424 VTNET_CORE_LOCK(sc); 425 vtnet_stop(sc); 426 VTNET_CORE_UNLOCK(sc); 427 428 callout_drain(&sc->vtnet_tick_ch); 429 vtnet_drain_taskqueues(sc); 430 431 ether_ifdetach(ifp); 432 } 433 434 vtnet_free_taskqueues(sc); 435 436 if (sc->vtnet_vlan_attach != NULL) { 437 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); 438 sc->vtnet_vlan_attach = NULL; 439 } 440 if (sc->vtnet_vlan_detach != NULL) { 441 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach); 442 sc->vtnet_vlan_detach = NULL; 443 } 444 445 ifmedia_removeall(&sc->vtnet_media); 446 447 if (ifp != NULL) { 448 if_free(ifp); 449 sc->vtnet_ifp = NULL; 450 } 451 452 vtnet_free_rxtx_queues(sc); 453 vtnet_free_rx_filters(sc); 454 455 if (sc->vtnet_ctrl_vq != NULL) 456 vtnet_free_ctrl_vq(sc); 457 458 VTNET_CORE_LOCK_DESTROY(sc); 459 460 return (0); 461} 462 463static int 464vtnet_suspend(device_t dev) 465{ 466 struct vtnet_softc *sc; 467 468 sc = device_get_softc(dev); 469 470 VTNET_CORE_LOCK(sc); 471 vtnet_stop(sc); 472 sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; 473 VTNET_CORE_UNLOCK(sc); 474 475 return (0); 476} 477 478static int 479vtnet_resume(device_t dev) 480{ 481 struct vtnet_softc *sc; 482 struct ifnet *ifp; 483 484 sc = device_get_softc(dev); 485 ifp = sc->vtnet_ifp; 486 487 VTNET_CORE_LOCK(sc); 488 if (ifp->if_flags & IFF_UP) 489 vtnet_init_locked(sc); 490 sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; 491 VTNET_CORE_UNLOCK(sc); 492 493 return (0); 494} 495 496static int 497vtnet_shutdown(device_t dev) 498{ 499 500 /* 501 * Suspend already does all of what we need to 502 * do here; we just never expect to be resumed. 503 */ 504 return (vtnet_suspend(dev)); 505} 506 507static int 508vtnet_attach_completed(device_t dev) 509{ 510 511 vtnet_attach_disable_promisc(device_get_softc(dev)); 512 513 return (0); 514} 515 516static int 517vtnet_config_change(device_t dev) 518{ 519 struct vtnet_softc *sc; 520 521 sc = device_get_softc(dev); 522 523 VTNET_CORE_LOCK(sc); 524 vtnet_update_link_status(sc); 525 if (sc->vtnet_link_active != 0) 526 vtnet_tx_start_all(sc); 527 VTNET_CORE_UNLOCK(sc); 528 529 return (0); 530} 531 532static void 533vtnet_negotiate_features(struct vtnet_softc *sc) 534{ 535 device_t dev; 536 uint64_t mask, features; 537 538 dev = sc->vtnet_dev; 539 mask = 0; 540 541 /* 542 * TSO and LRO are only available when their corresponding checksum 543 * offload feature is also negotiated. 544 */ 545 if (vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable)) { 546 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM; 547 mask |= VTNET_TSO_FEATURES | VTNET_LRO_FEATURES; 548 } 549 if (vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable)) 550 mask |= VTNET_TSO_FEATURES; 551 if (vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable)) 552 mask |= VTNET_LRO_FEATURES;
|
| 553#ifndef VTNET_LEGACY_TX
|
553 if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable)) 554 mask |= VIRTIO_NET_F_MQ;
| 554 if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable)) 555 mask |= VIRTIO_NET_F_MQ;
|
555#ifdef VTNET_LEGACY_TX
| 556#else
|
556 mask |= VIRTIO_NET_F_MQ; 557#endif 558 559 features = VTNET_FEATURES & ~mask; 560 sc->vtnet_features = virtio_negotiate_features(dev, features); 561
| 557 mask |= VIRTIO_NET_F_MQ; 558#endif 559 560 features = VTNET_FEATURES & ~mask; 561 sc->vtnet_features = virtio_negotiate_features(dev, features); 562
|
562 if (virtio_with_feature(dev, VTNET_LRO_FEATURES) == 0) 563 return; 564 if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) 565 return;
| 563 if (virtio_with_feature(dev, VTNET_LRO_FEATURES) && 564 virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) { 565 /* 566 * LRO without mergeable buffers requires special care. This 567 * is not ideal because every receive buffer must be large 568 * enough to hold the maximum TCP packet, the Ethernet header, 569 * and the header. This requires up to 34 descriptors with 570 * MCLBYTES clusters. If we do not have indirect descriptors, 571 * LRO is disabled since the virtqueue will not contain very 572 * many receive buffers. 573 */ 574 if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { 575 device_printf(dev, 576 "LRO disabled due to both mergeable buffers and " 577 "indirect descriptors not negotiated\n");
|
566
| 578
|
567 /* 568 * LRO without mergeable buffers requires special care. This is not 569 * ideal because every receive buffer must be large enough to hold 570 * the maximum TCP packet, the Ethernet header, and the header. This 571 * requires up to 34 descriptors with MCLBYTES clusters. If we do 572 * not have indirect descriptors, LRO is disabled since the virtqueue 573 * will not contain very many receive buffers. 574 */ 575 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 576 device_printf(dev, 577 "LRO disabled due to both mergeable buffers and indirect " 578 "descriptors not negotiated\n"); 579 580 features &= ~VTNET_LRO_FEATURES; 581 sc->vtnet_features = virtio_negotiate_features(dev, features); 582 } else 583 sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
| 579 features &= ~VTNET_LRO_FEATURES; 580 sc->vtnet_features = 581 virtio_negotiate_features(dev, features); 582 } else 583 sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; 584 }
|
584} 585 586static void 587vtnet_setup_features(struct vtnet_softc *sc) 588{ 589 device_t dev; 590 int max_pairs, max; 591 592 dev = sc->vtnet_dev; 593 594 vtnet_negotiate_features(sc); 595 596 if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX)) 597 sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; 598 599 if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { 600 /* This feature should always be negotiated. */ 601 sc->vtnet_flags |= VTNET_FLAG_MAC; 602 } 603 604 if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { 605 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; 606 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 607 } else 608 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 609 610 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) 611 sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS; 612 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 613 sc->vtnet_rx_nsegs = VTNET_MAX_RX_SEGS; 614 else 615 sc->vtnet_rx_nsegs = VTNET_MIN_RX_SEGS; 616 617 if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || 618 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || 619 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 620 sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS; 621 else 622 sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS; 623 624 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { 625 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; 626 627 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) 628 sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; 629 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) 630 sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; 631 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) 632 sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; 633 } 634 635 if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) && 636 sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 637 max_pairs = virtio_read_dev_config_2(dev, 638 offsetof(struct virtio_net_config, max_virtqueue_pairs)); 639 if (max_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 640 max_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 641 max_pairs = 1; 642 } else 643 max_pairs = 1; 644 645 if (max_pairs > 1) { 646 /* 647 * Limit the maximum number of queue pairs to the number of 648 * CPUs or the configured maximum. The actual number of 649 * queues that get used may be less. 650 */ 651 max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); 652 if (max > 0 && max_pairs > max) 653 max_pairs = max; 654 if (max_pairs > mp_ncpus) 655 max_pairs = mp_ncpus; 656 if (max_pairs > VTNET_MAX_QUEUE_PAIRS) 657 max_pairs = VTNET_MAX_QUEUE_PAIRS; 658 if (max_pairs > 1) 659 sc->vtnet_flags |= VTNET_FLAG_MULTIQ; 660 } 661 662 sc->vtnet_max_vq_pairs = max_pairs; 663} 664 665static int 666vtnet_init_rxq(struct vtnet_softc *sc, int id) 667{ 668 struct vtnet_rxq *rxq; 669 670 rxq = &sc->vtnet_rxqs[id]; 671 672 snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", 673 device_get_nameunit(sc->vtnet_dev), id); 674 mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); 675 676 rxq->vtnrx_sc = sc; 677 rxq->vtnrx_id = id; 678 679 rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); 680 if (rxq->vtnrx_sg == NULL) 681 return (ENOMEM); 682 683 TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); 684 rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, 685 taskqueue_thread_enqueue, &rxq->vtnrx_tq); 686 687 return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); 688} 689 690static int 691vtnet_init_txq(struct vtnet_softc *sc, int id) 692{ 693 struct vtnet_txq *txq; 694 695 txq = &sc->vtnet_txqs[id]; 696 697 snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", 698 device_get_nameunit(sc->vtnet_dev), id); 699 mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); 700 701 txq->vtntx_sc = sc; 702 txq->vtntx_id = id; 703 704 txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); 705 if (txq->vtntx_sg == NULL) 706 return (ENOMEM); 707 708#ifndef VTNET_LEGACY_TX 709 txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, 710 M_NOWAIT, &txq->vtntx_mtx); 711 if (txq->vtntx_br == NULL) 712 return (ENOMEM); 713 714 TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); 715#endif 716 TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); 717 txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, 718 taskqueue_thread_enqueue, &txq->vtntx_tq); 719 if (txq->vtntx_tq == NULL) 720 return (ENOMEM); 721 722 return (0); 723} 724 725static int 726vtnet_alloc_rxtx_queues(struct vtnet_softc *sc) 727{ 728 int i, npairs, error; 729 730 npairs = sc->vtnet_max_vq_pairs; 731 732 sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, 733 M_NOWAIT | M_ZERO); 734 sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, 735 M_NOWAIT | M_ZERO); 736 if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) 737 return (ENOMEM); 738 739 for (i = 0; i < npairs; i++) { 740 error = vtnet_init_rxq(sc, i); 741 if (error) 742 return (error); 743 error = vtnet_init_txq(sc, i); 744 if (error) 745 return (error); 746 } 747 748 vtnet_setup_queue_sysctl(sc); 749 750 return (0); 751} 752 753static void 754vtnet_destroy_rxq(struct vtnet_rxq *rxq) 755{ 756 757 rxq->vtnrx_sc = NULL; 758 rxq->vtnrx_id = -1; 759 760 if (rxq->vtnrx_sg != NULL) { 761 sglist_free(rxq->vtnrx_sg); 762 rxq->vtnrx_sg = NULL; 763 } 764 765 if (mtx_initialized(&rxq->vtnrx_mtx) != 0) 766 mtx_destroy(&rxq->vtnrx_mtx); 767} 768 769static void 770vtnet_destroy_txq(struct vtnet_txq *txq) 771{ 772 773 txq->vtntx_sc = NULL; 774 txq->vtntx_id = -1; 775 776 if (txq->vtntx_sg != NULL) { 777 sglist_free(txq->vtntx_sg); 778 txq->vtntx_sg = NULL; 779 } 780 781#ifndef VTNET_LEGACY_TX 782 if (txq->vtntx_br != NULL) { 783 buf_ring_free(txq->vtntx_br, M_DEVBUF); 784 txq->vtntx_br = NULL; 785 } 786#endif 787 788 if (mtx_initialized(&txq->vtntx_mtx) != 0) 789 mtx_destroy(&txq->vtntx_mtx); 790} 791 792static void 793vtnet_free_rxtx_queues(struct vtnet_softc *sc) 794{ 795 int i; 796 797 if (sc->vtnet_rxqs != NULL) { 798 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 799 vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); 800 free(sc->vtnet_rxqs, M_DEVBUF); 801 sc->vtnet_rxqs = NULL; 802 } 803 804 if (sc->vtnet_txqs != NULL) { 805 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 806 vtnet_destroy_txq(&sc->vtnet_txqs[i]); 807 free(sc->vtnet_txqs, M_DEVBUF); 808 sc->vtnet_txqs = NULL; 809 } 810} 811 812static int 813vtnet_alloc_rx_filters(struct vtnet_softc *sc) 814{ 815 816 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 817 sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), 818 M_DEVBUF, M_NOWAIT | M_ZERO); 819 if (sc->vtnet_mac_filter == NULL) 820 return (ENOMEM); 821 } 822 823 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 824 sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * 825 VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO); 826 if (sc->vtnet_vlan_filter == NULL) 827 return (ENOMEM); 828 } 829 830 return (0); 831} 832 833static void 834vtnet_free_rx_filters(struct vtnet_softc *sc) 835{ 836 837 if (sc->vtnet_mac_filter != NULL) { 838 free(sc->vtnet_mac_filter, M_DEVBUF); 839 sc->vtnet_mac_filter = NULL; 840 } 841 842 if (sc->vtnet_vlan_filter != NULL) { 843 free(sc->vtnet_vlan_filter, M_DEVBUF); 844 sc->vtnet_vlan_filter = NULL; 845 } 846} 847 848static int 849vtnet_alloc_virtqueues(struct vtnet_softc *sc) 850{ 851 device_t dev; 852 struct vq_alloc_info *info; 853 struct vtnet_rxq *rxq; 854 struct vtnet_txq *txq; 855 int i, idx, flags, nvqs, error; 856 857 dev = sc->vtnet_dev; 858 flags = 0; 859 860 nvqs = sc->vtnet_max_vq_pairs * 2; 861 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 862 nvqs++; 863 864 info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); 865 if (info == NULL) 866 return (ENOMEM); 867 868 for (i = 0, idx = 0; i < sc->vtnet_max_vq_pairs; i++, idx+=2) { 869 rxq = &sc->vtnet_rxqs[i]; 870 VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, 871 vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, 872 "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id); 873 874 txq = &sc->vtnet_txqs[i]; 875 VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs, 876 vtnet_tx_vq_intr, txq, &txq->vtntx_vq, 877 "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id); 878 } 879 880 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 881 VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL, 882 &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); 883 } 884 885 /* 886 * Enable interrupt binding if this is multiqueue. This only matters 887 * when per-vq MSIX is available. 888 */ 889 if (sc->vtnet_flags & VTNET_FLAG_MULTIQ) 890 flags |= 0; 891 892 error = virtio_alloc_virtqueues(dev, flags, nvqs, info); 893 free(info, M_TEMP); 894 895 return (error); 896} 897 898static int 899vtnet_setup_interface(struct vtnet_softc *sc) 900{ 901 device_t dev; 902 struct ifnet *ifp; 903 int limit; 904 905 dev = sc->vtnet_dev; 906 907 ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER); 908 if (ifp == NULL) { 909 device_printf(dev, "cannot allocate ifnet structure\n"); 910 return (ENOSPC); 911 } 912 913 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 914 if_initbaudrate(ifp, IF_Gbps(10)); /* Approx. */ 915 ifp->if_softc = sc; 916 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 917 ifp->if_init = vtnet_init; 918 ifp->if_ioctl = vtnet_ioctl; 919 920#ifndef VTNET_LEGACY_TX 921 ifp->if_transmit = vtnet_txq_mq_start; 922 ifp->if_qflush = vtnet_qflush; 923#else 924 struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; 925 ifp->if_start = vtnet_start; 926 IFQ_SET_MAXLEN(&ifp->if_snd, virtqueue_size(vq) - 1); 927 ifp->if_snd.ifq_drv_maxlen = virtqueue_size(vq) - 1; 928 IFQ_SET_READY(&ifp->if_snd); 929#endif 930 931 ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd, 932 vtnet_ifmedia_sts); 933 ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL); 934 ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE); 935 936 /* Read (or generate) the MAC address for the adapter. */ 937 vtnet_get_hwaddr(sc); 938 939 ether_ifattach(ifp, sc->vtnet_hwaddr); 940 941 if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) 942 ifp->if_capabilities |= IFCAP_LINKSTATE; 943 944 /* Tell the upper layer(s) we support long frames. */ 945 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 946 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 947 948 if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { 949 ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6; 950 951 if (virtio_with_feature(dev, VIRTIO_NET_F_GSO)) { 952 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 953 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 954 } else { 955 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) 956 ifp->if_capabilities |= IFCAP_TSO4; 957 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 958 ifp->if_capabilities |= IFCAP_TSO6; 959 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) 960 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 961 } 962 963 if (ifp->if_capabilities & IFCAP_TSO) 964 ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 965 } 966 967 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) 968 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; 969 970 if (ifp->if_capabilities & IFCAP_HWCSUM) { 971 /* 972 * VirtIO does not support VLAN tagging, but we can fake 973 * it by inserting and removing the 802.1Q header during 974 * transmit and receive. We are then able to do checksum 975 * offloading of VLAN frames. 976 */ 977 ifp->if_capabilities |= 978 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 979 } 980 981 ifp->if_capenable = ifp->if_capabilities; 982 983 /* 984 * Capabilities after here are not enabled by default. 985 */ 986 987 if (ifp->if_capabilities & IFCAP_RXCSUM) { 988 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) || 989 virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6)) 990 ifp->if_capabilities |= IFCAP_LRO; 991 } 992 993 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 994 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 995 996 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 997 vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 998 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 999 vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1000 } 1001 1002 limit = vtnet_tunable_int(sc, "rx_process_limit", 1003 vtnet_rx_process_limit); 1004 if (limit < 0) 1005 limit = INT_MAX; 1006 sc->vtnet_rx_process_limit = limit; 1007 1008 return (0); 1009} 1010 1011static int 1012vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu) 1013{ 1014 struct ifnet *ifp; 1015 int frame_size, clsize; 1016 1017 ifp = sc->vtnet_ifp; 1018 1019 if (new_mtu < ETHERMIN || new_mtu > VTNET_MAX_MTU) 1020 return (EINVAL); 1021 1022 frame_size = sc->vtnet_hdr_size + sizeof(struct ether_vlan_header) + 1023 new_mtu; 1024 1025 /* 1026 * Based on the new MTU (and hence frame size) determine which 1027 * cluster size is most appropriate for the receive queues. 1028 */ 1029 if (frame_size <= MCLBYTES) { 1030 clsize = MCLBYTES; 1031 } else if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1032 /* Avoid going past 9K jumbos. */ 1033 if (frame_size > MJUM9BYTES) 1034 return (EINVAL); 1035 clsize = MJUM9BYTES; 1036 } else 1037 clsize = MJUMPAGESIZE; 1038 1039 ifp->if_mtu = new_mtu; 1040 sc->vtnet_rx_new_clsize = clsize; 1041 1042 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1043 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1044 vtnet_init_locked(sc); 1045 } 1046 1047 return (0); 1048} 1049 1050static int 1051vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1052{ 1053 struct vtnet_softc *sc; 1054 struct ifreq *ifr; 1055 int reinit, mask, error; 1056 1057 sc = ifp->if_softc; 1058 ifr = (struct ifreq *) data; 1059 error = 0; 1060 1061 switch (cmd) { 1062 case SIOCSIFMTU: 1063 if (ifp->if_mtu != ifr->ifr_mtu) { 1064 VTNET_CORE_LOCK(sc); 1065 error = vtnet_change_mtu(sc, ifr->ifr_mtu); 1066 VTNET_CORE_UNLOCK(sc); 1067 } 1068 break; 1069 1070 case SIOCSIFFLAGS: 1071 VTNET_CORE_LOCK(sc); 1072 if ((ifp->if_flags & IFF_UP) == 0) { 1073 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1074 vtnet_stop(sc); 1075 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1076 if ((ifp->if_flags ^ sc->vtnet_if_flags) & 1077 (IFF_PROMISC | IFF_ALLMULTI)) { 1078 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) 1079 vtnet_rx_filter(sc); 1080 else 1081 error = ENOTSUP; 1082 } 1083 } else 1084 vtnet_init_locked(sc); 1085 1086 if (error == 0) 1087 sc->vtnet_if_flags = ifp->if_flags; 1088 VTNET_CORE_UNLOCK(sc); 1089 break; 1090 1091 case SIOCADDMULTI: 1092 case SIOCDELMULTI: 1093 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) 1094 break; 1095 VTNET_CORE_LOCK(sc); 1096 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1097 vtnet_rx_filter_mac(sc); 1098 VTNET_CORE_UNLOCK(sc); 1099 break; 1100 1101 case SIOCSIFMEDIA: 1102 case SIOCGIFMEDIA: 1103 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); 1104 break; 1105 1106 case SIOCSIFCAP: 1107 VTNET_CORE_LOCK(sc); 1108 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1109 1110 if (mask & IFCAP_TXCSUM) 1111 ifp->if_capenable ^= IFCAP_TXCSUM; 1112 if (mask & IFCAP_TXCSUM_IPV6) 1113 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1114 if (mask & IFCAP_TSO4) 1115 ifp->if_capenable ^= IFCAP_TSO4; 1116 if (mask & IFCAP_TSO6) 1117 ifp->if_capenable ^= IFCAP_TSO6; 1118 1119 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 1120 IFCAP_VLAN_HWFILTER)) { 1121 /* These Rx features require us to renegotiate. */ 1122 reinit = 1; 1123 1124 if (mask & IFCAP_RXCSUM) 1125 ifp->if_capenable ^= IFCAP_RXCSUM; 1126 if (mask & IFCAP_RXCSUM_IPV6) 1127 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1128 if (mask & IFCAP_LRO) 1129 ifp->if_capenable ^= IFCAP_LRO; 1130 if (mask & IFCAP_VLAN_HWFILTER) 1131 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1132 } else 1133 reinit = 0; 1134 1135 if (mask & IFCAP_VLAN_HWTSO) 1136 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1137 if (mask & IFCAP_VLAN_HWTAGGING) 1138 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1139 1140 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1141 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1142 vtnet_init_locked(sc); 1143 } 1144 1145 VTNET_CORE_UNLOCK(sc); 1146 VLAN_CAPABILITIES(ifp); 1147 1148 break; 1149 1150 default: 1151 error = ether_ioctl(ifp, cmd, data); 1152 break; 1153 } 1154 1155 VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc); 1156 1157 return (error); 1158} 1159 1160static int 1161vtnet_rxq_populate(struct vtnet_rxq *rxq) 1162{ 1163 struct virtqueue *vq; 1164 int nbufs, error; 1165 1166 vq = rxq->vtnrx_vq; 1167 error = ENOSPC; 1168 1169 for (nbufs = 0; !virtqueue_full(vq); nbufs++) { 1170 error = vtnet_rxq_new_buf(rxq); 1171 if (error) 1172 break; 1173 } 1174 1175 if (nbufs > 0) { 1176 virtqueue_notify(vq); 1177 /* 1178 * EMSGSIZE signifies the virtqueue did not have enough 1179 * entries available to hold the last mbuf. This is not 1180 * an error. 1181 */ 1182 if (error == EMSGSIZE) 1183 error = 0; 1184 } 1185 1186 return (error); 1187} 1188 1189static void 1190vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq) 1191{ 1192 struct virtqueue *vq; 1193 struct mbuf *m; 1194 int last; 1195 1196 vq = rxq->vtnrx_vq; 1197 last = 0; 1198 1199 while ((m = virtqueue_drain(vq, &last)) != NULL) 1200 m_freem(m); 1201 1202 KASSERT(virtqueue_empty(vq), 1203 ("%s: mbufs remaining in rx queue %p", __func__, rxq)); 1204} 1205 1206static struct mbuf * 1207vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) 1208{ 1209 struct mbuf *m_head, *m_tail, *m; 1210 int i, clsize; 1211 1212 clsize = sc->vtnet_rx_clsize; 1213 1214 KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1215 ("%s: chained mbuf %d request without LRO_NOMRG", __func__, nbufs)); 1216 1217 m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize); 1218 if (m_head == NULL) 1219 goto fail; 1220 1221 m_head->m_len = clsize; 1222 m_tail = m_head; 1223 1224 /* Allocate the rest of the chain. */ 1225 for (i = 1; i < nbufs; i++) { 1226 m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize); 1227 if (m == NULL) 1228 goto fail; 1229 1230 m->m_len = clsize; 1231 m_tail->m_next = m; 1232 m_tail = m; 1233 } 1234 1235 if (m_tailp != NULL) 1236 *m_tailp = m_tail; 1237 1238 return (m_head); 1239 1240fail: 1241 sc->vtnet_stats.mbuf_alloc_failed++; 1242 m_freem(m_head); 1243 1244 return (NULL); 1245} 1246 1247/* 1248 * Slow path for when LRO without mergeable buffers is negotiated. 1249 */ 1250static int 1251vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0, 1252 int len0) 1253{ 1254 struct vtnet_softc *sc; 1255 struct mbuf *m, *m_prev; 1256 struct mbuf *m_new, *m_tail; 1257 int len, clsize, nreplace, error; 1258 1259 sc = rxq->vtnrx_sc; 1260 clsize = sc->vtnet_rx_clsize; 1261 1262 m_prev = NULL; 1263 m_tail = NULL; 1264 nreplace = 0; 1265 1266 m = m0; 1267 len = len0; 1268 1269 /* 1270 * Since these mbuf chains are so large, we avoid allocating an 1271 * entire replacement chain if possible. When the received frame 1272 * did not consume the entire chain, the unused mbufs are moved 1273 * to the replacement chain. 1274 */ 1275 while (len > 0) { 1276 /* 1277 * Something is seriously wrong if we received a frame 1278 * larger than the chain. Drop it. 1279 */ 1280 if (m == NULL) { 1281 sc->vtnet_stats.rx_frame_too_large++; 1282 return (EMSGSIZE); 1283 } 1284 1285 /* We always allocate the same cluster size. */ 1286 KASSERT(m->m_len == clsize, 1287 ("%s: mbuf size %d is not the cluster size %d", 1288 __func__, m->m_len, clsize)); 1289 1290 m->m_len = MIN(m->m_len, len); 1291 len -= m->m_len; 1292 1293 m_prev = m; 1294 m = m->m_next; 1295 nreplace++; 1296 } 1297 1298 KASSERT(nreplace <= sc->vtnet_rx_nmbufs, 1299 ("%s: too many replacement mbufs %d max %d", __func__, nreplace, 1300 sc->vtnet_rx_nmbufs)); 1301 1302 m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail); 1303 if (m_new == NULL) { 1304 m_prev->m_len = clsize; 1305 return (ENOBUFS); 1306 } 1307 1308 /* 1309 * Move any unused mbufs from the received chain onto the end 1310 * of the new chain. 1311 */ 1312 if (m_prev->m_next != NULL) { 1313 m_tail->m_next = m_prev->m_next; 1314 m_prev->m_next = NULL; 1315 } 1316 1317 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1318 if (error) { 1319 /* 1320 * BAD! We could not enqueue the replacement mbuf chain. We 1321 * must restore the m0 chain to the original state if it was 1322 * modified so we can subsequently discard it. 1323 * 1324 * NOTE: The replacement is suppose to be an identical copy 1325 * to the one just dequeued so this is an unexpected error. 1326 */ 1327 sc->vtnet_stats.rx_enq_replacement_failed++; 1328 1329 if (m_tail->m_next != NULL) { 1330 m_prev->m_next = m_tail->m_next; 1331 m_tail->m_next = NULL; 1332 } 1333 1334 m_prev->m_len = clsize; 1335 m_freem(m_new); 1336 } 1337 1338 return (error); 1339} 1340 1341static int 1342vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len) 1343{ 1344 struct vtnet_softc *sc; 1345 struct mbuf *m_new; 1346 int error; 1347 1348 sc = rxq->vtnrx_sc; 1349 1350 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1351 ("%s: chained mbuf without LRO_NOMRG", __func__)); 1352 1353 if (m->m_next == NULL) { 1354 /* Fast-path for the common case of just one mbuf. */ 1355 if (m->m_len < len) 1356 return (EINVAL); 1357 1358 m_new = vtnet_rx_alloc_buf(sc, 1, NULL); 1359 if (m_new == NULL) 1360 return (ENOBUFS); 1361 1362 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1363 if (error) { 1364 /* 1365 * The new mbuf is suppose to be an identical 1366 * copy of the one just dequeued so this is an 1367 * unexpected error. 1368 */ 1369 m_freem(m_new); 1370 sc->vtnet_stats.rx_enq_replacement_failed++; 1371 } else 1372 m->m_len = len; 1373 } else 1374 error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len); 1375 1376 return (error); 1377} 1378 1379static int 1380vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1381{ 1382 struct vtnet_softc *sc; 1383 struct sglist *sg; 1384 struct vtnet_rx_header *rxhdr; 1385 uint8_t *mdata; 1386 int offset, error; 1387 1388 sc = rxq->vtnrx_sc; 1389 sg = rxq->vtnrx_sg; 1390 mdata = mtod(m, uint8_t *); 1391 1392 VTNET_RXQ_LOCK_ASSERT(rxq); 1393 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1394 ("%s: chained mbuf without LRO_NOMRG", __func__)); 1395 KASSERT(m->m_len == sc->vtnet_rx_clsize, 1396 ("%s: unexpected cluster size %d/%d", __func__, m->m_len, 1397 sc->vtnet_rx_clsize)); 1398 1399 sglist_reset(sg); 1400 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1401 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); 1402 rxhdr = (struct vtnet_rx_header *) mdata; 1403 sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size); 1404 offset = sizeof(struct vtnet_rx_header); 1405 } else 1406 offset = 0; 1407 1408 sglist_append(sg, mdata + offset, m->m_len - offset); 1409 if (m->m_next != NULL) { 1410 error = sglist_append_mbuf(sg, m->m_next); 1411 MPASS(error == 0); 1412 } 1413 1414 error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg); 1415 1416 return (error); 1417} 1418 1419static int 1420vtnet_rxq_new_buf(struct vtnet_rxq *rxq) 1421{ 1422 struct vtnet_softc *sc; 1423 struct mbuf *m; 1424 int error; 1425 1426 sc = rxq->vtnrx_sc; 1427 1428 m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); 1429 if (m == NULL) 1430 return (ENOBUFS); 1431 1432 error = vtnet_rxq_enqueue_buf(rxq, m); 1433 if (error) 1434 m_freem(m); 1435 1436 return (error); 1437} 1438 1439/* 1440 * Use the checksum offset in the VirtIO header to set the 1441 * correct CSUM_* flags. 1442 */ 1443static int 1444vtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m, 1445 uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1446{ 1447 struct vtnet_softc *sc; 1448#if defined(INET) || defined(INET6) 1449 int offset = hdr->csum_start + hdr->csum_offset; 1450#endif 1451 1452 sc = rxq->vtnrx_sc; 1453 1454 /* Only do a basic sanity check on the offset. */ 1455 switch (eth_type) { 1456#if defined(INET) 1457 case ETHERTYPE_IP: 1458 if (__predict_false(offset < ip_start + sizeof(struct ip))) 1459 return (1); 1460 break; 1461#endif 1462#if defined(INET6) 1463 case ETHERTYPE_IPV6: 1464 if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr))) 1465 return (1); 1466 break; 1467#endif 1468 default: 1469 sc->vtnet_stats.rx_csum_bad_ethtype++; 1470 return (1); 1471 } 1472 1473 /* 1474 * Use the offset to determine the appropriate CSUM_* flags. This is 1475 * a bit dirty, but we can get by with it since the checksum offsets 1476 * happen to be different. We assume the host host does not do IPv4 1477 * header checksum offloading. 1478 */ 1479 switch (hdr->csum_offset) { 1480 case offsetof(struct udphdr, uh_sum): 1481 case offsetof(struct tcphdr, th_sum): 1482 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1483 m->m_pkthdr.csum_data = 0xFFFF; 1484 break; 1485 case offsetof(struct sctphdr, checksum): 1486 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1487 break; 1488 default: 1489 sc->vtnet_stats.rx_csum_bad_offset++; 1490 return (1); 1491 } 1492 1493 return (0); 1494} 1495 1496static int 1497vtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m, 1498 uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1499{ 1500 struct vtnet_softc *sc; 1501 int offset, proto; 1502 1503 sc = rxq->vtnrx_sc; 1504 1505 switch (eth_type) { 1506#if defined(INET) 1507 case ETHERTYPE_IP: { 1508 struct ip *ip; 1509 if (__predict_false(m->m_len < ip_start + sizeof(struct ip))) 1510 return (1); 1511 ip = (struct ip *)(m->m_data + ip_start); 1512 proto = ip->ip_p; 1513 offset = ip_start + (ip->ip_hl << 2); 1514 break; 1515 } 1516#endif 1517#if defined(INET6) 1518 case ETHERTYPE_IPV6: 1519 if (__predict_false(m->m_len < ip_start + 1520 sizeof(struct ip6_hdr))) 1521 return (1); 1522 offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto); 1523 if (__predict_false(offset < 0)) 1524 return (1); 1525 break; 1526#endif 1527 default: 1528 sc->vtnet_stats.rx_csum_bad_ethtype++; 1529 return (1); 1530 } 1531 1532 switch (proto) { 1533 case IPPROTO_TCP: 1534 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) 1535 return (1); 1536 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1537 m->m_pkthdr.csum_data = 0xFFFF; 1538 break; 1539 case IPPROTO_UDP: 1540 if (__predict_false(m->m_len < offset + sizeof(struct udphdr))) 1541 return (1); 1542 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1543 m->m_pkthdr.csum_data = 0xFFFF; 1544 break; 1545 case IPPROTO_SCTP: 1546 if (__predict_false(m->m_len < offset + sizeof(struct sctphdr))) 1547 return (1); 1548 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1549 break; 1550 default: 1551 /* 1552 * For the remaining protocols, FreeBSD does not support 1553 * checksum offloading, so the checksum will be recomputed. 1554 */ 1555#if 0 1556 if_printf(sc->vtnet_ifp, "cksum offload of unsupported " 1557 "protocol eth_type=%#x proto=%d csum_start=%d " 1558 "csum_offset=%d\n", __func__, eth_type, proto, 1559 hdr->csum_start, hdr->csum_offset); 1560#endif 1561 break; 1562 } 1563 1564 return (0); 1565} 1566 1567/* 1568 * Set the appropriate CSUM_* flags. Unfortunately, the information 1569 * provided is not directly useful to us. The VirtIO header gives the 1570 * offset of the checksum, which is all Linux needs, but this is not 1571 * how FreeBSD does things. We are forced to peek inside the packet 1572 * a bit. 1573 * 1574 * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD 1575 * could accept the offsets and let the stack figure it out. 1576 */ 1577static int 1578vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, 1579 struct virtio_net_hdr *hdr) 1580{ 1581 struct ether_header *eh; 1582 struct ether_vlan_header *evh; 1583 uint16_t eth_type; 1584 int offset, error; 1585 1586 eh = mtod(m, struct ether_header *); 1587 eth_type = ntohs(eh->ether_type); 1588 if (eth_type == ETHERTYPE_VLAN) { 1589 /* BMV: We should handle nested VLAN tags too. */ 1590 evh = mtod(m, struct ether_vlan_header *); 1591 eth_type = ntohs(evh->evl_proto); 1592 offset = sizeof(struct ether_vlan_header); 1593 } else 1594 offset = sizeof(struct ether_header); 1595 1596 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1597 error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr); 1598 else 1599 error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr); 1600 1601 return (error); 1602} 1603 1604static void 1605vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) 1606{ 1607 struct mbuf *m; 1608 1609 while (--nbufs > 0) { 1610 m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); 1611 if (m == NULL) 1612 break; 1613 vtnet_rxq_discard_buf(rxq, m); 1614 } 1615} 1616 1617static void 1618vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1619{ 1620 int error; 1621 1622 /* 1623 * Requeue the discarded mbuf. This should always be successful 1624 * since it was just dequeued. 1625 */ 1626 error = vtnet_rxq_enqueue_buf(rxq, m); 1627 KASSERT(error == 0, 1628 ("%s: cannot requeue discarded mbuf %d", __func__, error)); 1629} 1630 1631static int 1632vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs) 1633{ 1634 struct vtnet_softc *sc; 1635 struct ifnet *ifp; 1636 struct virtqueue *vq; 1637 struct mbuf *m, *m_tail; 1638 int len; 1639 1640 sc = rxq->vtnrx_sc; 1641 vq = rxq->vtnrx_vq; 1642 ifp = sc->vtnet_ifp; 1643 m_tail = m_head; 1644 1645 while (--nbufs > 0) { 1646 m = virtqueue_dequeue(vq, &len); 1647 if (m == NULL) { 1648 rxq->vtnrx_stats.vrxs_ierrors++; 1649 goto fail; 1650 } 1651 1652 if (vtnet_rxq_new_buf(rxq) != 0) { 1653 rxq->vtnrx_stats.vrxs_iqdrops++; 1654 vtnet_rxq_discard_buf(rxq, m); 1655 if (nbufs > 1) 1656 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1657 goto fail; 1658 } 1659 1660 if (m->m_len < len) 1661 len = m->m_len; 1662 1663 m->m_len = len; 1664 m->m_flags &= ~M_PKTHDR; 1665 1666 m_head->m_pkthdr.len += len; 1667 m_tail->m_next = m; 1668 m_tail = m; 1669 } 1670 1671 return (0); 1672 1673fail: 1674 sc->vtnet_stats.rx_mergeable_failed++; 1675 m_freem(m_head); 1676 1677 return (1); 1678} 1679 1680static void 1681vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, 1682 struct virtio_net_hdr *hdr) 1683{ 1684 struct vtnet_softc *sc; 1685 struct ifnet *ifp; 1686 struct ether_header *eh; 1687 1688 sc = rxq->vtnrx_sc; 1689 ifp = sc->vtnet_ifp; 1690 1691 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1692 eh = mtod(m, struct ether_header *); 1693 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1694 vtnet_vlan_tag_remove(m); 1695 /* 1696 * With the 802.1Q header removed, update the 1697 * checksum starting location accordingly. 1698 */ 1699 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1700 hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; 1701 } 1702 } 1703 1704 m->m_pkthdr.flowid = rxq->vtnrx_id; 1705 m->m_flags |= M_FLOWID; 1706 1707 /* 1708 * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum 1709 * distinction that Linux does. Need to reevaluate if performing 1710 * offloading for the NEEDS_CSUM case is really appropriate. 1711 */ 1712 if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | 1713 VIRTIO_NET_HDR_F_DATA_VALID)) { 1714 if (vtnet_rxq_csum(rxq, m, hdr) == 0) 1715 rxq->vtnrx_stats.vrxs_csum++; 1716 else 1717 rxq->vtnrx_stats.vrxs_csum_failed++; 1718 } 1719 1720 rxq->vtnrx_stats.vrxs_ipackets++; 1721 rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; 1722 1723 VTNET_RXQ_UNLOCK(rxq); 1724 (*ifp->if_input)(ifp, m); 1725 VTNET_RXQ_LOCK(rxq); 1726} 1727 1728static int 1729vtnet_rxq_eof(struct vtnet_rxq *rxq) 1730{ 1731 struct virtio_net_hdr lhdr, *hdr; 1732 struct vtnet_softc *sc; 1733 struct ifnet *ifp; 1734 struct virtqueue *vq; 1735 struct mbuf *m; 1736 struct virtio_net_hdr_mrg_rxbuf *mhdr; 1737 int len, deq, nbufs, adjsz, count; 1738 1739 sc = rxq->vtnrx_sc; 1740 vq = rxq->vtnrx_vq; 1741 ifp = sc->vtnet_ifp; 1742 hdr = &lhdr; 1743 deq = 0; 1744 count = sc->vtnet_rx_process_limit; 1745 1746 VTNET_RXQ_LOCK_ASSERT(rxq); 1747 1748 while (count-- > 0) { 1749 m = virtqueue_dequeue(vq, &len); 1750 if (m == NULL) 1751 break; 1752 deq++; 1753 1754 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { 1755 rxq->vtnrx_stats.vrxs_ierrors++; 1756 vtnet_rxq_discard_buf(rxq, m); 1757 continue; 1758 } 1759 1760 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1761 nbufs = 1; 1762 adjsz = sizeof(struct vtnet_rx_header); 1763 /* 1764 * Account for our pad inserted between the header 1765 * and the actual start of the frame. 1766 */ 1767 len += VTNET_RX_HEADER_PAD; 1768 } else { 1769 mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *); 1770 nbufs = mhdr->num_buffers; 1771 adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1772 } 1773 1774 if (vtnet_rxq_replace_buf(rxq, m, len) != 0) { 1775 rxq->vtnrx_stats.vrxs_iqdrops++; 1776 vtnet_rxq_discard_buf(rxq, m); 1777 if (nbufs > 1) 1778 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1779 continue; 1780 } 1781 1782 m->m_pkthdr.len = len; 1783 m->m_pkthdr.rcvif = ifp; 1784 m->m_pkthdr.csum_flags = 0; 1785 1786 if (nbufs > 1) { 1787 /* Dequeue the rest of chain. */ 1788 if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0) 1789 continue; 1790 } 1791 1792 /* 1793 * Save copy of header before we strip it. For both mergeable 1794 * and non-mergeable, the header is at the beginning of the 1795 * mbuf data. We no longer need num_buffers, so always use a 1796 * regular header. 1797 * 1798 * BMV: Is this memcpy() expensive? We know the mbuf data is 1799 * still valid even after the m_adj(). 1800 */ 1801 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr)); 1802 m_adj(m, adjsz); 1803 1804 vtnet_rxq_input(rxq, m, hdr); 1805 1806 /* Must recheck after dropping the Rx lock. */ 1807 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1808 break; 1809 } 1810 1811 if (deq > 0) 1812 virtqueue_notify(vq); 1813 1814 return (count > 0 ? 0 : EAGAIN); 1815} 1816 1817static void 1818vtnet_rx_vq_intr(void *xrxq) 1819{ 1820 struct vtnet_softc *sc; 1821 struct vtnet_rxq *rxq; 1822 struct ifnet *ifp; 1823 int tries, more; 1824 1825 rxq = xrxq; 1826 sc = rxq->vtnrx_sc; 1827 ifp = sc->vtnet_ifp; 1828 tries = 0; 1829 1830 if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { 1831 /* 1832 * Ignore this interrupt. Either this is a spurious interrupt 1833 * or multiqueue without per-VQ MSIX so every queue needs to 1834 * be polled (a brain dead configuration we could try harder 1835 * to avoid). 1836 */ 1837 vtnet_rxq_disable_intr(rxq); 1838 return; 1839 } 1840 1841 VTNET_RXQ_LOCK(rxq); 1842 1843again: 1844 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1845 VTNET_RXQ_UNLOCK(rxq); 1846 return; 1847 } 1848 1849 more = vtnet_rxq_eof(rxq); 1850 if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1851 if (!more) 1852 vtnet_rxq_disable_intr(rxq); 1853 /* 1854 * This is an occasional condition or race (when !more), 1855 * so retry a few times before scheduling the taskqueue. 1856 */ 1857 if (tries++ < VTNET_INTR_DISABLE_RETRIES) 1858 goto again; 1859 1860 VTNET_RXQ_UNLOCK(rxq); 1861 rxq->vtnrx_stats.vrxs_rescheduled++; 1862 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1863 } else 1864 VTNET_RXQ_UNLOCK(rxq); 1865} 1866 1867static void 1868vtnet_rxq_tq_intr(void *xrxq, int pending) 1869{ 1870 struct vtnet_softc *sc; 1871 struct vtnet_rxq *rxq; 1872 struct ifnet *ifp; 1873 int more; 1874 1875 rxq = xrxq; 1876 sc = rxq->vtnrx_sc; 1877 ifp = sc->vtnet_ifp; 1878 1879 VTNET_RXQ_LOCK(rxq); 1880 1881 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1882 VTNET_RXQ_UNLOCK(rxq); 1883 return; 1884 } 1885 1886 more = vtnet_rxq_eof(rxq); 1887 if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1888 if (!more) 1889 vtnet_rxq_disable_intr(rxq); 1890 rxq->vtnrx_stats.vrxs_rescheduled++; 1891 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1892 } 1893 1894 VTNET_RXQ_UNLOCK(rxq); 1895} 1896 1897static void 1898vtnet_txq_free_mbufs(struct vtnet_txq *txq) 1899{ 1900 struct virtqueue *vq; 1901 struct vtnet_tx_header *txhdr; 1902 int last; 1903 1904 vq = txq->vtntx_vq; 1905 last = 0; 1906 1907 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 1908 m_freem(txhdr->vth_mbuf); 1909 uma_zfree(vtnet_tx_header_zone, txhdr); 1910 } 1911 1912 KASSERT(virtqueue_empty(vq), 1913 ("%s: mbufs remaining in tx queue %p", __func__, txq)); 1914} 1915 1916/* 1917 * BMV: Much of this can go away once we finally have offsets in 1918 * the mbuf packet header. Bug andre@. 1919 */ 1920static int 1921vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, 1922 int *etype, int *proto, int *start) 1923{ 1924 struct vtnet_softc *sc; 1925 struct ether_vlan_header *evh; 1926 int offset; 1927 1928 sc = txq->vtntx_sc; 1929 1930 evh = mtod(m, struct ether_vlan_header *); 1931 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1932 /* BMV: We should handle nested VLAN tags too. */ 1933 *etype = ntohs(evh->evl_proto); 1934 offset = sizeof(struct ether_vlan_header); 1935 } else { 1936 *etype = ntohs(evh->evl_encap_proto); 1937 offset = sizeof(struct ether_header); 1938 } 1939 1940 switch (*etype) { 1941#if defined(INET) 1942 case ETHERTYPE_IP: { 1943 struct ip *ip, iphdr; 1944 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 1945 m_copydata(m, offset, sizeof(struct ip), 1946 (caddr_t) &iphdr); 1947 ip = &iphdr; 1948 } else 1949 ip = (struct ip *)(m->m_data + offset); 1950 *proto = ip->ip_p; 1951 *start = offset + (ip->ip_hl << 2); 1952 break; 1953 } 1954#endif 1955#if defined(INET6) 1956 case ETHERTYPE_IPV6: 1957 *proto = -1; 1958 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 1959 /* Assert the network stack sent us a valid packet. */ 1960 KASSERT(*start > offset, 1961 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 1962 *start, offset, *proto)); 1963 break; 1964#endif 1965 default: 1966 sc->vtnet_stats.tx_csum_bad_ethtype++; 1967 return (EINVAL); 1968 } 1969 1970 return (0); 1971} 1972 1973static int 1974vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, 1975 int offset, struct virtio_net_hdr *hdr) 1976{ 1977 static struct timeval lastecn; 1978 static int curecn; 1979 struct vtnet_softc *sc; 1980 struct tcphdr *tcp, tcphdr; 1981 1982 sc = txq->vtntx_sc; 1983 1984 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { 1985 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); 1986 tcp = &tcphdr; 1987 } else 1988 tcp = (struct tcphdr *)(m->m_data + offset); 1989 1990 hdr->hdr_len = offset + (tcp->th_off << 2); 1991 hdr->gso_size = m->m_pkthdr.tso_segsz; 1992 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : 1993 VIRTIO_NET_HDR_GSO_TCPV6; 1994 1995 if (tcp->th_flags & TH_CWR) { 1996 /* 1997 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD, 1998 * ECN support is not on a per-interface basis, but globally via 1999 * the net.inet.tcp.ecn.enable sysctl knob. The default is off. 2000 */ 2001 if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { 2002 if (ppsratecheck(&lastecn, &curecn, 1)) 2003 if_printf(sc->vtnet_ifp, 2004 "TSO with ECN not negotiated with host\n"); 2005 return (ENOTSUP); 2006 } 2007 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 2008 } 2009 2010 txq->vtntx_stats.vtxs_tso++; 2011 2012 return (0); 2013} 2014 2015static struct mbuf * 2016vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, 2017 struct virtio_net_hdr *hdr) 2018{ 2019 struct vtnet_softc *sc; 2020 int flags, etype, csum_start, proto, error; 2021 2022 sc = txq->vtntx_sc; 2023 flags = m->m_pkthdr.csum_flags; 2024 2025 error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); 2026 if (error) 2027 goto drop; 2028 2029 if ((etype == ETHERTYPE_IP && flags & VTNET_CSUM_OFFLOAD) || 2030 (etype == ETHERTYPE_IPV6 && flags & VTNET_CSUM_OFFLOAD_IPV6)) { 2031 /* 2032 * We could compare the IP protocol vs the CSUM_ flag too, 2033 * but that really should not be necessary. 2034 */ 2035 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; 2036 hdr->csum_start = csum_start; 2037 hdr->csum_offset = m->m_pkthdr.csum_data; 2038 txq->vtntx_stats.vtxs_csum++; 2039 } 2040 2041 if (flags & CSUM_TSO) { 2042 if (__predict_false(proto != IPPROTO_TCP)) { 2043 /* Likely failed to correctly parse the mbuf. */ 2044 sc->vtnet_stats.tx_tso_not_tcp++; 2045 goto drop; 2046 } 2047 2048 KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM, 2049 ("%s: mbuf %p TSO without checksum offload %#x", 2050 __func__, m, flags)); 2051 2052 error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); 2053 if (error) 2054 goto drop; 2055 } 2056 2057 return (m); 2058 2059drop: 2060 m_freem(m); 2061 return (NULL); 2062} 2063 2064static int 2065vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, 2066 struct vtnet_tx_header *txhdr) 2067{ 2068 struct vtnet_softc *sc; 2069 struct virtqueue *vq; 2070 struct sglist *sg; 2071 struct mbuf *m; 2072 int error; 2073 2074 sc = txq->vtntx_sc; 2075 vq = txq->vtntx_vq; 2076 sg = txq->vtntx_sg; 2077 m = *m_head; 2078 2079 sglist_reset(sg); 2080 error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); 2081 KASSERT(error == 0 && sg->sg_nseg == 1, 2082 ("%s: error %d adding header to sglist", __func__, error)); 2083 2084 error = sglist_append_mbuf(sg, m); 2085 if (error) { 2086 m = m_defrag(m, M_NOWAIT); 2087 if (m == NULL) 2088 goto fail; 2089 2090 *m_head = m; 2091 sc->vtnet_stats.tx_defragged++; 2092 2093 error = sglist_append_mbuf(sg, m); 2094 if (error) 2095 goto fail; 2096 } 2097 2098 txhdr->vth_mbuf = m; 2099 error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); 2100 2101 return (error); 2102 2103fail: 2104 sc->vtnet_stats.tx_defrag_failed++; 2105 m_freem(*m_head); 2106 *m_head = NULL; 2107 2108 return (ENOBUFS); 2109} 2110 2111static int 2112vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head) 2113{
| 585} 586 587static void 588vtnet_setup_features(struct vtnet_softc *sc) 589{ 590 device_t dev; 591 int max_pairs, max; 592 593 dev = sc->vtnet_dev; 594 595 vtnet_negotiate_features(sc); 596 597 if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX)) 598 sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; 599 600 if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { 601 /* This feature should always be negotiated. */ 602 sc->vtnet_flags |= VTNET_FLAG_MAC; 603 } 604 605 if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { 606 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; 607 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 608 } else 609 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 610 611 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) 612 sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS; 613 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 614 sc->vtnet_rx_nsegs = VTNET_MAX_RX_SEGS; 615 else 616 sc->vtnet_rx_nsegs = VTNET_MIN_RX_SEGS; 617 618 if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || 619 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || 620 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 621 sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS; 622 else 623 sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS; 624 625 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { 626 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; 627 628 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) 629 sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; 630 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) 631 sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; 632 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) 633 sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; 634 } 635 636 if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) && 637 sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 638 max_pairs = virtio_read_dev_config_2(dev, 639 offsetof(struct virtio_net_config, max_virtqueue_pairs)); 640 if (max_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 641 max_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 642 max_pairs = 1; 643 } else 644 max_pairs = 1; 645 646 if (max_pairs > 1) { 647 /* 648 * Limit the maximum number of queue pairs to the number of 649 * CPUs or the configured maximum. The actual number of 650 * queues that get used may be less. 651 */ 652 max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); 653 if (max > 0 && max_pairs > max) 654 max_pairs = max; 655 if (max_pairs > mp_ncpus) 656 max_pairs = mp_ncpus; 657 if (max_pairs > VTNET_MAX_QUEUE_PAIRS) 658 max_pairs = VTNET_MAX_QUEUE_PAIRS; 659 if (max_pairs > 1) 660 sc->vtnet_flags |= VTNET_FLAG_MULTIQ; 661 } 662 663 sc->vtnet_max_vq_pairs = max_pairs; 664} 665 666static int 667vtnet_init_rxq(struct vtnet_softc *sc, int id) 668{ 669 struct vtnet_rxq *rxq; 670 671 rxq = &sc->vtnet_rxqs[id]; 672 673 snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", 674 device_get_nameunit(sc->vtnet_dev), id); 675 mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); 676 677 rxq->vtnrx_sc = sc; 678 rxq->vtnrx_id = id; 679 680 rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); 681 if (rxq->vtnrx_sg == NULL) 682 return (ENOMEM); 683 684 TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); 685 rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, 686 taskqueue_thread_enqueue, &rxq->vtnrx_tq); 687 688 return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); 689} 690 691static int 692vtnet_init_txq(struct vtnet_softc *sc, int id) 693{ 694 struct vtnet_txq *txq; 695 696 txq = &sc->vtnet_txqs[id]; 697 698 snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", 699 device_get_nameunit(sc->vtnet_dev), id); 700 mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); 701 702 txq->vtntx_sc = sc; 703 txq->vtntx_id = id; 704 705 txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); 706 if (txq->vtntx_sg == NULL) 707 return (ENOMEM); 708 709#ifndef VTNET_LEGACY_TX 710 txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, 711 M_NOWAIT, &txq->vtntx_mtx); 712 if (txq->vtntx_br == NULL) 713 return (ENOMEM); 714 715 TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); 716#endif 717 TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); 718 txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, 719 taskqueue_thread_enqueue, &txq->vtntx_tq); 720 if (txq->vtntx_tq == NULL) 721 return (ENOMEM); 722 723 return (0); 724} 725 726static int 727vtnet_alloc_rxtx_queues(struct vtnet_softc *sc) 728{ 729 int i, npairs, error; 730 731 npairs = sc->vtnet_max_vq_pairs; 732 733 sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, 734 M_NOWAIT | M_ZERO); 735 sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, 736 M_NOWAIT | M_ZERO); 737 if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) 738 return (ENOMEM); 739 740 for (i = 0; i < npairs; i++) { 741 error = vtnet_init_rxq(sc, i); 742 if (error) 743 return (error); 744 error = vtnet_init_txq(sc, i); 745 if (error) 746 return (error); 747 } 748 749 vtnet_setup_queue_sysctl(sc); 750 751 return (0); 752} 753 754static void 755vtnet_destroy_rxq(struct vtnet_rxq *rxq) 756{ 757 758 rxq->vtnrx_sc = NULL; 759 rxq->vtnrx_id = -1; 760 761 if (rxq->vtnrx_sg != NULL) { 762 sglist_free(rxq->vtnrx_sg); 763 rxq->vtnrx_sg = NULL; 764 } 765 766 if (mtx_initialized(&rxq->vtnrx_mtx) != 0) 767 mtx_destroy(&rxq->vtnrx_mtx); 768} 769 770static void 771vtnet_destroy_txq(struct vtnet_txq *txq) 772{ 773 774 txq->vtntx_sc = NULL; 775 txq->vtntx_id = -1; 776 777 if (txq->vtntx_sg != NULL) { 778 sglist_free(txq->vtntx_sg); 779 txq->vtntx_sg = NULL; 780 } 781 782#ifndef VTNET_LEGACY_TX 783 if (txq->vtntx_br != NULL) { 784 buf_ring_free(txq->vtntx_br, M_DEVBUF); 785 txq->vtntx_br = NULL; 786 } 787#endif 788 789 if (mtx_initialized(&txq->vtntx_mtx) != 0) 790 mtx_destroy(&txq->vtntx_mtx); 791} 792 793static void 794vtnet_free_rxtx_queues(struct vtnet_softc *sc) 795{ 796 int i; 797 798 if (sc->vtnet_rxqs != NULL) { 799 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 800 vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); 801 free(sc->vtnet_rxqs, M_DEVBUF); 802 sc->vtnet_rxqs = NULL; 803 } 804 805 if (sc->vtnet_txqs != NULL) { 806 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 807 vtnet_destroy_txq(&sc->vtnet_txqs[i]); 808 free(sc->vtnet_txqs, M_DEVBUF); 809 sc->vtnet_txqs = NULL; 810 } 811} 812 813static int 814vtnet_alloc_rx_filters(struct vtnet_softc *sc) 815{ 816 817 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 818 sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), 819 M_DEVBUF, M_NOWAIT | M_ZERO); 820 if (sc->vtnet_mac_filter == NULL) 821 return (ENOMEM); 822 } 823 824 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 825 sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * 826 VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO); 827 if (sc->vtnet_vlan_filter == NULL) 828 return (ENOMEM); 829 } 830 831 return (0); 832} 833 834static void 835vtnet_free_rx_filters(struct vtnet_softc *sc) 836{ 837 838 if (sc->vtnet_mac_filter != NULL) { 839 free(sc->vtnet_mac_filter, M_DEVBUF); 840 sc->vtnet_mac_filter = NULL; 841 } 842 843 if (sc->vtnet_vlan_filter != NULL) { 844 free(sc->vtnet_vlan_filter, M_DEVBUF); 845 sc->vtnet_vlan_filter = NULL; 846 } 847} 848 849static int 850vtnet_alloc_virtqueues(struct vtnet_softc *sc) 851{ 852 device_t dev; 853 struct vq_alloc_info *info; 854 struct vtnet_rxq *rxq; 855 struct vtnet_txq *txq; 856 int i, idx, flags, nvqs, error; 857 858 dev = sc->vtnet_dev; 859 flags = 0; 860 861 nvqs = sc->vtnet_max_vq_pairs * 2; 862 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 863 nvqs++; 864 865 info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); 866 if (info == NULL) 867 return (ENOMEM); 868 869 for (i = 0, idx = 0; i < sc->vtnet_max_vq_pairs; i++, idx+=2) { 870 rxq = &sc->vtnet_rxqs[i]; 871 VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, 872 vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, 873 "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id); 874 875 txq = &sc->vtnet_txqs[i]; 876 VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs, 877 vtnet_tx_vq_intr, txq, &txq->vtntx_vq, 878 "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id); 879 } 880 881 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 882 VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL, 883 &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); 884 } 885 886 /* 887 * Enable interrupt binding if this is multiqueue. This only matters 888 * when per-vq MSIX is available. 889 */ 890 if (sc->vtnet_flags & VTNET_FLAG_MULTIQ) 891 flags |= 0; 892 893 error = virtio_alloc_virtqueues(dev, flags, nvqs, info); 894 free(info, M_TEMP); 895 896 return (error); 897} 898 899static int 900vtnet_setup_interface(struct vtnet_softc *sc) 901{ 902 device_t dev; 903 struct ifnet *ifp; 904 int limit; 905 906 dev = sc->vtnet_dev; 907 908 ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER); 909 if (ifp == NULL) { 910 device_printf(dev, "cannot allocate ifnet structure\n"); 911 return (ENOSPC); 912 } 913 914 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 915 if_initbaudrate(ifp, IF_Gbps(10)); /* Approx. */ 916 ifp->if_softc = sc; 917 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 918 ifp->if_init = vtnet_init; 919 ifp->if_ioctl = vtnet_ioctl; 920 921#ifndef VTNET_LEGACY_TX 922 ifp->if_transmit = vtnet_txq_mq_start; 923 ifp->if_qflush = vtnet_qflush; 924#else 925 struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; 926 ifp->if_start = vtnet_start; 927 IFQ_SET_MAXLEN(&ifp->if_snd, virtqueue_size(vq) - 1); 928 ifp->if_snd.ifq_drv_maxlen = virtqueue_size(vq) - 1; 929 IFQ_SET_READY(&ifp->if_snd); 930#endif 931 932 ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd, 933 vtnet_ifmedia_sts); 934 ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL); 935 ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE); 936 937 /* Read (or generate) the MAC address for the adapter. */ 938 vtnet_get_hwaddr(sc); 939 940 ether_ifattach(ifp, sc->vtnet_hwaddr); 941 942 if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) 943 ifp->if_capabilities |= IFCAP_LINKSTATE; 944 945 /* Tell the upper layer(s) we support long frames. */ 946 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 947 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 948 949 if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { 950 ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6; 951 952 if (virtio_with_feature(dev, VIRTIO_NET_F_GSO)) { 953 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 954 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 955 } else { 956 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) 957 ifp->if_capabilities |= IFCAP_TSO4; 958 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 959 ifp->if_capabilities |= IFCAP_TSO6; 960 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) 961 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 962 } 963 964 if (ifp->if_capabilities & IFCAP_TSO) 965 ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 966 } 967 968 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) 969 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6; 970 971 if (ifp->if_capabilities & IFCAP_HWCSUM) { 972 /* 973 * VirtIO does not support VLAN tagging, but we can fake 974 * it by inserting and removing the 802.1Q header during 975 * transmit and receive. We are then able to do checksum 976 * offloading of VLAN frames. 977 */ 978 ifp->if_capabilities |= 979 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 980 } 981 982 ifp->if_capenable = ifp->if_capabilities; 983 984 /* 985 * Capabilities after here are not enabled by default. 986 */ 987 988 if (ifp->if_capabilities & IFCAP_RXCSUM) { 989 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) || 990 virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6)) 991 ifp->if_capabilities |= IFCAP_LRO; 992 } 993 994 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 995 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 996 997 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 998 vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 999 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 1000 vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1001 } 1002 1003 limit = vtnet_tunable_int(sc, "rx_process_limit", 1004 vtnet_rx_process_limit); 1005 if (limit < 0) 1006 limit = INT_MAX; 1007 sc->vtnet_rx_process_limit = limit; 1008 1009 return (0); 1010} 1011 1012static int 1013vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu) 1014{ 1015 struct ifnet *ifp; 1016 int frame_size, clsize; 1017 1018 ifp = sc->vtnet_ifp; 1019 1020 if (new_mtu < ETHERMIN || new_mtu > VTNET_MAX_MTU) 1021 return (EINVAL); 1022 1023 frame_size = sc->vtnet_hdr_size + sizeof(struct ether_vlan_header) + 1024 new_mtu; 1025 1026 /* 1027 * Based on the new MTU (and hence frame size) determine which 1028 * cluster size is most appropriate for the receive queues. 1029 */ 1030 if (frame_size <= MCLBYTES) { 1031 clsize = MCLBYTES; 1032 } else if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1033 /* Avoid going past 9K jumbos. */ 1034 if (frame_size > MJUM9BYTES) 1035 return (EINVAL); 1036 clsize = MJUM9BYTES; 1037 } else 1038 clsize = MJUMPAGESIZE; 1039 1040 ifp->if_mtu = new_mtu; 1041 sc->vtnet_rx_new_clsize = clsize; 1042 1043 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1044 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1045 vtnet_init_locked(sc); 1046 } 1047 1048 return (0); 1049} 1050 1051static int 1052vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1053{ 1054 struct vtnet_softc *sc; 1055 struct ifreq *ifr; 1056 int reinit, mask, error; 1057 1058 sc = ifp->if_softc; 1059 ifr = (struct ifreq *) data; 1060 error = 0; 1061 1062 switch (cmd) { 1063 case SIOCSIFMTU: 1064 if (ifp->if_mtu != ifr->ifr_mtu) { 1065 VTNET_CORE_LOCK(sc); 1066 error = vtnet_change_mtu(sc, ifr->ifr_mtu); 1067 VTNET_CORE_UNLOCK(sc); 1068 } 1069 break; 1070 1071 case SIOCSIFFLAGS: 1072 VTNET_CORE_LOCK(sc); 1073 if ((ifp->if_flags & IFF_UP) == 0) { 1074 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1075 vtnet_stop(sc); 1076 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1077 if ((ifp->if_flags ^ sc->vtnet_if_flags) & 1078 (IFF_PROMISC | IFF_ALLMULTI)) { 1079 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) 1080 vtnet_rx_filter(sc); 1081 else 1082 error = ENOTSUP; 1083 } 1084 } else 1085 vtnet_init_locked(sc); 1086 1087 if (error == 0) 1088 sc->vtnet_if_flags = ifp->if_flags; 1089 VTNET_CORE_UNLOCK(sc); 1090 break; 1091 1092 case SIOCADDMULTI: 1093 case SIOCDELMULTI: 1094 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) 1095 break; 1096 VTNET_CORE_LOCK(sc); 1097 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1098 vtnet_rx_filter_mac(sc); 1099 VTNET_CORE_UNLOCK(sc); 1100 break; 1101 1102 case SIOCSIFMEDIA: 1103 case SIOCGIFMEDIA: 1104 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); 1105 break; 1106 1107 case SIOCSIFCAP: 1108 VTNET_CORE_LOCK(sc); 1109 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1110 1111 if (mask & IFCAP_TXCSUM) 1112 ifp->if_capenable ^= IFCAP_TXCSUM; 1113 if (mask & IFCAP_TXCSUM_IPV6) 1114 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1115 if (mask & IFCAP_TSO4) 1116 ifp->if_capenable ^= IFCAP_TSO4; 1117 if (mask & IFCAP_TSO6) 1118 ifp->if_capenable ^= IFCAP_TSO6; 1119 1120 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 1121 IFCAP_VLAN_HWFILTER)) { 1122 /* These Rx features require us to renegotiate. */ 1123 reinit = 1; 1124 1125 if (mask & IFCAP_RXCSUM) 1126 ifp->if_capenable ^= IFCAP_RXCSUM; 1127 if (mask & IFCAP_RXCSUM_IPV6) 1128 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1129 if (mask & IFCAP_LRO) 1130 ifp->if_capenable ^= IFCAP_LRO; 1131 if (mask & IFCAP_VLAN_HWFILTER) 1132 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1133 } else 1134 reinit = 0; 1135 1136 if (mask & IFCAP_VLAN_HWTSO) 1137 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1138 if (mask & IFCAP_VLAN_HWTAGGING) 1139 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1140 1141 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1142 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1143 vtnet_init_locked(sc); 1144 } 1145 1146 VTNET_CORE_UNLOCK(sc); 1147 VLAN_CAPABILITIES(ifp); 1148 1149 break; 1150 1151 default: 1152 error = ether_ioctl(ifp, cmd, data); 1153 break; 1154 } 1155 1156 VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc); 1157 1158 return (error); 1159} 1160 1161static int 1162vtnet_rxq_populate(struct vtnet_rxq *rxq) 1163{ 1164 struct virtqueue *vq; 1165 int nbufs, error; 1166 1167 vq = rxq->vtnrx_vq; 1168 error = ENOSPC; 1169 1170 for (nbufs = 0; !virtqueue_full(vq); nbufs++) { 1171 error = vtnet_rxq_new_buf(rxq); 1172 if (error) 1173 break; 1174 } 1175 1176 if (nbufs > 0) { 1177 virtqueue_notify(vq); 1178 /* 1179 * EMSGSIZE signifies the virtqueue did not have enough 1180 * entries available to hold the last mbuf. This is not 1181 * an error. 1182 */ 1183 if (error == EMSGSIZE) 1184 error = 0; 1185 } 1186 1187 return (error); 1188} 1189 1190static void 1191vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq) 1192{ 1193 struct virtqueue *vq; 1194 struct mbuf *m; 1195 int last; 1196 1197 vq = rxq->vtnrx_vq; 1198 last = 0; 1199 1200 while ((m = virtqueue_drain(vq, &last)) != NULL) 1201 m_freem(m); 1202 1203 KASSERT(virtqueue_empty(vq), 1204 ("%s: mbufs remaining in rx queue %p", __func__, rxq)); 1205} 1206 1207static struct mbuf * 1208vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) 1209{ 1210 struct mbuf *m_head, *m_tail, *m; 1211 int i, clsize; 1212 1213 clsize = sc->vtnet_rx_clsize; 1214 1215 KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1216 ("%s: chained mbuf %d request without LRO_NOMRG", __func__, nbufs)); 1217 1218 m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize); 1219 if (m_head == NULL) 1220 goto fail; 1221 1222 m_head->m_len = clsize; 1223 m_tail = m_head; 1224 1225 /* Allocate the rest of the chain. */ 1226 for (i = 1; i < nbufs; i++) { 1227 m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize); 1228 if (m == NULL) 1229 goto fail; 1230 1231 m->m_len = clsize; 1232 m_tail->m_next = m; 1233 m_tail = m; 1234 } 1235 1236 if (m_tailp != NULL) 1237 *m_tailp = m_tail; 1238 1239 return (m_head); 1240 1241fail: 1242 sc->vtnet_stats.mbuf_alloc_failed++; 1243 m_freem(m_head); 1244 1245 return (NULL); 1246} 1247 1248/* 1249 * Slow path for when LRO without mergeable buffers is negotiated. 1250 */ 1251static int 1252vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0, 1253 int len0) 1254{ 1255 struct vtnet_softc *sc; 1256 struct mbuf *m, *m_prev; 1257 struct mbuf *m_new, *m_tail; 1258 int len, clsize, nreplace, error; 1259 1260 sc = rxq->vtnrx_sc; 1261 clsize = sc->vtnet_rx_clsize; 1262 1263 m_prev = NULL; 1264 m_tail = NULL; 1265 nreplace = 0; 1266 1267 m = m0; 1268 len = len0; 1269 1270 /* 1271 * Since these mbuf chains are so large, we avoid allocating an 1272 * entire replacement chain if possible. When the received frame 1273 * did not consume the entire chain, the unused mbufs are moved 1274 * to the replacement chain. 1275 */ 1276 while (len > 0) { 1277 /* 1278 * Something is seriously wrong if we received a frame 1279 * larger than the chain. Drop it. 1280 */ 1281 if (m == NULL) { 1282 sc->vtnet_stats.rx_frame_too_large++; 1283 return (EMSGSIZE); 1284 } 1285 1286 /* We always allocate the same cluster size. */ 1287 KASSERT(m->m_len == clsize, 1288 ("%s: mbuf size %d is not the cluster size %d", 1289 __func__, m->m_len, clsize)); 1290 1291 m->m_len = MIN(m->m_len, len); 1292 len -= m->m_len; 1293 1294 m_prev = m; 1295 m = m->m_next; 1296 nreplace++; 1297 } 1298 1299 KASSERT(nreplace <= sc->vtnet_rx_nmbufs, 1300 ("%s: too many replacement mbufs %d max %d", __func__, nreplace, 1301 sc->vtnet_rx_nmbufs)); 1302 1303 m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail); 1304 if (m_new == NULL) { 1305 m_prev->m_len = clsize; 1306 return (ENOBUFS); 1307 } 1308 1309 /* 1310 * Move any unused mbufs from the received chain onto the end 1311 * of the new chain. 1312 */ 1313 if (m_prev->m_next != NULL) { 1314 m_tail->m_next = m_prev->m_next; 1315 m_prev->m_next = NULL; 1316 } 1317 1318 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1319 if (error) { 1320 /* 1321 * BAD! We could not enqueue the replacement mbuf chain. We 1322 * must restore the m0 chain to the original state if it was 1323 * modified so we can subsequently discard it. 1324 * 1325 * NOTE: The replacement is suppose to be an identical copy 1326 * to the one just dequeued so this is an unexpected error. 1327 */ 1328 sc->vtnet_stats.rx_enq_replacement_failed++; 1329 1330 if (m_tail->m_next != NULL) { 1331 m_prev->m_next = m_tail->m_next; 1332 m_tail->m_next = NULL; 1333 } 1334 1335 m_prev->m_len = clsize; 1336 m_freem(m_new); 1337 } 1338 1339 return (error); 1340} 1341 1342static int 1343vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len) 1344{ 1345 struct vtnet_softc *sc; 1346 struct mbuf *m_new; 1347 int error; 1348 1349 sc = rxq->vtnrx_sc; 1350 1351 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1352 ("%s: chained mbuf without LRO_NOMRG", __func__)); 1353 1354 if (m->m_next == NULL) { 1355 /* Fast-path for the common case of just one mbuf. */ 1356 if (m->m_len < len) 1357 return (EINVAL); 1358 1359 m_new = vtnet_rx_alloc_buf(sc, 1, NULL); 1360 if (m_new == NULL) 1361 return (ENOBUFS); 1362 1363 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1364 if (error) { 1365 /* 1366 * The new mbuf is suppose to be an identical 1367 * copy of the one just dequeued so this is an 1368 * unexpected error. 1369 */ 1370 m_freem(m_new); 1371 sc->vtnet_stats.rx_enq_replacement_failed++; 1372 } else 1373 m->m_len = len; 1374 } else 1375 error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len); 1376 1377 return (error); 1378} 1379 1380static int 1381vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1382{ 1383 struct vtnet_softc *sc; 1384 struct sglist *sg; 1385 struct vtnet_rx_header *rxhdr; 1386 uint8_t *mdata; 1387 int offset, error; 1388 1389 sc = rxq->vtnrx_sc; 1390 sg = rxq->vtnrx_sg; 1391 mdata = mtod(m, uint8_t *); 1392 1393 VTNET_RXQ_LOCK_ASSERT(rxq); 1394 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL, 1395 ("%s: chained mbuf without LRO_NOMRG", __func__)); 1396 KASSERT(m->m_len == sc->vtnet_rx_clsize, 1397 ("%s: unexpected cluster size %d/%d", __func__, m->m_len, 1398 sc->vtnet_rx_clsize)); 1399 1400 sglist_reset(sg); 1401 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1402 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); 1403 rxhdr = (struct vtnet_rx_header *) mdata; 1404 sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size); 1405 offset = sizeof(struct vtnet_rx_header); 1406 } else 1407 offset = 0; 1408 1409 sglist_append(sg, mdata + offset, m->m_len - offset); 1410 if (m->m_next != NULL) { 1411 error = sglist_append_mbuf(sg, m->m_next); 1412 MPASS(error == 0); 1413 } 1414 1415 error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg); 1416 1417 return (error); 1418} 1419 1420static int 1421vtnet_rxq_new_buf(struct vtnet_rxq *rxq) 1422{ 1423 struct vtnet_softc *sc; 1424 struct mbuf *m; 1425 int error; 1426 1427 sc = rxq->vtnrx_sc; 1428 1429 m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); 1430 if (m == NULL) 1431 return (ENOBUFS); 1432 1433 error = vtnet_rxq_enqueue_buf(rxq, m); 1434 if (error) 1435 m_freem(m); 1436 1437 return (error); 1438} 1439 1440/* 1441 * Use the checksum offset in the VirtIO header to set the 1442 * correct CSUM_* flags. 1443 */ 1444static int 1445vtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m, 1446 uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1447{ 1448 struct vtnet_softc *sc; 1449#if defined(INET) || defined(INET6) 1450 int offset = hdr->csum_start + hdr->csum_offset; 1451#endif 1452 1453 sc = rxq->vtnrx_sc; 1454 1455 /* Only do a basic sanity check on the offset. */ 1456 switch (eth_type) { 1457#if defined(INET) 1458 case ETHERTYPE_IP: 1459 if (__predict_false(offset < ip_start + sizeof(struct ip))) 1460 return (1); 1461 break; 1462#endif 1463#if defined(INET6) 1464 case ETHERTYPE_IPV6: 1465 if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr))) 1466 return (1); 1467 break; 1468#endif 1469 default: 1470 sc->vtnet_stats.rx_csum_bad_ethtype++; 1471 return (1); 1472 } 1473 1474 /* 1475 * Use the offset to determine the appropriate CSUM_* flags. This is 1476 * a bit dirty, but we can get by with it since the checksum offsets 1477 * happen to be different. We assume the host host does not do IPv4 1478 * header checksum offloading. 1479 */ 1480 switch (hdr->csum_offset) { 1481 case offsetof(struct udphdr, uh_sum): 1482 case offsetof(struct tcphdr, th_sum): 1483 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1484 m->m_pkthdr.csum_data = 0xFFFF; 1485 break; 1486 case offsetof(struct sctphdr, checksum): 1487 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1488 break; 1489 default: 1490 sc->vtnet_stats.rx_csum_bad_offset++; 1491 return (1); 1492 } 1493 1494 return (0); 1495} 1496 1497static int 1498vtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m, 1499 uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr) 1500{ 1501 struct vtnet_softc *sc; 1502 int offset, proto; 1503 1504 sc = rxq->vtnrx_sc; 1505 1506 switch (eth_type) { 1507#if defined(INET) 1508 case ETHERTYPE_IP: { 1509 struct ip *ip; 1510 if (__predict_false(m->m_len < ip_start + sizeof(struct ip))) 1511 return (1); 1512 ip = (struct ip *)(m->m_data + ip_start); 1513 proto = ip->ip_p; 1514 offset = ip_start + (ip->ip_hl << 2); 1515 break; 1516 } 1517#endif 1518#if defined(INET6) 1519 case ETHERTYPE_IPV6: 1520 if (__predict_false(m->m_len < ip_start + 1521 sizeof(struct ip6_hdr))) 1522 return (1); 1523 offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto); 1524 if (__predict_false(offset < 0)) 1525 return (1); 1526 break; 1527#endif 1528 default: 1529 sc->vtnet_stats.rx_csum_bad_ethtype++; 1530 return (1); 1531 } 1532 1533 switch (proto) { 1534 case IPPROTO_TCP: 1535 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) 1536 return (1); 1537 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1538 m->m_pkthdr.csum_data = 0xFFFF; 1539 break; 1540 case IPPROTO_UDP: 1541 if (__predict_false(m->m_len < offset + sizeof(struct udphdr))) 1542 return (1); 1543 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1544 m->m_pkthdr.csum_data = 0xFFFF; 1545 break; 1546 case IPPROTO_SCTP: 1547 if (__predict_false(m->m_len < offset + sizeof(struct sctphdr))) 1548 return (1); 1549 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1550 break; 1551 default: 1552 /* 1553 * For the remaining protocols, FreeBSD does not support 1554 * checksum offloading, so the checksum will be recomputed. 1555 */ 1556#if 0 1557 if_printf(sc->vtnet_ifp, "cksum offload of unsupported " 1558 "protocol eth_type=%#x proto=%d csum_start=%d " 1559 "csum_offset=%d\n", __func__, eth_type, proto, 1560 hdr->csum_start, hdr->csum_offset); 1561#endif 1562 break; 1563 } 1564 1565 return (0); 1566} 1567 1568/* 1569 * Set the appropriate CSUM_* flags. Unfortunately, the information 1570 * provided is not directly useful to us. The VirtIO header gives the 1571 * offset of the checksum, which is all Linux needs, but this is not 1572 * how FreeBSD does things. We are forced to peek inside the packet 1573 * a bit. 1574 * 1575 * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD 1576 * could accept the offsets and let the stack figure it out. 1577 */ 1578static int 1579vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, 1580 struct virtio_net_hdr *hdr) 1581{ 1582 struct ether_header *eh; 1583 struct ether_vlan_header *evh; 1584 uint16_t eth_type; 1585 int offset, error; 1586 1587 eh = mtod(m, struct ether_header *); 1588 eth_type = ntohs(eh->ether_type); 1589 if (eth_type == ETHERTYPE_VLAN) { 1590 /* BMV: We should handle nested VLAN tags too. */ 1591 evh = mtod(m, struct ether_vlan_header *); 1592 eth_type = ntohs(evh->evl_proto); 1593 offset = sizeof(struct ether_vlan_header); 1594 } else 1595 offset = sizeof(struct ether_header); 1596 1597 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1598 error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr); 1599 else 1600 error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr); 1601 1602 return (error); 1603} 1604 1605static void 1606vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) 1607{ 1608 struct mbuf *m; 1609 1610 while (--nbufs > 0) { 1611 m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); 1612 if (m == NULL) 1613 break; 1614 vtnet_rxq_discard_buf(rxq, m); 1615 } 1616} 1617 1618static void 1619vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1620{ 1621 int error; 1622 1623 /* 1624 * Requeue the discarded mbuf. This should always be successful 1625 * since it was just dequeued. 1626 */ 1627 error = vtnet_rxq_enqueue_buf(rxq, m); 1628 KASSERT(error == 0, 1629 ("%s: cannot requeue discarded mbuf %d", __func__, error)); 1630} 1631 1632static int 1633vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs) 1634{ 1635 struct vtnet_softc *sc; 1636 struct ifnet *ifp; 1637 struct virtqueue *vq; 1638 struct mbuf *m, *m_tail; 1639 int len; 1640 1641 sc = rxq->vtnrx_sc; 1642 vq = rxq->vtnrx_vq; 1643 ifp = sc->vtnet_ifp; 1644 m_tail = m_head; 1645 1646 while (--nbufs > 0) { 1647 m = virtqueue_dequeue(vq, &len); 1648 if (m == NULL) { 1649 rxq->vtnrx_stats.vrxs_ierrors++; 1650 goto fail; 1651 } 1652 1653 if (vtnet_rxq_new_buf(rxq) != 0) { 1654 rxq->vtnrx_stats.vrxs_iqdrops++; 1655 vtnet_rxq_discard_buf(rxq, m); 1656 if (nbufs > 1) 1657 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1658 goto fail; 1659 } 1660 1661 if (m->m_len < len) 1662 len = m->m_len; 1663 1664 m->m_len = len; 1665 m->m_flags &= ~M_PKTHDR; 1666 1667 m_head->m_pkthdr.len += len; 1668 m_tail->m_next = m; 1669 m_tail = m; 1670 } 1671 1672 return (0); 1673 1674fail: 1675 sc->vtnet_stats.rx_mergeable_failed++; 1676 m_freem(m_head); 1677 1678 return (1); 1679} 1680 1681static void 1682vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, 1683 struct virtio_net_hdr *hdr) 1684{ 1685 struct vtnet_softc *sc; 1686 struct ifnet *ifp; 1687 struct ether_header *eh; 1688 1689 sc = rxq->vtnrx_sc; 1690 ifp = sc->vtnet_ifp; 1691 1692 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1693 eh = mtod(m, struct ether_header *); 1694 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1695 vtnet_vlan_tag_remove(m); 1696 /* 1697 * With the 802.1Q header removed, update the 1698 * checksum starting location accordingly. 1699 */ 1700 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1701 hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; 1702 } 1703 } 1704 1705 m->m_pkthdr.flowid = rxq->vtnrx_id; 1706 m->m_flags |= M_FLOWID; 1707 1708 /* 1709 * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum 1710 * distinction that Linux does. Need to reevaluate if performing 1711 * offloading for the NEEDS_CSUM case is really appropriate. 1712 */ 1713 if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | 1714 VIRTIO_NET_HDR_F_DATA_VALID)) { 1715 if (vtnet_rxq_csum(rxq, m, hdr) == 0) 1716 rxq->vtnrx_stats.vrxs_csum++; 1717 else 1718 rxq->vtnrx_stats.vrxs_csum_failed++; 1719 } 1720 1721 rxq->vtnrx_stats.vrxs_ipackets++; 1722 rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; 1723 1724 VTNET_RXQ_UNLOCK(rxq); 1725 (*ifp->if_input)(ifp, m); 1726 VTNET_RXQ_LOCK(rxq); 1727} 1728 1729static int 1730vtnet_rxq_eof(struct vtnet_rxq *rxq) 1731{ 1732 struct virtio_net_hdr lhdr, *hdr; 1733 struct vtnet_softc *sc; 1734 struct ifnet *ifp; 1735 struct virtqueue *vq; 1736 struct mbuf *m; 1737 struct virtio_net_hdr_mrg_rxbuf *mhdr; 1738 int len, deq, nbufs, adjsz, count; 1739 1740 sc = rxq->vtnrx_sc; 1741 vq = rxq->vtnrx_vq; 1742 ifp = sc->vtnet_ifp; 1743 hdr = &lhdr; 1744 deq = 0; 1745 count = sc->vtnet_rx_process_limit; 1746 1747 VTNET_RXQ_LOCK_ASSERT(rxq); 1748 1749 while (count-- > 0) { 1750 m = virtqueue_dequeue(vq, &len); 1751 if (m == NULL) 1752 break; 1753 deq++; 1754 1755 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { 1756 rxq->vtnrx_stats.vrxs_ierrors++; 1757 vtnet_rxq_discard_buf(rxq, m); 1758 continue; 1759 } 1760 1761 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1762 nbufs = 1; 1763 adjsz = sizeof(struct vtnet_rx_header); 1764 /* 1765 * Account for our pad inserted between the header 1766 * and the actual start of the frame. 1767 */ 1768 len += VTNET_RX_HEADER_PAD; 1769 } else { 1770 mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *); 1771 nbufs = mhdr->num_buffers; 1772 adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1773 } 1774 1775 if (vtnet_rxq_replace_buf(rxq, m, len) != 0) { 1776 rxq->vtnrx_stats.vrxs_iqdrops++; 1777 vtnet_rxq_discard_buf(rxq, m); 1778 if (nbufs > 1) 1779 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 1780 continue; 1781 } 1782 1783 m->m_pkthdr.len = len; 1784 m->m_pkthdr.rcvif = ifp; 1785 m->m_pkthdr.csum_flags = 0; 1786 1787 if (nbufs > 1) { 1788 /* Dequeue the rest of chain. */ 1789 if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0) 1790 continue; 1791 } 1792 1793 /* 1794 * Save copy of header before we strip it. For both mergeable 1795 * and non-mergeable, the header is at the beginning of the 1796 * mbuf data. We no longer need num_buffers, so always use a 1797 * regular header. 1798 * 1799 * BMV: Is this memcpy() expensive? We know the mbuf data is 1800 * still valid even after the m_adj(). 1801 */ 1802 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr)); 1803 m_adj(m, adjsz); 1804 1805 vtnet_rxq_input(rxq, m, hdr); 1806 1807 /* Must recheck after dropping the Rx lock. */ 1808 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1809 break; 1810 } 1811 1812 if (deq > 0) 1813 virtqueue_notify(vq); 1814 1815 return (count > 0 ? 0 : EAGAIN); 1816} 1817 1818static void 1819vtnet_rx_vq_intr(void *xrxq) 1820{ 1821 struct vtnet_softc *sc; 1822 struct vtnet_rxq *rxq; 1823 struct ifnet *ifp; 1824 int tries, more; 1825 1826 rxq = xrxq; 1827 sc = rxq->vtnrx_sc; 1828 ifp = sc->vtnet_ifp; 1829 tries = 0; 1830 1831 if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { 1832 /* 1833 * Ignore this interrupt. Either this is a spurious interrupt 1834 * or multiqueue without per-VQ MSIX so every queue needs to 1835 * be polled (a brain dead configuration we could try harder 1836 * to avoid). 1837 */ 1838 vtnet_rxq_disable_intr(rxq); 1839 return; 1840 } 1841 1842 VTNET_RXQ_LOCK(rxq); 1843 1844again: 1845 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1846 VTNET_RXQ_UNLOCK(rxq); 1847 return; 1848 } 1849 1850 more = vtnet_rxq_eof(rxq); 1851 if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1852 if (!more) 1853 vtnet_rxq_disable_intr(rxq); 1854 /* 1855 * This is an occasional condition or race (when !more), 1856 * so retry a few times before scheduling the taskqueue. 1857 */ 1858 if (tries++ < VTNET_INTR_DISABLE_RETRIES) 1859 goto again; 1860 1861 VTNET_RXQ_UNLOCK(rxq); 1862 rxq->vtnrx_stats.vrxs_rescheduled++; 1863 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1864 } else 1865 VTNET_RXQ_UNLOCK(rxq); 1866} 1867 1868static void 1869vtnet_rxq_tq_intr(void *xrxq, int pending) 1870{ 1871 struct vtnet_softc *sc; 1872 struct vtnet_rxq *rxq; 1873 struct ifnet *ifp; 1874 int more; 1875 1876 rxq = xrxq; 1877 sc = rxq->vtnrx_sc; 1878 ifp = sc->vtnet_ifp; 1879 1880 VTNET_RXQ_LOCK(rxq); 1881 1882 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1883 VTNET_RXQ_UNLOCK(rxq); 1884 return; 1885 } 1886 1887 more = vtnet_rxq_eof(rxq); 1888 if (more || vtnet_rxq_enable_intr(rxq) != 0) { 1889 if (!more) 1890 vtnet_rxq_disable_intr(rxq); 1891 rxq->vtnrx_stats.vrxs_rescheduled++; 1892 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 1893 } 1894 1895 VTNET_RXQ_UNLOCK(rxq); 1896} 1897 1898static void 1899vtnet_txq_free_mbufs(struct vtnet_txq *txq) 1900{ 1901 struct virtqueue *vq; 1902 struct vtnet_tx_header *txhdr; 1903 int last; 1904 1905 vq = txq->vtntx_vq; 1906 last = 0; 1907 1908 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 1909 m_freem(txhdr->vth_mbuf); 1910 uma_zfree(vtnet_tx_header_zone, txhdr); 1911 } 1912 1913 KASSERT(virtqueue_empty(vq), 1914 ("%s: mbufs remaining in tx queue %p", __func__, txq)); 1915} 1916 1917/* 1918 * BMV: Much of this can go away once we finally have offsets in 1919 * the mbuf packet header. Bug andre@. 1920 */ 1921static int 1922vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, 1923 int *etype, int *proto, int *start) 1924{ 1925 struct vtnet_softc *sc; 1926 struct ether_vlan_header *evh; 1927 int offset; 1928 1929 sc = txq->vtntx_sc; 1930 1931 evh = mtod(m, struct ether_vlan_header *); 1932 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1933 /* BMV: We should handle nested VLAN tags too. */ 1934 *etype = ntohs(evh->evl_proto); 1935 offset = sizeof(struct ether_vlan_header); 1936 } else { 1937 *etype = ntohs(evh->evl_encap_proto); 1938 offset = sizeof(struct ether_header); 1939 } 1940 1941 switch (*etype) { 1942#if defined(INET) 1943 case ETHERTYPE_IP: { 1944 struct ip *ip, iphdr; 1945 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 1946 m_copydata(m, offset, sizeof(struct ip), 1947 (caddr_t) &iphdr); 1948 ip = &iphdr; 1949 } else 1950 ip = (struct ip *)(m->m_data + offset); 1951 *proto = ip->ip_p; 1952 *start = offset + (ip->ip_hl << 2); 1953 break; 1954 } 1955#endif 1956#if defined(INET6) 1957 case ETHERTYPE_IPV6: 1958 *proto = -1; 1959 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 1960 /* Assert the network stack sent us a valid packet. */ 1961 KASSERT(*start > offset, 1962 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 1963 *start, offset, *proto)); 1964 break; 1965#endif 1966 default: 1967 sc->vtnet_stats.tx_csum_bad_ethtype++; 1968 return (EINVAL); 1969 } 1970 1971 return (0); 1972} 1973 1974static int 1975vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, 1976 int offset, struct virtio_net_hdr *hdr) 1977{ 1978 static struct timeval lastecn; 1979 static int curecn; 1980 struct vtnet_softc *sc; 1981 struct tcphdr *tcp, tcphdr; 1982 1983 sc = txq->vtntx_sc; 1984 1985 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { 1986 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); 1987 tcp = &tcphdr; 1988 } else 1989 tcp = (struct tcphdr *)(m->m_data + offset); 1990 1991 hdr->hdr_len = offset + (tcp->th_off << 2); 1992 hdr->gso_size = m->m_pkthdr.tso_segsz; 1993 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : 1994 VIRTIO_NET_HDR_GSO_TCPV6; 1995 1996 if (tcp->th_flags & TH_CWR) { 1997 /* 1998 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD, 1999 * ECN support is not on a per-interface basis, but globally via 2000 * the net.inet.tcp.ecn.enable sysctl knob. The default is off. 2001 */ 2002 if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { 2003 if (ppsratecheck(&lastecn, &curecn, 1)) 2004 if_printf(sc->vtnet_ifp, 2005 "TSO with ECN not negotiated with host\n"); 2006 return (ENOTSUP); 2007 } 2008 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 2009 } 2010 2011 txq->vtntx_stats.vtxs_tso++; 2012 2013 return (0); 2014} 2015 2016static struct mbuf * 2017vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, 2018 struct virtio_net_hdr *hdr) 2019{ 2020 struct vtnet_softc *sc; 2021 int flags, etype, csum_start, proto, error; 2022 2023 sc = txq->vtntx_sc; 2024 flags = m->m_pkthdr.csum_flags; 2025 2026 error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); 2027 if (error) 2028 goto drop; 2029 2030 if ((etype == ETHERTYPE_IP && flags & VTNET_CSUM_OFFLOAD) || 2031 (etype == ETHERTYPE_IPV6 && flags & VTNET_CSUM_OFFLOAD_IPV6)) { 2032 /* 2033 * We could compare the IP protocol vs the CSUM_ flag too, 2034 * but that really should not be necessary. 2035 */ 2036 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; 2037 hdr->csum_start = csum_start; 2038 hdr->csum_offset = m->m_pkthdr.csum_data; 2039 txq->vtntx_stats.vtxs_csum++; 2040 } 2041 2042 if (flags & CSUM_TSO) { 2043 if (__predict_false(proto != IPPROTO_TCP)) { 2044 /* Likely failed to correctly parse the mbuf. */ 2045 sc->vtnet_stats.tx_tso_not_tcp++; 2046 goto drop; 2047 } 2048 2049 KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM, 2050 ("%s: mbuf %p TSO without checksum offload %#x", 2051 __func__, m, flags)); 2052 2053 error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); 2054 if (error) 2055 goto drop; 2056 } 2057 2058 return (m); 2059 2060drop: 2061 m_freem(m); 2062 return (NULL); 2063} 2064 2065static int 2066vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, 2067 struct vtnet_tx_header *txhdr) 2068{ 2069 struct vtnet_softc *sc; 2070 struct virtqueue *vq; 2071 struct sglist *sg; 2072 struct mbuf *m; 2073 int error; 2074 2075 sc = txq->vtntx_sc; 2076 vq = txq->vtntx_vq; 2077 sg = txq->vtntx_sg; 2078 m = *m_head; 2079 2080 sglist_reset(sg); 2081 error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); 2082 KASSERT(error == 0 && sg->sg_nseg == 1, 2083 ("%s: error %d adding header to sglist", __func__, error)); 2084 2085 error = sglist_append_mbuf(sg, m); 2086 if (error) { 2087 m = m_defrag(m, M_NOWAIT); 2088 if (m == NULL) 2089 goto fail; 2090 2091 *m_head = m; 2092 sc->vtnet_stats.tx_defragged++; 2093 2094 error = sglist_append_mbuf(sg, m); 2095 if (error) 2096 goto fail; 2097 } 2098 2099 txhdr->vth_mbuf = m; 2100 error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); 2101 2102 return (error); 2103 2104fail: 2105 sc->vtnet_stats.tx_defrag_failed++; 2106 m_freem(*m_head); 2107 *m_head = NULL; 2108 2109 return (ENOBUFS); 2110} 2111 2112static int 2113vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head) 2114{
|
2114 struct vtnet_softc *sc;
| |
2115 struct vtnet_tx_header *txhdr; 2116 struct virtio_net_hdr *hdr; 2117 struct mbuf *m; 2118 int error; 2119
| 2115 struct vtnet_tx_header *txhdr; 2116 struct virtio_net_hdr *hdr; 2117 struct mbuf *m; 2118 int error; 2119
|
2120 sc = txq->vtntx_sc;
| |
2121 m = *m_head; 2122 M_ASSERTPKTHDR(m); 2123 2124 txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO); 2125 if (txhdr == NULL) { 2126 m_freem(m); 2127 *m_head = NULL; 2128 return (ENOMEM); 2129 } 2130 2131 /* 2132 * Always use the non-mergeable header, regardless if the feature 2133 * was negotiated. For transmit, num_buffers is always zero. The 2134 * vtnet_hdr_size is used to enqueue the correct header size. 2135 */ 2136 hdr = &txhdr->vth_uhdr.hdr; 2137 2138 if (m->m_flags & M_VLANTAG) { 2139 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 2140 if ((*m_head = m) == NULL) { 2141 error = ENOBUFS; 2142 goto fail; 2143 } 2144 m->m_flags &= ~M_VLANTAG; 2145 } 2146 2147 if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { 2148 m = vtnet_txq_offload(txq, m, hdr); 2149 if ((*m_head = m) == NULL) { 2150 error = ENOBUFS; 2151 goto fail; 2152 } 2153 } 2154 2155 error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); 2156 if (error == 0) 2157 return (0); 2158 2159fail: 2160 uma_zfree(vtnet_tx_header_zone, txhdr); 2161 2162 return (error); 2163} 2164 2165#ifdef VTNET_LEGACY_TX 2166 2167static void 2168vtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp) 2169{ 2170 struct vtnet_softc *sc; 2171 struct virtqueue *vq; 2172 struct mbuf *m0; 2173 int enq; 2174 2175 sc = txq->vtntx_sc; 2176 vq = txq->vtntx_vq; 2177 enq = 0; 2178 2179 VTNET_TXQ_LOCK_ASSERT(txq); 2180 2181 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2182 sc->vtnet_link_active == 0) 2183 return; 2184 2185 vtnet_txq_eof(txq); 2186 2187 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2188 if (virtqueue_full(vq)) 2189 break; 2190 2191 IFQ_DRV_DEQUEUE(&ifp->if_snd, m0); 2192 if (m0 == NULL) 2193 break; 2194 2195 if (vtnet_txq_encap(txq, &m0) != 0) { 2196 if (m0 != NULL) 2197 IFQ_DRV_PREPEND(&ifp->if_snd, m0); 2198 break; 2199 } 2200 2201 enq++; 2202 ETHER_BPF_MTAP(ifp, m0); 2203 } 2204 2205 if (enq > 0) { 2206 virtqueue_notify(vq); 2207 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 2208 } 2209} 2210 2211static void 2212vtnet_start(struct ifnet *ifp) 2213{ 2214 struct vtnet_softc *sc; 2215 struct vtnet_txq *txq; 2216 2217 sc = ifp->if_softc; 2218 txq = &sc->vtnet_txqs[0]; 2219 2220 VTNET_TXQ_LOCK(txq); 2221 vtnet_start_locked(txq, ifp); 2222 VTNET_TXQ_UNLOCK(txq); 2223} 2224 2225#else /* !VTNET_LEGACY_TX */ 2226 2227static int 2228vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m) 2229{ 2230 struct vtnet_softc *sc; 2231 struct virtqueue *vq; 2232 struct buf_ring *br; 2233 struct ifnet *ifp; 2234 int enq, error; 2235 2236 sc = txq->vtntx_sc; 2237 vq = txq->vtntx_vq; 2238 br = txq->vtntx_br; 2239 ifp = sc->vtnet_ifp; 2240 enq = 0; 2241 error = 0; 2242 2243 VTNET_TXQ_LOCK_ASSERT(txq); 2244 2245 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2246 sc->vtnet_link_active == 0) { 2247 if (m != NULL) 2248 error = drbr_enqueue(ifp, br, m); 2249 return (error); 2250 } 2251 2252 if (m != NULL) { 2253 error = drbr_enqueue(ifp, br, m); 2254 if (error) 2255 return (error); 2256 } 2257 2258 vtnet_txq_eof(txq); 2259 2260 while ((m = drbr_peek(ifp, br)) != NULL) { 2261 if (virtqueue_full(vq)) { 2262 drbr_putback(ifp, br, m); 2263 break; 2264 } 2265 2266 error = vtnet_txq_encap(txq, &m); 2267 if (error) { 2268 if (m != NULL) 2269 drbr_putback(ifp, br, m); 2270 else 2271 drbr_advance(ifp, br); 2272 break; 2273 } 2274 drbr_advance(ifp, br); 2275 2276 enq++; 2277 ETHER_BPF_MTAP(ifp, m); 2278 } 2279 2280 if (enq > 0) { 2281 virtqueue_notify(vq); 2282 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 2283 } 2284 2285 return (0); 2286} 2287 2288static int 2289vtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2290{ 2291 struct vtnet_softc *sc; 2292 struct vtnet_txq *txq; 2293 int i, npairs, error; 2294 2295 sc = ifp->if_softc; 2296 npairs = sc->vtnet_act_vq_pairs; 2297 2298 if (m->m_flags & M_FLOWID) 2299 i = m->m_pkthdr.flowid % npairs; 2300 else 2301 i = curcpu % npairs; 2302 2303 txq = &sc->vtnet_txqs[i]; 2304 2305 if (VTNET_TXQ_TRYLOCK(txq) != 0) { 2306 error = vtnet_txq_mq_start_locked(txq, m); 2307 VTNET_TXQ_UNLOCK(txq); 2308 } else { 2309 error = drbr_enqueue(ifp, txq->vtntx_br, m); 2310 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); 2311 } 2312 2313 return (error); 2314} 2315 2316static void 2317vtnet_txq_tq_deferred(void *xtxq, int pending) 2318{ 2319 struct vtnet_softc *sc; 2320 struct vtnet_txq *txq; 2321 2322 txq = xtxq; 2323 sc = txq->vtntx_sc; 2324 2325 VTNET_TXQ_LOCK(txq); 2326 if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) 2327 vtnet_txq_mq_start_locked(txq, NULL); 2328 VTNET_TXQ_UNLOCK(txq); 2329} 2330 2331#endif /* VTNET_LEGACY_TX */ 2332 2333static void 2334vtnet_txq_start(struct vtnet_txq *txq) 2335{ 2336 struct vtnet_softc *sc; 2337 struct ifnet *ifp; 2338 2339 sc = txq->vtntx_sc; 2340 ifp = sc->vtnet_ifp; 2341 2342#ifdef VTNET_LEGACY_TX 2343 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 2344 vtnet_start_locked(txq, ifp); 2345#else 2346 if (!drbr_empty(ifp, txq->vtntx_br)) 2347 vtnet_txq_mq_start_locked(txq, NULL); 2348#endif 2349} 2350 2351static void 2352vtnet_txq_tq_intr(void *xtxq, int pending) 2353{ 2354 struct vtnet_softc *sc; 2355 struct vtnet_txq *txq; 2356 struct ifnet *ifp; 2357 2358 txq = xtxq; 2359 sc = txq->vtntx_sc; 2360 ifp = sc->vtnet_ifp; 2361 2362 VTNET_TXQ_LOCK(txq); 2363 2364 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2365 VTNET_TXQ_UNLOCK(txq); 2366 return; 2367 } 2368 2369 vtnet_txq_eof(txq); 2370 2371 vtnet_txq_start(txq); 2372 2373 if (vtnet_txq_enable_intr(txq) != 0) { 2374 vtnet_txq_disable_intr(txq); 2375 txq->vtntx_stats.vtxs_rescheduled++; 2376 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2377 } 2378 2379 VTNET_TXQ_UNLOCK(txq); 2380} 2381 2382static void 2383vtnet_txq_eof(struct vtnet_txq *txq) 2384{ 2385 struct virtqueue *vq; 2386 struct vtnet_tx_header *txhdr; 2387 struct mbuf *m; 2388 2389 vq = txq->vtntx_vq; 2390 VTNET_TXQ_LOCK_ASSERT(txq); 2391 2392 while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { 2393 m = txhdr->vth_mbuf; 2394 2395 txq->vtntx_stats.vtxs_opackets++; 2396 txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; 2397 if (m->m_flags & M_MCAST) 2398 txq->vtntx_stats.vtxs_omcasts++; 2399 2400 m_freem(m); 2401 uma_zfree(vtnet_tx_header_zone, txhdr); 2402 } 2403 2404 if (virtqueue_empty(vq)) 2405 txq->vtntx_watchdog = 0; 2406} 2407 2408static void 2409vtnet_tx_vq_intr(void *xtxq) 2410{ 2411 struct vtnet_softc *sc; 2412 struct vtnet_txq *txq; 2413 struct ifnet *ifp; 2414 int tries; 2415 2416 txq = xtxq; 2417 sc = txq->vtntx_sc; 2418 ifp = sc->vtnet_ifp; 2419 tries = 0; 2420 2421 if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { 2422 /* 2423 * Ignore this interrupt. Either this is a spurious interrupt 2424 * or multiqueue without per-VQ MSIX so every queue needs to 2425 * be polled (a brain dead configuration we could try harder 2426 * to avoid). 2427 */ 2428 vtnet_txq_disable_intr(txq); 2429 return; 2430 } 2431 2432 VTNET_TXQ_LOCK(txq); 2433 2434again: 2435 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2436 VTNET_TXQ_UNLOCK(txq); 2437 return; 2438 } 2439 2440 vtnet_txq_eof(txq); 2441 2442 vtnet_txq_start(txq); 2443 2444 if (vtnet_txq_enable_intr(txq) != 0) { 2445 vtnet_txq_disable_intr(txq); 2446 /* 2447 * This is an occasional race, so retry a few times 2448 * before scheduling the taskqueue. 2449 */ 2450 if (tries++ < VTNET_INTR_DISABLE_RETRIES) 2451 goto again; 2452 2453 VTNET_TXQ_UNLOCK(txq); 2454 txq->vtntx_stats.vtxs_rescheduled++; 2455 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2456 } else 2457 VTNET_TXQ_UNLOCK(txq); 2458} 2459 2460static void 2461vtnet_tx_start_all(struct vtnet_softc *sc) 2462{ 2463 struct vtnet_txq *txq; 2464 int i; 2465 2466 VTNET_CORE_LOCK_ASSERT(sc); 2467 2468 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2469 txq = &sc->vtnet_txqs[i]; 2470 2471 VTNET_TXQ_LOCK(txq); 2472 vtnet_txq_start(txq); 2473 VTNET_TXQ_UNLOCK(txq); 2474 } 2475} 2476 2477#ifndef VTNET_LEGACY_TX 2478static void 2479vtnet_qflush(struct ifnet *ifp) 2480{ 2481 struct vtnet_softc *sc; 2482 struct vtnet_txq *txq; 2483 struct mbuf *m; 2484 int i; 2485 2486 sc = ifp->if_softc; 2487 2488 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2489 txq = &sc->vtnet_txqs[i]; 2490 2491 VTNET_TXQ_LOCK(txq); 2492 while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) 2493 m_freem(m); 2494 VTNET_TXQ_UNLOCK(txq); 2495 } 2496 2497 if_qflush(ifp); 2498} 2499#endif 2500 2501static int 2502vtnet_watchdog(struct vtnet_txq *txq) 2503{ 2504 struct vtnet_softc *sc; 2505 2506 sc = txq->vtntx_sc; 2507 2508 VTNET_TXQ_LOCK(txq); 2509 if (sc->vtnet_flags & VTNET_FLAG_EVENT_IDX) 2510 vtnet_txq_eof(txq); 2511 if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { 2512 VTNET_TXQ_UNLOCK(txq); 2513 return (0); 2514 } 2515 VTNET_TXQ_UNLOCK(txq); 2516 2517 if_printf(sc->vtnet_ifp, "watchdog timeout on queue %d\n", 2518 txq->vtntx_id); 2519 return (1); 2520} 2521 2522static void 2523vtnet_rxq_accum_stats(struct vtnet_rxq *rxq, struct vtnet_rxq_stats *accum) 2524{ 2525 struct vtnet_rxq_stats *st; 2526 2527 st = &rxq->vtnrx_stats; 2528 2529 accum->vrxs_ipackets += st->vrxs_ipackets; 2530 accum->vrxs_ibytes += st->vrxs_ibytes; 2531 accum->vrxs_iqdrops += st->vrxs_iqdrops; 2532 accum->vrxs_csum += st->vrxs_csum; 2533 accum->vrxs_csum_failed += st->vrxs_csum_failed; 2534 accum->vrxs_rescheduled += st->vrxs_rescheduled; 2535} 2536 2537static void 2538vtnet_txq_accum_stats(struct vtnet_txq *txq, struct vtnet_txq_stats *accum) 2539{ 2540 struct vtnet_txq_stats *st; 2541 2542 st = &txq->vtntx_stats; 2543 2544 accum->vtxs_opackets += st->vtxs_opackets; 2545 accum->vtxs_obytes += st->vtxs_obytes; 2546 accum->vtxs_csum += st->vtxs_csum; 2547 accum->vtxs_tso += st->vtxs_tso; 2548 accum->vtxs_rescheduled += st->vtxs_rescheduled; 2549} 2550 2551static void 2552vtnet_accumulate_stats(struct vtnet_softc *sc) 2553{ 2554 struct ifnet *ifp; 2555 struct vtnet_statistics *st; 2556 struct vtnet_rxq_stats rxaccum; 2557 struct vtnet_txq_stats txaccum; 2558 int i; 2559 2560 ifp = sc->vtnet_ifp; 2561 st = &sc->vtnet_stats; 2562 bzero(&rxaccum, sizeof(struct vtnet_rxq_stats)); 2563 bzero(&txaccum, sizeof(struct vtnet_txq_stats)); 2564 2565 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2566 vtnet_rxq_accum_stats(&sc->vtnet_rxqs[i], &rxaccum); 2567 vtnet_txq_accum_stats(&sc->vtnet_txqs[i], &txaccum); 2568 } 2569 2570 st->rx_csum_offloaded = rxaccum.vrxs_csum; 2571 st->rx_csum_failed = rxaccum.vrxs_csum_failed; 2572 st->rx_task_rescheduled = rxaccum.vrxs_rescheduled; 2573 st->tx_csum_offloaded = txaccum.vtxs_csum; 2574 st->tx_tso_offloaded = txaccum.vtxs_tso; 2575 st->tx_task_rescheduled = txaccum.vtxs_rescheduled; 2576 2577 /* 2578 * With the exception of if_ierrors, these ifnet statistics are 2579 * only updated in the driver, so just set them to our accumulated 2580 * values. if_ierrors is updated in ether_input() for malformed 2581 * frames that we should have already discarded. 2582 */ 2583 ifp->if_ipackets = rxaccum.vrxs_ipackets; 2584 ifp->if_iqdrops = rxaccum.vrxs_iqdrops; 2585 ifp->if_ierrors = rxaccum.vrxs_ierrors; 2586 ifp->if_opackets = txaccum.vtxs_opackets; 2587#ifndef VTNET_LEGACY_TX 2588 ifp->if_obytes = txaccum.vtxs_obytes; 2589 ifp->if_omcasts = txaccum.vtxs_omcasts; 2590#endif 2591} 2592 2593static void 2594vtnet_tick(void *xsc) 2595{ 2596 struct vtnet_softc *sc; 2597 struct ifnet *ifp; 2598 int i, timedout; 2599 2600 sc = xsc; 2601 ifp = sc->vtnet_ifp; 2602 timedout = 0; 2603 2604 VTNET_CORE_LOCK_ASSERT(sc); 2605 vtnet_accumulate_stats(sc); 2606 2607 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 2608 timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); 2609 2610 if (timedout != 0) { 2611 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2612 vtnet_init_locked(sc); 2613 } else 2614 callout_schedule(&sc->vtnet_tick_ch, hz); 2615} 2616 2617static void 2618vtnet_start_taskqueues(struct vtnet_softc *sc) 2619{ 2620 device_t dev; 2621 struct vtnet_rxq *rxq; 2622 struct vtnet_txq *txq; 2623 int i, error; 2624 2625 dev = sc->vtnet_dev; 2626 2627 /* 2628 * Errors here are very difficult to recover from - we cannot 2629 * easily fail because, if this is during boot, we will hang 2630 * when freeing any successfully started taskqueues because 2631 * the scheduler isn't up yet. 2632 * 2633 * Most drivers just ignore the return value - it only fails 2634 * with ENOMEM so an error is not likely. 2635 */ 2636 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2637 rxq = &sc->vtnet_rxqs[i]; 2638 error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, 2639 "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); 2640 if (error) { 2641 device_printf(dev, "failed to start rx taskq %d\n", 2642 rxq->vtnrx_id); 2643 } 2644 2645 txq = &sc->vtnet_txqs[i]; 2646 error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, 2647 "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); 2648 if (error) { 2649 device_printf(dev, "failed to start tx taskq %d\n", 2650 txq->vtntx_id); 2651 } 2652 } 2653} 2654 2655static void 2656vtnet_free_taskqueues(struct vtnet_softc *sc) 2657{ 2658 struct vtnet_rxq *rxq; 2659 struct vtnet_txq *txq; 2660 int i; 2661 2662 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2663 rxq = &sc->vtnet_rxqs[i]; 2664 if (rxq->vtnrx_tq != NULL) { 2665 taskqueue_free(rxq->vtnrx_tq); 2666 rxq->vtnrx_vq = NULL; 2667 } 2668 2669 txq = &sc->vtnet_txqs[i]; 2670 if (txq->vtntx_tq != NULL) { 2671 taskqueue_free(txq->vtntx_tq); 2672 txq->vtntx_tq = NULL; 2673 } 2674 } 2675} 2676 2677static void 2678vtnet_drain_taskqueues(struct vtnet_softc *sc) 2679{ 2680 struct vtnet_rxq *rxq; 2681 struct vtnet_txq *txq; 2682 int i; 2683 2684 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2685 rxq = &sc->vtnet_rxqs[i]; 2686 if (rxq->vtnrx_tq != NULL) 2687 taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 2688 2689 txq = &sc->vtnet_txqs[i]; 2690 if (txq->vtntx_tq != NULL) { 2691 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); 2692#ifndef VTNET_LEGACY_TX 2693 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); 2694#endif 2695 } 2696 } 2697} 2698 2699static void 2700vtnet_drain_rxtx_queues(struct vtnet_softc *sc) 2701{ 2702 struct vtnet_rxq *rxq; 2703 struct vtnet_txq *txq; 2704 int i; 2705 2706 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2707 rxq = &sc->vtnet_rxqs[i]; 2708 vtnet_rxq_free_mbufs(rxq); 2709 2710 txq = &sc->vtnet_txqs[i]; 2711 vtnet_txq_free_mbufs(txq); 2712 } 2713} 2714 2715static void 2716vtnet_stop_rendezvous(struct vtnet_softc *sc) 2717{ 2718 struct vtnet_rxq *rxq; 2719 struct vtnet_txq *txq; 2720 int i; 2721 2722 /* 2723 * Lock and unlock the per-queue mutex so we known the stop 2724 * state is visible. Doing only the active queues should be 2725 * sufficient, but it does not cost much extra to do all the 2726 * queues. Note we hold the core mutex here too. 2727 */ 2728 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2729 rxq = &sc->vtnet_rxqs[i]; 2730 VTNET_RXQ_LOCK(rxq); 2731 VTNET_RXQ_UNLOCK(rxq); 2732 2733 txq = &sc->vtnet_txqs[i]; 2734 VTNET_TXQ_LOCK(txq); 2735 VTNET_TXQ_UNLOCK(txq); 2736 } 2737} 2738 2739static void 2740vtnet_stop(struct vtnet_softc *sc) 2741{ 2742 device_t dev; 2743 struct ifnet *ifp; 2744 2745 dev = sc->vtnet_dev; 2746 ifp = sc->vtnet_ifp; 2747 2748 VTNET_CORE_LOCK_ASSERT(sc); 2749 2750 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2751 sc->vtnet_link_active = 0; 2752 callout_stop(&sc->vtnet_tick_ch); 2753 2754 /* Only advisory. */ 2755 vtnet_disable_interrupts(sc); 2756 2757 /* 2758 * Stop the host adapter. This resets it to the pre-initialized 2759 * state. It will not generate any interrupts until after it is 2760 * reinitialized. 2761 */ 2762 virtio_stop(dev); 2763 vtnet_stop_rendezvous(sc); 2764 2765 /* Free any mbufs left in the virtqueues. */ 2766 vtnet_drain_rxtx_queues(sc); 2767} 2768 2769static int 2770vtnet_virtio_reinit(struct vtnet_softc *sc) 2771{ 2772 device_t dev; 2773 struct ifnet *ifp; 2774 uint64_t features; 2775 int mask, error; 2776 2777 dev = sc->vtnet_dev; 2778 ifp = sc->vtnet_ifp; 2779 features = sc->vtnet_features; 2780 2781 mask = 0; 2782#if defined(INET) 2783 mask |= IFCAP_RXCSUM; 2784#endif 2785#if defined (INET6) 2786 mask |= IFCAP_RXCSUM_IPV6; 2787#endif 2788 2789 /* 2790 * Re-negotiate with the host, removing any disabled receive 2791 * features. Transmit features are disabled only on our side 2792 * via if_capenable and if_hwassist. 2793 */ 2794 2795 if (ifp->if_capabilities & mask) { 2796 /* 2797 * We require both IPv4 and IPv6 offloading to be enabled 2798 * in order to negotiated it: VirtIO does not distinguish 2799 * between the two. 2800 */ 2801 if ((ifp->if_capenable & mask) != mask) 2802 features &= ~VIRTIO_NET_F_GUEST_CSUM; 2803 } 2804 2805 if (ifp->if_capabilities & IFCAP_LRO) { 2806 if ((ifp->if_capenable & IFCAP_LRO) == 0) 2807 features &= ~VTNET_LRO_FEATURES; 2808 } 2809 2810 if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) { 2811 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 2812 features &= ~VIRTIO_NET_F_CTRL_VLAN; 2813 } 2814 2815 error = virtio_reinit(dev, features); 2816 if (error) 2817 device_printf(dev, "virtio reinit error %d\n", error); 2818 2819 return (error); 2820} 2821 2822static void 2823vtnet_init_rx_filters(struct vtnet_softc *sc) 2824{ 2825 struct ifnet *ifp; 2826 2827 ifp = sc->vtnet_ifp; 2828 2829 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 2830 /* Restore promiscuous and all-multicast modes. */ 2831 vtnet_rx_filter(sc); 2832 /* Restore filtered MAC addresses. */ 2833 vtnet_rx_filter_mac(sc); 2834 } 2835 2836 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2837 vtnet_rx_filter_vlan(sc); 2838} 2839 2840static int 2841vtnet_init_rx_queues(struct vtnet_softc *sc) 2842{ 2843 device_t dev; 2844 struct vtnet_rxq *rxq; 2845 int i, clsize, error; 2846 2847 dev = sc->vtnet_dev; 2848 2849 /* 2850 * Use the new cluster size if one has been set (via a MTU 2851 * change). Otherwise, use the standard 2K clusters. 2852 * 2853 * BMV: It might make sense to use page sized clusters as 2854 * the default (depending on the features negotiated). 2855 */ 2856 if (sc->vtnet_rx_new_clsize != 0) { 2857 clsize = sc->vtnet_rx_new_clsize; 2858 sc->vtnet_rx_new_clsize = 0; 2859 } else 2860 clsize = MCLBYTES; 2861 2862 sc->vtnet_rx_clsize = clsize; 2863 sc->vtnet_rx_nmbufs = VTNET_NEEDED_RX_MBUFS(sc, clsize); 2864 2865 KASSERT(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS || 2866 sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, 2867 ("%s: too many rx mbufs %d for %d segments", __func__, 2868 sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); 2869 2870 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2871 rxq = &sc->vtnet_rxqs[i]; 2872 2873 /* Hold the lock to satisfy asserts. */ 2874 VTNET_RXQ_LOCK(rxq); 2875 error = vtnet_rxq_populate(rxq); 2876 VTNET_RXQ_UNLOCK(rxq); 2877 2878 if (error) { 2879 device_printf(dev, 2880 "cannot allocate mbufs for Rx queue %d\n", i); 2881 return (error); 2882 } 2883 } 2884 2885 return (0); 2886} 2887 2888static int 2889vtnet_init_tx_queues(struct vtnet_softc *sc) 2890{ 2891 struct vtnet_txq *txq; 2892 int i; 2893 2894 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2895 txq = &sc->vtnet_txqs[i]; 2896 txq->vtntx_watchdog = 0; 2897 } 2898 2899 return (0); 2900} 2901 2902static int 2903vtnet_init_rxtx_queues(struct vtnet_softc *sc) 2904{ 2905 int error; 2906 2907 error = vtnet_init_rx_queues(sc); 2908 if (error) 2909 return (error); 2910 2911 error = vtnet_init_tx_queues(sc); 2912 if (error) 2913 return (error); 2914 2915 return (0); 2916} 2917 2918static void 2919vtnet_set_active_vq_pairs(struct vtnet_softc *sc) 2920{ 2921 device_t dev; 2922 int npairs; 2923 2924 dev = sc->vtnet_dev; 2925 2926 if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) { 2927 MPASS(sc->vtnet_max_vq_pairs == 1); 2928 sc->vtnet_act_vq_pairs = 1; 2929 return; 2930 } 2931 2932 /* BMV: Just use the maximum configured for now. */ 2933 npairs = sc->vtnet_max_vq_pairs; 2934 2935 if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { 2936 device_printf(dev, 2937 "cannot set active queue pairs to %d\n", npairs); 2938 npairs = 1; 2939 } 2940 2941 sc->vtnet_act_vq_pairs = npairs; 2942} 2943 2944static int 2945vtnet_reinit(struct vtnet_softc *sc) 2946{
| 2120 m = *m_head; 2121 M_ASSERTPKTHDR(m); 2122 2123 txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO); 2124 if (txhdr == NULL) { 2125 m_freem(m); 2126 *m_head = NULL; 2127 return (ENOMEM); 2128 } 2129 2130 /* 2131 * Always use the non-mergeable header, regardless if the feature 2132 * was negotiated. For transmit, num_buffers is always zero. The 2133 * vtnet_hdr_size is used to enqueue the correct header size. 2134 */ 2135 hdr = &txhdr->vth_uhdr.hdr; 2136 2137 if (m->m_flags & M_VLANTAG) { 2138 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 2139 if ((*m_head = m) == NULL) { 2140 error = ENOBUFS; 2141 goto fail; 2142 } 2143 m->m_flags &= ~M_VLANTAG; 2144 } 2145 2146 if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { 2147 m = vtnet_txq_offload(txq, m, hdr); 2148 if ((*m_head = m) == NULL) { 2149 error = ENOBUFS; 2150 goto fail; 2151 } 2152 } 2153 2154 error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); 2155 if (error == 0) 2156 return (0); 2157 2158fail: 2159 uma_zfree(vtnet_tx_header_zone, txhdr); 2160 2161 return (error); 2162} 2163 2164#ifdef VTNET_LEGACY_TX 2165 2166static void 2167vtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp) 2168{ 2169 struct vtnet_softc *sc; 2170 struct virtqueue *vq; 2171 struct mbuf *m0; 2172 int enq; 2173 2174 sc = txq->vtntx_sc; 2175 vq = txq->vtntx_vq; 2176 enq = 0; 2177 2178 VTNET_TXQ_LOCK_ASSERT(txq); 2179 2180 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2181 sc->vtnet_link_active == 0) 2182 return; 2183 2184 vtnet_txq_eof(txq); 2185 2186 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2187 if (virtqueue_full(vq)) 2188 break; 2189 2190 IFQ_DRV_DEQUEUE(&ifp->if_snd, m0); 2191 if (m0 == NULL) 2192 break; 2193 2194 if (vtnet_txq_encap(txq, &m0) != 0) { 2195 if (m0 != NULL) 2196 IFQ_DRV_PREPEND(&ifp->if_snd, m0); 2197 break; 2198 } 2199 2200 enq++; 2201 ETHER_BPF_MTAP(ifp, m0); 2202 } 2203 2204 if (enq > 0) { 2205 virtqueue_notify(vq); 2206 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 2207 } 2208} 2209 2210static void 2211vtnet_start(struct ifnet *ifp) 2212{ 2213 struct vtnet_softc *sc; 2214 struct vtnet_txq *txq; 2215 2216 sc = ifp->if_softc; 2217 txq = &sc->vtnet_txqs[0]; 2218 2219 VTNET_TXQ_LOCK(txq); 2220 vtnet_start_locked(txq, ifp); 2221 VTNET_TXQ_UNLOCK(txq); 2222} 2223 2224#else /* !VTNET_LEGACY_TX */ 2225 2226static int 2227vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m) 2228{ 2229 struct vtnet_softc *sc; 2230 struct virtqueue *vq; 2231 struct buf_ring *br; 2232 struct ifnet *ifp; 2233 int enq, error; 2234 2235 sc = txq->vtntx_sc; 2236 vq = txq->vtntx_vq; 2237 br = txq->vtntx_br; 2238 ifp = sc->vtnet_ifp; 2239 enq = 0; 2240 error = 0; 2241 2242 VTNET_TXQ_LOCK_ASSERT(txq); 2243 2244 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2245 sc->vtnet_link_active == 0) { 2246 if (m != NULL) 2247 error = drbr_enqueue(ifp, br, m); 2248 return (error); 2249 } 2250 2251 if (m != NULL) { 2252 error = drbr_enqueue(ifp, br, m); 2253 if (error) 2254 return (error); 2255 } 2256 2257 vtnet_txq_eof(txq); 2258 2259 while ((m = drbr_peek(ifp, br)) != NULL) { 2260 if (virtqueue_full(vq)) { 2261 drbr_putback(ifp, br, m); 2262 break; 2263 } 2264 2265 error = vtnet_txq_encap(txq, &m); 2266 if (error) { 2267 if (m != NULL) 2268 drbr_putback(ifp, br, m); 2269 else 2270 drbr_advance(ifp, br); 2271 break; 2272 } 2273 drbr_advance(ifp, br); 2274 2275 enq++; 2276 ETHER_BPF_MTAP(ifp, m); 2277 } 2278 2279 if (enq > 0) { 2280 virtqueue_notify(vq); 2281 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 2282 } 2283 2284 return (0); 2285} 2286 2287static int 2288vtnet_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2289{ 2290 struct vtnet_softc *sc; 2291 struct vtnet_txq *txq; 2292 int i, npairs, error; 2293 2294 sc = ifp->if_softc; 2295 npairs = sc->vtnet_act_vq_pairs; 2296 2297 if (m->m_flags & M_FLOWID) 2298 i = m->m_pkthdr.flowid % npairs; 2299 else 2300 i = curcpu % npairs; 2301 2302 txq = &sc->vtnet_txqs[i]; 2303 2304 if (VTNET_TXQ_TRYLOCK(txq) != 0) { 2305 error = vtnet_txq_mq_start_locked(txq, m); 2306 VTNET_TXQ_UNLOCK(txq); 2307 } else { 2308 error = drbr_enqueue(ifp, txq->vtntx_br, m); 2309 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); 2310 } 2311 2312 return (error); 2313} 2314 2315static void 2316vtnet_txq_tq_deferred(void *xtxq, int pending) 2317{ 2318 struct vtnet_softc *sc; 2319 struct vtnet_txq *txq; 2320 2321 txq = xtxq; 2322 sc = txq->vtntx_sc; 2323 2324 VTNET_TXQ_LOCK(txq); 2325 if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) 2326 vtnet_txq_mq_start_locked(txq, NULL); 2327 VTNET_TXQ_UNLOCK(txq); 2328} 2329 2330#endif /* VTNET_LEGACY_TX */ 2331 2332static void 2333vtnet_txq_start(struct vtnet_txq *txq) 2334{ 2335 struct vtnet_softc *sc; 2336 struct ifnet *ifp; 2337 2338 sc = txq->vtntx_sc; 2339 ifp = sc->vtnet_ifp; 2340 2341#ifdef VTNET_LEGACY_TX 2342 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 2343 vtnet_start_locked(txq, ifp); 2344#else 2345 if (!drbr_empty(ifp, txq->vtntx_br)) 2346 vtnet_txq_mq_start_locked(txq, NULL); 2347#endif 2348} 2349 2350static void 2351vtnet_txq_tq_intr(void *xtxq, int pending) 2352{ 2353 struct vtnet_softc *sc; 2354 struct vtnet_txq *txq; 2355 struct ifnet *ifp; 2356 2357 txq = xtxq; 2358 sc = txq->vtntx_sc; 2359 ifp = sc->vtnet_ifp; 2360 2361 VTNET_TXQ_LOCK(txq); 2362 2363 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2364 VTNET_TXQ_UNLOCK(txq); 2365 return; 2366 } 2367 2368 vtnet_txq_eof(txq); 2369 2370 vtnet_txq_start(txq); 2371 2372 if (vtnet_txq_enable_intr(txq) != 0) { 2373 vtnet_txq_disable_intr(txq); 2374 txq->vtntx_stats.vtxs_rescheduled++; 2375 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2376 } 2377 2378 VTNET_TXQ_UNLOCK(txq); 2379} 2380 2381static void 2382vtnet_txq_eof(struct vtnet_txq *txq) 2383{ 2384 struct virtqueue *vq; 2385 struct vtnet_tx_header *txhdr; 2386 struct mbuf *m; 2387 2388 vq = txq->vtntx_vq; 2389 VTNET_TXQ_LOCK_ASSERT(txq); 2390 2391 while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { 2392 m = txhdr->vth_mbuf; 2393 2394 txq->vtntx_stats.vtxs_opackets++; 2395 txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; 2396 if (m->m_flags & M_MCAST) 2397 txq->vtntx_stats.vtxs_omcasts++; 2398 2399 m_freem(m); 2400 uma_zfree(vtnet_tx_header_zone, txhdr); 2401 } 2402 2403 if (virtqueue_empty(vq)) 2404 txq->vtntx_watchdog = 0; 2405} 2406 2407static void 2408vtnet_tx_vq_intr(void *xtxq) 2409{ 2410 struct vtnet_softc *sc; 2411 struct vtnet_txq *txq; 2412 struct ifnet *ifp; 2413 int tries; 2414 2415 txq = xtxq; 2416 sc = txq->vtntx_sc; 2417 ifp = sc->vtnet_ifp; 2418 tries = 0; 2419 2420 if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { 2421 /* 2422 * Ignore this interrupt. Either this is a spurious interrupt 2423 * or multiqueue without per-VQ MSIX so every queue needs to 2424 * be polled (a brain dead configuration we could try harder 2425 * to avoid). 2426 */ 2427 vtnet_txq_disable_intr(txq); 2428 return; 2429 } 2430 2431 VTNET_TXQ_LOCK(txq); 2432 2433again: 2434 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 2435 VTNET_TXQ_UNLOCK(txq); 2436 return; 2437 } 2438 2439 vtnet_txq_eof(txq); 2440 2441 vtnet_txq_start(txq); 2442 2443 if (vtnet_txq_enable_intr(txq) != 0) { 2444 vtnet_txq_disable_intr(txq); 2445 /* 2446 * This is an occasional race, so retry a few times 2447 * before scheduling the taskqueue. 2448 */ 2449 if (tries++ < VTNET_INTR_DISABLE_RETRIES) 2450 goto again; 2451 2452 VTNET_TXQ_UNLOCK(txq); 2453 txq->vtntx_stats.vtxs_rescheduled++; 2454 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2455 } else 2456 VTNET_TXQ_UNLOCK(txq); 2457} 2458 2459static void 2460vtnet_tx_start_all(struct vtnet_softc *sc) 2461{ 2462 struct vtnet_txq *txq; 2463 int i; 2464 2465 VTNET_CORE_LOCK_ASSERT(sc); 2466 2467 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2468 txq = &sc->vtnet_txqs[i]; 2469 2470 VTNET_TXQ_LOCK(txq); 2471 vtnet_txq_start(txq); 2472 VTNET_TXQ_UNLOCK(txq); 2473 } 2474} 2475 2476#ifndef VTNET_LEGACY_TX 2477static void 2478vtnet_qflush(struct ifnet *ifp) 2479{ 2480 struct vtnet_softc *sc; 2481 struct vtnet_txq *txq; 2482 struct mbuf *m; 2483 int i; 2484 2485 sc = ifp->if_softc; 2486 2487 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2488 txq = &sc->vtnet_txqs[i]; 2489 2490 VTNET_TXQ_LOCK(txq); 2491 while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) 2492 m_freem(m); 2493 VTNET_TXQ_UNLOCK(txq); 2494 } 2495 2496 if_qflush(ifp); 2497} 2498#endif 2499 2500static int 2501vtnet_watchdog(struct vtnet_txq *txq) 2502{ 2503 struct vtnet_softc *sc; 2504 2505 sc = txq->vtntx_sc; 2506 2507 VTNET_TXQ_LOCK(txq); 2508 if (sc->vtnet_flags & VTNET_FLAG_EVENT_IDX) 2509 vtnet_txq_eof(txq); 2510 if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { 2511 VTNET_TXQ_UNLOCK(txq); 2512 return (0); 2513 } 2514 VTNET_TXQ_UNLOCK(txq); 2515 2516 if_printf(sc->vtnet_ifp, "watchdog timeout on queue %d\n", 2517 txq->vtntx_id); 2518 return (1); 2519} 2520 2521static void 2522vtnet_rxq_accum_stats(struct vtnet_rxq *rxq, struct vtnet_rxq_stats *accum) 2523{ 2524 struct vtnet_rxq_stats *st; 2525 2526 st = &rxq->vtnrx_stats; 2527 2528 accum->vrxs_ipackets += st->vrxs_ipackets; 2529 accum->vrxs_ibytes += st->vrxs_ibytes; 2530 accum->vrxs_iqdrops += st->vrxs_iqdrops; 2531 accum->vrxs_csum += st->vrxs_csum; 2532 accum->vrxs_csum_failed += st->vrxs_csum_failed; 2533 accum->vrxs_rescheduled += st->vrxs_rescheduled; 2534} 2535 2536static void 2537vtnet_txq_accum_stats(struct vtnet_txq *txq, struct vtnet_txq_stats *accum) 2538{ 2539 struct vtnet_txq_stats *st; 2540 2541 st = &txq->vtntx_stats; 2542 2543 accum->vtxs_opackets += st->vtxs_opackets; 2544 accum->vtxs_obytes += st->vtxs_obytes; 2545 accum->vtxs_csum += st->vtxs_csum; 2546 accum->vtxs_tso += st->vtxs_tso; 2547 accum->vtxs_rescheduled += st->vtxs_rescheduled; 2548} 2549 2550static void 2551vtnet_accumulate_stats(struct vtnet_softc *sc) 2552{ 2553 struct ifnet *ifp; 2554 struct vtnet_statistics *st; 2555 struct vtnet_rxq_stats rxaccum; 2556 struct vtnet_txq_stats txaccum; 2557 int i; 2558 2559 ifp = sc->vtnet_ifp; 2560 st = &sc->vtnet_stats; 2561 bzero(&rxaccum, sizeof(struct vtnet_rxq_stats)); 2562 bzero(&txaccum, sizeof(struct vtnet_txq_stats)); 2563 2564 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2565 vtnet_rxq_accum_stats(&sc->vtnet_rxqs[i], &rxaccum); 2566 vtnet_txq_accum_stats(&sc->vtnet_txqs[i], &txaccum); 2567 } 2568 2569 st->rx_csum_offloaded = rxaccum.vrxs_csum; 2570 st->rx_csum_failed = rxaccum.vrxs_csum_failed; 2571 st->rx_task_rescheduled = rxaccum.vrxs_rescheduled; 2572 st->tx_csum_offloaded = txaccum.vtxs_csum; 2573 st->tx_tso_offloaded = txaccum.vtxs_tso; 2574 st->tx_task_rescheduled = txaccum.vtxs_rescheduled; 2575 2576 /* 2577 * With the exception of if_ierrors, these ifnet statistics are 2578 * only updated in the driver, so just set them to our accumulated 2579 * values. if_ierrors is updated in ether_input() for malformed 2580 * frames that we should have already discarded. 2581 */ 2582 ifp->if_ipackets = rxaccum.vrxs_ipackets; 2583 ifp->if_iqdrops = rxaccum.vrxs_iqdrops; 2584 ifp->if_ierrors = rxaccum.vrxs_ierrors; 2585 ifp->if_opackets = txaccum.vtxs_opackets; 2586#ifndef VTNET_LEGACY_TX 2587 ifp->if_obytes = txaccum.vtxs_obytes; 2588 ifp->if_omcasts = txaccum.vtxs_omcasts; 2589#endif 2590} 2591 2592static void 2593vtnet_tick(void *xsc) 2594{ 2595 struct vtnet_softc *sc; 2596 struct ifnet *ifp; 2597 int i, timedout; 2598 2599 sc = xsc; 2600 ifp = sc->vtnet_ifp; 2601 timedout = 0; 2602 2603 VTNET_CORE_LOCK_ASSERT(sc); 2604 vtnet_accumulate_stats(sc); 2605 2606 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 2607 timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); 2608 2609 if (timedout != 0) { 2610 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2611 vtnet_init_locked(sc); 2612 } else 2613 callout_schedule(&sc->vtnet_tick_ch, hz); 2614} 2615 2616static void 2617vtnet_start_taskqueues(struct vtnet_softc *sc) 2618{ 2619 device_t dev; 2620 struct vtnet_rxq *rxq; 2621 struct vtnet_txq *txq; 2622 int i, error; 2623 2624 dev = sc->vtnet_dev; 2625 2626 /* 2627 * Errors here are very difficult to recover from - we cannot 2628 * easily fail because, if this is during boot, we will hang 2629 * when freeing any successfully started taskqueues because 2630 * the scheduler isn't up yet. 2631 * 2632 * Most drivers just ignore the return value - it only fails 2633 * with ENOMEM so an error is not likely. 2634 */ 2635 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2636 rxq = &sc->vtnet_rxqs[i]; 2637 error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, 2638 "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); 2639 if (error) { 2640 device_printf(dev, "failed to start rx taskq %d\n", 2641 rxq->vtnrx_id); 2642 } 2643 2644 txq = &sc->vtnet_txqs[i]; 2645 error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, 2646 "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); 2647 if (error) { 2648 device_printf(dev, "failed to start tx taskq %d\n", 2649 txq->vtntx_id); 2650 } 2651 } 2652} 2653 2654static void 2655vtnet_free_taskqueues(struct vtnet_softc *sc) 2656{ 2657 struct vtnet_rxq *rxq; 2658 struct vtnet_txq *txq; 2659 int i; 2660 2661 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2662 rxq = &sc->vtnet_rxqs[i]; 2663 if (rxq->vtnrx_tq != NULL) { 2664 taskqueue_free(rxq->vtnrx_tq); 2665 rxq->vtnrx_vq = NULL; 2666 } 2667 2668 txq = &sc->vtnet_txqs[i]; 2669 if (txq->vtntx_tq != NULL) { 2670 taskqueue_free(txq->vtntx_tq); 2671 txq->vtntx_tq = NULL; 2672 } 2673 } 2674} 2675 2676static void 2677vtnet_drain_taskqueues(struct vtnet_softc *sc) 2678{ 2679 struct vtnet_rxq *rxq; 2680 struct vtnet_txq *txq; 2681 int i; 2682 2683 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2684 rxq = &sc->vtnet_rxqs[i]; 2685 if (rxq->vtnrx_tq != NULL) 2686 taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 2687 2688 txq = &sc->vtnet_txqs[i]; 2689 if (txq->vtntx_tq != NULL) { 2690 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); 2691#ifndef VTNET_LEGACY_TX 2692 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); 2693#endif 2694 } 2695 } 2696} 2697 2698static void 2699vtnet_drain_rxtx_queues(struct vtnet_softc *sc) 2700{ 2701 struct vtnet_rxq *rxq; 2702 struct vtnet_txq *txq; 2703 int i; 2704 2705 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2706 rxq = &sc->vtnet_rxqs[i]; 2707 vtnet_rxq_free_mbufs(rxq); 2708 2709 txq = &sc->vtnet_txqs[i]; 2710 vtnet_txq_free_mbufs(txq); 2711 } 2712} 2713 2714static void 2715vtnet_stop_rendezvous(struct vtnet_softc *sc) 2716{ 2717 struct vtnet_rxq *rxq; 2718 struct vtnet_txq *txq; 2719 int i; 2720 2721 /* 2722 * Lock and unlock the per-queue mutex so we known the stop 2723 * state is visible. Doing only the active queues should be 2724 * sufficient, but it does not cost much extra to do all the 2725 * queues. Note we hold the core mutex here too. 2726 */ 2727 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 2728 rxq = &sc->vtnet_rxqs[i]; 2729 VTNET_RXQ_LOCK(rxq); 2730 VTNET_RXQ_UNLOCK(rxq); 2731 2732 txq = &sc->vtnet_txqs[i]; 2733 VTNET_TXQ_LOCK(txq); 2734 VTNET_TXQ_UNLOCK(txq); 2735 } 2736} 2737 2738static void 2739vtnet_stop(struct vtnet_softc *sc) 2740{ 2741 device_t dev; 2742 struct ifnet *ifp; 2743 2744 dev = sc->vtnet_dev; 2745 ifp = sc->vtnet_ifp; 2746 2747 VTNET_CORE_LOCK_ASSERT(sc); 2748 2749 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2750 sc->vtnet_link_active = 0; 2751 callout_stop(&sc->vtnet_tick_ch); 2752 2753 /* Only advisory. */ 2754 vtnet_disable_interrupts(sc); 2755 2756 /* 2757 * Stop the host adapter. This resets it to the pre-initialized 2758 * state. It will not generate any interrupts until after it is 2759 * reinitialized. 2760 */ 2761 virtio_stop(dev); 2762 vtnet_stop_rendezvous(sc); 2763 2764 /* Free any mbufs left in the virtqueues. */ 2765 vtnet_drain_rxtx_queues(sc); 2766} 2767 2768static int 2769vtnet_virtio_reinit(struct vtnet_softc *sc) 2770{ 2771 device_t dev; 2772 struct ifnet *ifp; 2773 uint64_t features; 2774 int mask, error; 2775 2776 dev = sc->vtnet_dev; 2777 ifp = sc->vtnet_ifp; 2778 features = sc->vtnet_features; 2779 2780 mask = 0; 2781#if defined(INET) 2782 mask |= IFCAP_RXCSUM; 2783#endif 2784#if defined (INET6) 2785 mask |= IFCAP_RXCSUM_IPV6; 2786#endif 2787 2788 /* 2789 * Re-negotiate with the host, removing any disabled receive 2790 * features. Transmit features are disabled only on our side 2791 * via if_capenable and if_hwassist. 2792 */ 2793 2794 if (ifp->if_capabilities & mask) { 2795 /* 2796 * We require both IPv4 and IPv6 offloading to be enabled 2797 * in order to negotiated it: VirtIO does not distinguish 2798 * between the two. 2799 */ 2800 if ((ifp->if_capenable & mask) != mask) 2801 features &= ~VIRTIO_NET_F_GUEST_CSUM; 2802 } 2803 2804 if (ifp->if_capabilities & IFCAP_LRO) { 2805 if ((ifp->if_capenable & IFCAP_LRO) == 0) 2806 features &= ~VTNET_LRO_FEATURES; 2807 } 2808 2809 if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) { 2810 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 2811 features &= ~VIRTIO_NET_F_CTRL_VLAN; 2812 } 2813 2814 error = virtio_reinit(dev, features); 2815 if (error) 2816 device_printf(dev, "virtio reinit error %d\n", error); 2817 2818 return (error); 2819} 2820 2821static void 2822vtnet_init_rx_filters(struct vtnet_softc *sc) 2823{ 2824 struct ifnet *ifp; 2825 2826 ifp = sc->vtnet_ifp; 2827 2828 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 2829 /* Restore promiscuous and all-multicast modes. */ 2830 vtnet_rx_filter(sc); 2831 /* Restore filtered MAC addresses. */ 2832 vtnet_rx_filter_mac(sc); 2833 } 2834 2835 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2836 vtnet_rx_filter_vlan(sc); 2837} 2838 2839static int 2840vtnet_init_rx_queues(struct vtnet_softc *sc) 2841{ 2842 device_t dev; 2843 struct vtnet_rxq *rxq; 2844 int i, clsize, error; 2845 2846 dev = sc->vtnet_dev; 2847 2848 /* 2849 * Use the new cluster size if one has been set (via a MTU 2850 * change). Otherwise, use the standard 2K clusters. 2851 * 2852 * BMV: It might make sense to use page sized clusters as 2853 * the default (depending on the features negotiated). 2854 */ 2855 if (sc->vtnet_rx_new_clsize != 0) { 2856 clsize = sc->vtnet_rx_new_clsize; 2857 sc->vtnet_rx_new_clsize = 0; 2858 } else 2859 clsize = MCLBYTES; 2860 2861 sc->vtnet_rx_clsize = clsize; 2862 sc->vtnet_rx_nmbufs = VTNET_NEEDED_RX_MBUFS(sc, clsize); 2863 2864 KASSERT(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS || 2865 sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, 2866 ("%s: too many rx mbufs %d for %d segments", __func__, 2867 sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); 2868 2869 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2870 rxq = &sc->vtnet_rxqs[i]; 2871 2872 /* Hold the lock to satisfy asserts. */ 2873 VTNET_RXQ_LOCK(rxq); 2874 error = vtnet_rxq_populate(rxq); 2875 VTNET_RXQ_UNLOCK(rxq); 2876 2877 if (error) { 2878 device_printf(dev, 2879 "cannot allocate mbufs for Rx queue %d\n", i); 2880 return (error); 2881 } 2882 } 2883 2884 return (0); 2885} 2886 2887static int 2888vtnet_init_tx_queues(struct vtnet_softc *sc) 2889{ 2890 struct vtnet_txq *txq; 2891 int i; 2892 2893 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 2894 txq = &sc->vtnet_txqs[i]; 2895 txq->vtntx_watchdog = 0; 2896 } 2897 2898 return (0); 2899} 2900 2901static int 2902vtnet_init_rxtx_queues(struct vtnet_softc *sc) 2903{ 2904 int error; 2905 2906 error = vtnet_init_rx_queues(sc); 2907 if (error) 2908 return (error); 2909 2910 error = vtnet_init_tx_queues(sc); 2911 if (error) 2912 return (error); 2913 2914 return (0); 2915} 2916 2917static void 2918vtnet_set_active_vq_pairs(struct vtnet_softc *sc) 2919{ 2920 device_t dev; 2921 int npairs; 2922 2923 dev = sc->vtnet_dev; 2924 2925 if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) { 2926 MPASS(sc->vtnet_max_vq_pairs == 1); 2927 sc->vtnet_act_vq_pairs = 1; 2928 return; 2929 } 2930 2931 /* BMV: Just use the maximum configured for now. */ 2932 npairs = sc->vtnet_max_vq_pairs; 2933 2934 if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { 2935 device_printf(dev, 2936 "cannot set active queue pairs to %d\n", npairs); 2937 npairs = 1; 2938 } 2939 2940 sc->vtnet_act_vq_pairs = npairs; 2941} 2942 2943static int 2944vtnet_reinit(struct vtnet_softc *sc) 2945{
|
2947 device_t dev;
| |
2948 struct ifnet *ifp; 2949 int error; 2950
| 2946 struct ifnet *ifp; 2947 int error; 2948
|
2951 dev = sc->vtnet_dev;
| |
2952 ifp = sc->vtnet_ifp; 2953 2954 /* Use the current MAC address. */ 2955 bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); 2956 vtnet_set_hwaddr(sc); 2957 2958 vtnet_set_active_vq_pairs(sc); 2959 2960 ifp->if_hwassist = 0; 2961 if (ifp->if_capenable & IFCAP_TXCSUM) 2962 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD; 2963 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 2964 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD_IPV6; 2965 if (ifp->if_capenable & IFCAP_TSO4) 2966 ifp->if_hwassist |= CSUM_TSO; 2967 if (ifp->if_capenable & IFCAP_TSO6) 2968 ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */ 2969 2970 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 2971 vtnet_init_rx_filters(sc); 2972 2973 error = vtnet_init_rxtx_queues(sc); 2974 if (error) 2975 return (error); 2976 2977 vtnet_enable_interrupts(sc); 2978 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2979 2980 return (0); 2981} 2982 2983static void 2984vtnet_init_locked(struct vtnet_softc *sc) 2985{ 2986 device_t dev; 2987 struct ifnet *ifp; 2988 2989 dev = sc->vtnet_dev; 2990 ifp = sc->vtnet_ifp; 2991 2992 VTNET_CORE_LOCK_ASSERT(sc); 2993 2994 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2995 return; 2996 2997 vtnet_stop(sc); 2998 2999 /* Reinitialize with the host. */ 3000 if (vtnet_virtio_reinit(sc) != 0) 3001 goto fail; 3002 3003 if (vtnet_reinit(sc) != 0) 3004 goto fail; 3005 3006 virtio_reinit_complete(dev); 3007 3008 vtnet_update_link_status(sc); 3009 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 3010 3011 return; 3012 3013fail: 3014 vtnet_stop(sc); 3015} 3016 3017static void 3018vtnet_init(void *xsc) 3019{ 3020 struct vtnet_softc *sc; 3021 3022 sc = xsc; 3023 3024 VTNET_CORE_LOCK(sc); 3025 vtnet_init_locked(sc); 3026 VTNET_CORE_UNLOCK(sc); 3027} 3028 3029static void 3030vtnet_free_ctrl_vq(struct vtnet_softc *sc) 3031{ 3032 struct virtqueue *vq; 3033 3034 vq = sc->vtnet_ctrl_vq; 3035 3036 /* 3037 * The control virtqueue is only polled and therefore it should 3038 * already be empty. 3039 */ 3040 KASSERT(virtqueue_empty(vq), 3041 ("%s: ctrl vq %p not empty", __func__, vq)); 3042} 3043 3044static void 3045vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie, 3046 struct sglist *sg, int readable, int writable) 3047{ 3048 struct virtqueue *vq; 3049 3050 vq = sc->vtnet_ctrl_vq; 3051 3052 VTNET_CORE_LOCK_ASSERT(sc); 3053 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ, 3054 ("%s: CTRL_VQ feature not negotiated", __func__)); 3055 3056 if (!virtqueue_empty(vq)) 3057 return; 3058 if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0) 3059 return; 3060 3061 /* 3062 * Poll for the response, but the command is likely already 3063 * done when we return from the notify. 3064 */ 3065 virtqueue_notify(vq); 3066 virtqueue_poll(vq, NULL); 3067} 3068 3069static int 3070vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr) 3071{
| 2949 ifp = sc->vtnet_ifp; 2950 2951 /* Use the current MAC address. */ 2952 bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); 2953 vtnet_set_hwaddr(sc); 2954 2955 vtnet_set_active_vq_pairs(sc); 2956 2957 ifp->if_hwassist = 0; 2958 if (ifp->if_capenable & IFCAP_TXCSUM) 2959 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD; 2960 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 2961 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD_IPV6; 2962 if (ifp->if_capenable & IFCAP_TSO4) 2963 ifp->if_hwassist |= CSUM_TSO; 2964 if (ifp->if_capenable & IFCAP_TSO6) 2965 ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */ 2966 2967 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 2968 vtnet_init_rx_filters(sc); 2969 2970 error = vtnet_init_rxtx_queues(sc); 2971 if (error) 2972 return (error); 2973 2974 vtnet_enable_interrupts(sc); 2975 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2976 2977 return (0); 2978} 2979 2980static void 2981vtnet_init_locked(struct vtnet_softc *sc) 2982{ 2983 device_t dev; 2984 struct ifnet *ifp; 2985 2986 dev = sc->vtnet_dev; 2987 ifp = sc->vtnet_ifp; 2988 2989 VTNET_CORE_LOCK_ASSERT(sc); 2990 2991 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2992 return; 2993 2994 vtnet_stop(sc); 2995 2996 /* Reinitialize with the host. */ 2997 if (vtnet_virtio_reinit(sc) != 0) 2998 goto fail; 2999 3000 if (vtnet_reinit(sc) != 0) 3001 goto fail; 3002 3003 virtio_reinit_complete(dev); 3004 3005 vtnet_update_link_status(sc); 3006 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 3007 3008 return; 3009 3010fail: 3011 vtnet_stop(sc); 3012} 3013 3014static void 3015vtnet_init(void *xsc) 3016{ 3017 struct vtnet_softc *sc; 3018 3019 sc = xsc; 3020 3021 VTNET_CORE_LOCK(sc); 3022 vtnet_init_locked(sc); 3023 VTNET_CORE_UNLOCK(sc); 3024} 3025 3026static void 3027vtnet_free_ctrl_vq(struct vtnet_softc *sc) 3028{ 3029 struct virtqueue *vq; 3030 3031 vq = sc->vtnet_ctrl_vq; 3032 3033 /* 3034 * The control virtqueue is only polled and therefore it should 3035 * already be empty. 3036 */ 3037 KASSERT(virtqueue_empty(vq), 3038 ("%s: ctrl vq %p not empty", __func__, vq)); 3039} 3040 3041static void 3042vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie, 3043 struct sglist *sg, int readable, int writable) 3044{ 3045 struct virtqueue *vq; 3046 3047 vq = sc->vtnet_ctrl_vq; 3048 3049 VTNET_CORE_LOCK_ASSERT(sc); 3050 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ, 3051 ("%s: CTRL_VQ feature not negotiated", __func__)); 3052 3053 if (!virtqueue_empty(vq)) 3054 return; 3055 if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0) 3056 return; 3057 3058 /* 3059 * Poll for the response, but the command is likely already 3060 * done when we return from the notify. 3061 */ 3062 virtqueue_notify(vq); 3063 virtqueue_poll(vq, NULL); 3064} 3065 3066static int 3067vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr) 3068{
|
3072 struct virtio_net_ctrl_hdr hdr;
| 3069 struct virtio_net_ctrl_hdr hdr __aligned(2);
|
3073 struct sglist_seg segs[3]; 3074 struct sglist sg; 3075 uint8_t ack; 3076 int error; 3077 3078 hdr.class = VIRTIO_NET_CTRL_MAC; 3079 hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 3080 ack = VIRTIO_NET_ERR; 3081 3082 sglist_init(&sg, 3, segs); 3083 error = 0; 3084 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3085 error |= sglist_append(&sg, hwaddr, ETHER_ADDR_LEN); 3086 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3087 KASSERT(error == 0 && sg.sg_nseg == 3, 3088 ("%s: error %d adding set MAC msg to sglist", __func__, error)); 3089 3090 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3091 3092 return (ack == VIRTIO_NET_OK ? 0 : EIO); 3093} 3094 3095static int 3096vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs) 3097{ 3098 struct sglist_seg segs[3]; 3099 struct sglist sg; 3100 struct { 3101 struct virtio_net_ctrl_hdr hdr; 3102 uint8_t pad1; 3103 struct virtio_net_ctrl_mq mq; 3104 uint8_t pad2; 3105 uint8_t ack;
| 3070 struct sglist_seg segs[3]; 3071 struct sglist sg; 3072 uint8_t ack; 3073 int error; 3074 3075 hdr.class = VIRTIO_NET_CTRL_MAC; 3076 hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 3077 ack = VIRTIO_NET_ERR; 3078 3079 sglist_init(&sg, 3, segs); 3080 error = 0; 3081 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3082 error |= sglist_append(&sg, hwaddr, ETHER_ADDR_LEN); 3083 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3084 KASSERT(error == 0 && sg.sg_nseg == 3, 3085 ("%s: error %d adding set MAC msg to sglist", __func__, error)); 3086 3087 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3088 3089 return (ack == VIRTIO_NET_OK ? 0 : EIO); 3090} 3091 3092static int 3093vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs) 3094{ 3095 struct sglist_seg segs[3]; 3096 struct sglist sg; 3097 struct { 3098 struct virtio_net_ctrl_hdr hdr; 3099 uint8_t pad1; 3100 struct virtio_net_ctrl_mq mq; 3101 uint8_t pad2; 3102 uint8_t ack;
|
3106 } s;
| 3103 } s __aligned(2);
|
3107 int error; 3108 3109 s.hdr.class = VIRTIO_NET_CTRL_MQ; 3110 s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 3111 s.mq.virtqueue_pairs = npairs; 3112 s.ack = VIRTIO_NET_ERR; 3113 3114 sglist_init(&sg, 3, segs); 3115 error = 0; 3116 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3117 error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq)); 3118 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3119 KASSERT(error == 0 && sg.sg_nseg == 3, 3120 ("%s: error %d adding MQ message to sglist", __func__, error)); 3121 3122 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3123 3124 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3125} 3126 3127static int 3128vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on) 3129{ 3130 struct sglist_seg segs[3]; 3131 struct sglist sg; 3132 struct { 3133 struct virtio_net_ctrl_hdr hdr; 3134 uint8_t pad1; 3135 uint8_t onoff; 3136 uint8_t pad2; 3137 uint8_t ack;
| 3104 int error; 3105 3106 s.hdr.class = VIRTIO_NET_CTRL_MQ; 3107 s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 3108 s.mq.virtqueue_pairs = npairs; 3109 s.ack = VIRTIO_NET_ERR; 3110 3111 sglist_init(&sg, 3, segs); 3112 error = 0; 3113 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3114 error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq)); 3115 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3116 KASSERT(error == 0 && sg.sg_nseg == 3, 3117 ("%s: error %d adding MQ message to sglist", __func__, error)); 3118 3119 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3120 3121 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3122} 3123 3124static int 3125vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on) 3126{ 3127 struct sglist_seg segs[3]; 3128 struct sglist sg; 3129 struct { 3130 struct virtio_net_ctrl_hdr hdr; 3131 uint8_t pad1; 3132 uint8_t onoff; 3133 uint8_t pad2; 3134 uint8_t ack;
|
3138 } s;
| 3135 } s __aligned(2);
|
3139 int error; 3140 3141 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3142 ("%s: CTRL_RX feature not negotiated", __func__)); 3143 3144 s.hdr.class = VIRTIO_NET_CTRL_RX; 3145 s.hdr.cmd = cmd; 3146 s.onoff = !!on; 3147 s.ack = VIRTIO_NET_ERR; 3148 3149 sglist_init(&sg, 3, segs); 3150 error = 0; 3151 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3152 error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t)); 3153 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3154 KASSERT(error == 0 && sg.sg_nseg == 3, 3155 ("%s: error %d adding Rx message to sglist", __func__, error)); 3156 3157 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3158 3159 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3160} 3161 3162static int 3163vtnet_set_promisc(struct vtnet_softc *sc, int on) 3164{ 3165 3166 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); 3167} 3168 3169static int 3170vtnet_set_allmulti(struct vtnet_softc *sc, int on) 3171{ 3172 3173 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); 3174} 3175 3176/* 3177 * The device defaults to promiscuous mode for backwards compatibility. 3178 * Turn it off at attach time if possible. 3179 */ 3180static void 3181vtnet_attach_disable_promisc(struct vtnet_softc *sc) 3182{ 3183 struct ifnet *ifp; 3184 3185 ifp = sc->vtnet_ifp; 3186 3187 VTNET_CORE_LOCK(sc); 3188 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) { 3189 ifp->if_flags |= IFF_PROMISC; 3190 } else if (vtnet_set_promisc(sc, 0) != 0) { 3191 ifp->if_flags |= IFF_PROMISC; 3192 device_printf(sc->vtnet_dev, 3193 "cannot disable default promiscuous mode\n"); 3194 } 3195 VTNET_CORE_UNLOCK(sc); 3196} 3197 3198static void 3199vtnet_rx_filter(struct vtnet_softc *sc) 3200{ 3201 device_t dev; 3202 struct ifnet *ifp; 3203 3204 dev = sc->vtnet_dev; 3205 ifp = sc->vtnet_ifp; 3206 3207 VTNET_CORE_LOCK_ASSERT(sc); 3208 3209 if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0) 3210 device_printf(dev, "cannot %s promiscuous mode\n", 3211 ifp->if_flags & IFF_PROMISC ? "enable" : "disable"); 3212 3213 if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0) 3214 device_printf(dev, "cannot %s all-multicast mode\n", 3215 ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable"); 3216} 3217 3218static void 3219vtnet_rx_filter_mac(struct vtnet_softc *sc) 3220{
| 3136 int error; 3137 3138 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3139 ("%s: CTRL_RX feature not negotiated", __func__)); 3140 3141 s.hdr.class = VIRTIO_NET_CTRL_RX; 3142 s.hdr.cmd = cmd; 3143 s.onoff = !!on; 3144 s.ack = VIRTIO_NET_ERR; 3145 3146 sglist_init(&sg, 3, segs); 3147 error = 0; 3148 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3149 error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t)); 3150 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3151 KASSERT(error == 0 && sg.sg_nseg == 3, 3152 ("%s: error %d adding Rx message to sglist", __func__, error)); 3153 3154 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3155 3156 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3157} 3158 3159static int 3160vtnet_set_promisc(struct vtnet_softc *sc, int on) 3161{ 3162 3163 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); 3164} 3165 3166static int 3167vtnet_set_allmulti(struct vtnet_softc *sc, int on) 3168{ 3169 3170 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); 3171} 3172 3173/* 3174 * The device defaults to promiscuous mode for backwards compatibility. 3175 * Turn it off at attach time if possible. 3176 */ 3177static void 3178vtnet_attach_disable_promisc(struct vtnet_softc *sc) 3179{ 3180 struct ifnet *ifp; 3181 3182 ifp = sc->vtnet_ifp; 3183 3184 VTNET_CORE_LOCK(sc); 3185 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) { 3186 ifp->if_flags |= IFF_PROMISC; 3187 } else if (vtnet_set_promisc(sc, 0) != 0) { 3188 ifp->if_flags |= IFF_PROMISC; 3189 device_printf(sc->vtnet_dev, 3190 "cannot disable default promiscuous mode\n"); 3191 } 3192 VTNET_CORE_UNLOCK(sc); 3193} 3194 3195static void 3196vtnet_rx_filter(struct vtnet_softc *sc) 3197{ 3198 device_t dev; 3199 struct ifnet *ifp; 3200 3201 dev = sc->vtnet_dev; 3202 ifp = sc->vtnet_ifp; 3203 3204 VTNET_CORE_LOCK_ASSERT(sc); 3205 3206 if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0) 3207 device_printf(dev, "cannot %s promiscuous mode\n", 3208 ifp->if_flags & IFF_PROMISC ? "enable" : "disable"); 3209 3210 if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0) 3211 device_printf(dev, "cannot %s all-multicast mode\n", 3212 ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable"); 3213} 3214 3215static void 3216vtnet_rx_filter_mac(struct vtnet_softc *sc) 3217{
|
3221 struct virtio_net_ctrl_hdr hdr;
| 3218 struct virtio_net_ctrl_hdr hdr __aligned(2);
|
3222 struct vtnet_mac_filter *filter; 3223 struct sglist_seg segs[4]; 3224 struct sglist sg; 3225 struct ifnet *ifp; 3226 struct ifaddr *ifa; 3227 struct ifmultiaddr *ifma; 3228 int ucnt, mcnt, promisc, allmulti, error; 3229 uint8_t ack; 3230 3231 ifp = sc->vtnet_ifp; 3232 filter = sc->vtnet_mac_filter; 3233 ucnt = 0; 3234 mcnt = 0; 3235 promisc = 0; 3236 allmulti = 0; 3237 3238 VTNET_CORE_LOCK_ASSERT(sc); 3239 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3240 ("%s: CTRL_RX feature not negotiated", __func__)); 3241 3242 /* Unicast MAC addresses: */ 3243 if_addr_rlock(ifp); 3244 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 3245 if (ifa->ifa_addr->sa_family != AF_LINK) 3246 continue; 3247 else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3248 sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) 3249 continue; 3250 else if (ucnt == VTNET_MAX_MAC_ENTRIES) { 3251 promisc = 1; 3252 break; 3253 } 3254 3255 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3256 &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN); 3257 ucnt++; 3258 } 3259 if_addr_runlock(ifp); 3260 3261 if (promisc != 0) { 3262 filter->vmf_unicast.nentries = 0; 3263 if_printf(ifp, "more than %d MAC addresses assigned, " 3264 "falling back to promiscuous mode\n", 3265 VTNET_MAX_MAC_ENTRIES); 3266 } else 3267 filter->vmf_unicast.nentries = ucnt; 3268 3269 /* Multicast MAC addresses: */ 3270 if_maddr_rlock(ifp); 3271 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3272 if (ifma->ifma_addr->sa_family != AF_LINK) 3273 continue; 3274 else if (mcnt == VTNET_MAX_MAC_ENTRIES) { 3275 allmulti = 1; 3276 break; 3277 } 3278 3279 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3280 &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN); 3281 mcnt++; 3282 } 3283 if_maddr_runlock(ifp); 3284 3285 if (allmulti != 0) { 3286 filter->vmf_multicast.nentries = 0; 3287 if_printf(ifp, "more than %d multicast MAC addresses " 3288 "assigned, falling back to all-multicast mode\n", 3289 VTNET_MAX_MAC_ENTRIES); 3290 } else 3291 filter->vmf_multicast.nentries = mcnt; 3292 3293 if (promisc != 0 && allmulti != 0) 3294 goto out; 3295 3296 hdr.class = VIRTIO_NET_CTRL_MAC; 3297 hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 3298 ack = VIRTIO_NET_ERR; 3299 3300 sglist_init(&sg, 4, segs); 3301 error = 0; 3302 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3303 error |= sglist_append(&sg, &filter->vmf_unicast, 3304 sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN); 3305 error |= sglist_append(&sg, &filter->vmf_multicast, 3306 sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN); 3307 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3308 KASSERT(error == 0 && sg.sg_nseg == 4, 3309 ("%s: error %d adding MAC filter msg to sglist", __func__, error)); 3310 3311 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3312 3313 if (ack != VIRTIO_NET_OK) 3314 if_printf(ifp, "error setting host MAC filter table\n"); 3315 3316out: 3317 if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0) 3318 if_printf(ifp, "cannot enable promiscuous mode\n"); 3319 if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0) 3320 if_printf(ifp, "cannot enable all-multicast mode\n"); 3321} 3322 3323static int 3324vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3325{ 3326 struct sglist_seg segs[3]; 3327 struct sglist sg; 3328 struct { 3329 struct virtio_net_ctrl_hdr hdr; 3330 uint8_t pad1; 3331 uint16_t tag; 3332 uint8_t pad2; 3333 uint8_t ack;
| 3219 struct vtnet_mac_filter *filter; 3220 struct sglist_seg segs[4]; 3221 struct sglist sg; 3222 struct ifnet *ifp; 3223 struct ifaddr *ifa; 3224 struct ifmultiaddr *ifma; 3225 int ucnt, mcnt, promisc, allmulti, error; 3226 uint8_t ack; 3227 3228 ifp = sc->vtnet_ifp; 3229 filter = sc->vtnet_mac_filter; 3230 ucnt = 0; 3231 mcnt = 0; 3232 promisc = 0; 3233 allmulti = 0; 3234 3235 VTNET_CORE_LOCK_ASSERT(sc); 3236 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 3237 ("%s: CTRL_RX feature not negotiated", __func__)); 3238 3239 /* Unicast MAC addresses: */ 3240 if_addr_rlock(ifp); 3241 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { 3242 if (ifa->ifa_addr->sa_family != AF_LINK) 3243 continue; 3244 else if (memcmp(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3245 sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) 3246 continue; 3247 else if (ucnt == VTNET_MAX_MAC_ENTRIES) { 3248 promisc = 1; 3249 break; 3250 } 3251 3252 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 3253 &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN); 3254 ucnt++; 3255 } 3256 if_addr_runlock(ifp); 3257 3258 if (promisc != 0) { 3259 filter->vmf_unicast.nentries = 0; 3260 if_printf(ifp, "more than %d MAC addresses assigned, " 3261 "falling back to promiscuous mode\n", 3262 VTNET_MAX_MAC_ENTRIES); 3263 } else 3264 filter->vmf_unicast.nentries = ucnt; 3265 3266 /* Multicast MAC addresses: */ 3267 if_maddr_rlock(ifp); 3268 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3269 if (ifma->ifma_addr->sa_family != AF_LINK) 3270 continue; 3271 else if (mcnt == VTNET_MAX_MAC_ENTRIES) { 3272 allmulti = 1; 3273 break; 3274 } 3275 3276 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3277 &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN); 3278 mcnt++; 3279 } 3280 if_maddr_runlock(ifp); 3281 3282 if (allmulti != 0) { 3283 filter->vmf_multicast.nentries = 0; 3284 if_printf(ifp, "more than %d multicast MAC addresses " 3285 "assigned, falling back to all-multicast mode\n", 3286 VTNET_MAX_MAC_ENTRIES); 3287 } else 3288 filter->vmf_multicast.nentries = mcnt; 3289 3290 if (promisc != 0 && allmulti != 0) 3291 goto out; 3292 3293 hdr.class = VIRTIO_NET_CTRL_MAC; 3294 hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 3295 ack = VIRTIO_NET_ERR; 3296 3297 sglist_init(&sg, 4, segs); 3298 error = 0; 3299 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 3300 error |= sglist_append(&sg, &filter->vmf_unicast, 3301 sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN); 3302 error |= sglist_append(&sg, &filter->vmf_multicast, 3303 sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN); 3304 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 3305 KASSERT(error == 0 && sg.sg_nseg == 4, 3306 ("%s: error %d adding MAC filter msg to sglist", __func__, error)); 3307 3308 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 3309 3310 if (ack != VIRTIO_NET_OK) 3311 if_printf(ifp, "error setting host MAC filter table\n"); 3312 3313out: 3314 if (promisc != 0 && vtnet_set_promisc(sc, 1) != 0) 3315 if_printf(ifp, "cannot enable promiscuous mode\n"); 3316 if (allmulti != 0 && vtnet_set_allmulti(sc, 1) != 0) 3317 if_printf(ifp, "cannot enable all-multicast mode\n"); 3318} 3319 3320static int 3321vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3322{ 3323 struct sglist_seg segs[3]; 3324 struct sglist sg; 3325 struct { 3326 struct virtio_net_ctrl_hdr hdr; 3327 uint8_t pad1; 3328 uint16_t tag; 3329 uint8_t pad2; 3330 uint8_t ack;
|
3334 } s;
| 3331 } s __aligned(2);
|
3335 int error; 3336 3337 s.hdr.class = VIRTIO_NET_CTRL_VLAN; 3338 s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 3339 s.tag = tag; 3340 s.ack = VIRTIO_NET_ERR; 3341 3342 sglist_init(&sg, 3, segs); 3343 error = 0; 3344 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3345 error |= sglist_append(&sg, &s.tag, sizeof(uint16_t)); 3346 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3347 KASSERT(error == 0 && sg.sg_nseg == 3, 3348 ("%s: error %d adding VLAN message to sglist", __func__, error)); 3349 3350 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3351 3352 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3353} 3354 3355static void 3356vtnet_rx_filter_vlan(struct vtnet_softc *sc) 3357{ 3358 uint32_t w; 3359 uint16_t tag; 3360 int i, bit; 3361 3362 VTNET_CORE_LOCK_ASSERT(sc); 3363 KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER, 3364 ("%s: VLAN_FILTER feature not negotiated", __func__)); 3365 3366 /* Enable the filter for each configured VLAN. */ 3367 for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) { 3368 w = sc->vtnet_vlan_filter[i]; 3369 3370 while ((bit = ffs(w) - 1) != -1) { 3371 w &= ~(1 << bit); 3372 tag = sizeof(w) * CHAR_BIT * i + bit; 3373 3374 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) { 3375 device_printf(sc->vtnet_dev, 3376 "cannot enable VLAN %d filter\n", tag); 3377 } 3378 } 3379 } 3380} 3381 3382static void 3383vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3384{ 3385 struct ifnet *ifp; 3386 int idx, bit; 3387 3388 ifp = sc->vtnet_ifp; 3389 idx = (tag >> 5) & 0x7F; 3390 bit = tag & 0x1F; 3391 3392 if (tag == 0 || tag > 4095) 3393 return; 3394 3395 VTNET_CORE_LOCK(sc); 3396 3397 if (add) 3398 sc->vtnet_vlan_filter[idx] |= (1 << bit); 3399 else 3400 sc->vtnet_vlan_filter[idx] &= ~(1 << bit); 3401 3402 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER && 3403 vtnet_exec_vlan_filter(sc, add, tag) != 0) { 3404 device_printf(sc->vtnet_dev, 3405 "cannot %s VLAN %d %s the host filter table\n", 3406 add ? "add" : "remove", tag, add ? "to" : "from"); 3407 } 3408 3409 VTNET_CORE_UNLOCK(sc); 3410} 3411 3412static void 3413vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3414{ 3415 3416 if (ifp->if_softc != arg) 3417 return; 3418 3419 vtnet_update_vlan_filter(arg, 1, tag); 3420} 3421 3422static void 3423vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3424{ 3425 3426 if (ifp->if_softc != arg) 3427 return; 3428 3429 vtnet_update_vlan_filter(arg, 0, tag); 3430} 3431 3432static int 3433vtnet_is_link_up(struct vtnet_softc *sc) 3434{ 3435 device_t dev; 3436 struct ifnet *ifp; 3437 uint16_t status; 3438 3439 dev = sc->vtnet_dev; 3440 ifp = sc->vtnet_ifp; 3441 3442 if ((ifp->if_capabilities & IFCAP_LINKSTATE) == 0) 3443 status = VIRTIO_NET_S_LINK_UP; 3444 else 3445 status = virtio_read_dev_config_2(dev, 3446 offsetof(struct virtio_net_config, status)); 3447 3448 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 3449} 3450 3451static void 3452vtnet_update_link_status(struct vtnet_softc *sc) 3453{ 3454 struct ifnet *ifp; 3455 int link; 3456 3457 ifp = sc->vtnet_ifp; 3458 3459 VTNET_CORE_LOCK_ASSERT(sc); 3460 link = vtnet_is_link_up(sc); 3461 3462 /* Notify if the link status has changed. */ 3463 if (link != 0 && sc->vtnet_link_active == 0) { 3464 sc->vtnet_link_active = 1; 3465 if_link_state_change(ifp, LINK_STATE_UP); 3466 } else if (link == 0 && sc->vtnet_link_active != 0) { 3467 sc->vtnet_link_active = 0; 3468 if_link_state_change(ifp, LINK_STATE_DOWN); 3469 } 3470} 3471 3472static int 3473vtnet_ifmedia_upd(struct ifnet *ifp) 3474{ 3475 struct vtnet_softc *sc; 3476 struct ifmedia *ifm; 3477 3478 sc = ifp->if_softc; 3479 ifm = &sc->vtnet_media; 3480 3481 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3482 return (EINVAL); 3483 3484 return (0); 3485} 3486 3487static void 3488vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 3489{ 3490 struct vtnet_softc *sc; 3491 3492 sc = ifp->if_softc; 3493 3494 ifmr->ifm_status = IFM_AVALID; 3495 ifmr->ifm_active = IFM_ETHER; 3496 3497 VTNET_CORE_LOCK(sc); 3498 if (vtnet_is_link_up(sc) != 0) { 3499 ifmr->ifm_status |= IFM_ACTIVE; 3500 ifmr->ifm_active |= VTNET_MEDIATYPE; 3501 } else 3502 ifmr->ifm_active |= IFM_NONE; 3503 VTNET_CORE_UNLOCK(sc); 3504} 3505 3506static void 3507vtnet_set_hwaddr(struct vtnet_softc *sc) 3508{ 3509 device_t dev; 3510 int i; 3511 3512 dev = sc->vtnet_dev; 3513 3514 if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { 3515 if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0) 3516 device_printf(dev, "unable to set MAC address\n"); 3517 } else if (sc->vtnet_flags & VTNET_FLAG_MAC) { 3518 for (i = 0; i < ETHER_ADDR_LEN; i++) { 3519 virtio_write_dev_config_1(dev, 3520 offsetof(struct virtio_net_config, mac) + i, 3521 sc->vtnet_hwaddr[i]); 3522 } 3523 } 3524} 3525 3526static void 3527vtnet_get_hwaddr(struct vtnet_softc *sc) 3528{ 3529 device_t dev; 3530 int i; 3531 3532 dev = sc->vtnet_dev; 3533 3534 if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) { 3535 /* 3536 * Generate a random locally administered unicast address. 3537 * 3538 * It would be nice to generate the same MAC address across 3539 * reboots, but it seems all the hosts currently available 3540 * support the MAC feature, so this isn't too important. 3541 */ 3542 sc->vtnet_hwaddr[0] = 0xB2; 3543 arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); 3544 vtnet_set_hwaddr(sc); 3545 return; 3546 } 3547 3548 for (i = 0; i < ETHER_ADDR_LEN; i++) { 3549 sc->vtnet_hwaddr[i] = virtio_read_dev_config_1(dev, 3550 offsetof(struct virtio_net_config, mac) + i); 3551 } 3552} 3553 3554static void 3555vtnet_vlan_tag_remove(struct mbuf *m) 3556{ 3557 struct ether_vlan_header *evh; 3558 3559 evh = mtod(m, struct ether_vlan_header *); 3560 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 3561 m->m_flags |= M_VLANTAG; 3562 3563 /* Strip the 802.1Q header. */ 3564 bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 3565 ETHER_HDR_LEN - ETHER_TYPE_LEN); 3566 m_adj(m, ETHER_VLAN_ENCAP_LEN); 3567} 3568 3569static void 3570vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, 3571 struct sysctl_oid_list *child, struct vtnet_rxq *rxq) 3572{ 3573 struct sysctl_oid *node; 3574 struct sysctl_oid_list *list; 3575 struct vtnet_rxq_stats *stats; 3576 char namebuf[16]; 3577 3578 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); 3579 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3580 CTLFLAG_RD, NULL, "Receive Queue"); 3581 list = SYSCTL_CHILDREN(node); 3582 3583 stats = &rxq->vtnrx_stats; 3584 3585 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3586 &stats->vrxs_ipackets, "Receive packets"); 3587 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3588 &stats->vrxs_ibytes, "Receive bytes"); 3589 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3590 &stats->vrxs_iqdrops, "Receive drops"); 3591 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3592 &stats->vrxs_ierrors, "Receive errors"); 3593 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3594 &stats->vrxs_csum, "Receive checksum offloaded"); 3595 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD, 3596 &stats->vrxs_csum_failed, "Receive checksum offload failed"); 3597 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3598 &stats->vrxs_rescheduled, 3599 "Receive interrupt handler rescheduled"); 3600} 3601 3602static void 3603vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, 3604 struct sysctl_oid_list *child, struct vtnet_txq *txq) 3605{ 3606 struct sysctl_oid *node; 3607 struct sysctl_oid_list *list; 3608 struct vtnet_txq_stats *stats; 3609 char namebuf[16]; 3610 3611 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); 3612 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3613 CTLFLAG_RD, NULL, "Transmit Queue"); 3614 list = SYSCTL_CHILDREN(node); 3615 3616 stats = &txq->vtntx_stats; 3617 3618 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3619 &stats->vtxs_opackets, "Transmit packets"); 3620 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3621 &stats->vtxs_obytes, "Transmit bytes"); 3622 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3623 &stats->vtxs_omcasts, "Transmit multicasts"); 3624 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3625 &stats->vtxs_csum, "Transmit checksum offloaded"); 3626 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3627 &stats->vtxs_tso, "Transmit segmentation offloaded"); 3628 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3629 &stats->vtxs_rescheduled, 3630 "Transmit interrupt handler rescheduled"); 3631} 3632 3633static void 3634vtnet_setup_queue_sysctl(struct vtnet_softc *sc) 3635{ 3636 device_t dev; 3637 struct sysctl_ctx_list *ctx; 3638 struct sysctl_oid *tree; 3639 struct sysctl_oid_list *child; 3640 int i; 3641 3642 dev = sc->vtnet_dev; 3643 ctx = device_get_sysctl_ctx(dev); 3644 tree = device_get_sysctl_tree(dev); 3645 child = SYSCTL_CHILDREN(tree); 3646 3647 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3648 vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); 3649 vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); 3650 } 3651} 3652 3653static void 3654vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, 3655 struct sysctl_oid_list *child, struct vtnet_softc *sc) 3656{ 3657 struct vtnet_statistics *stats; 3658 3659 stats = &sc->vtnet_stats; 3660 3661 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", 3662 CTLFLAG_RD, &stats->mbuf_alloc_failed, 3663 "Mbuf cluster allocation failures"); 3664 3665 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", 3666 CTLFLAG_RD, &stats->rx_frame_too_large, 3667 "Received frame larger than the mbuf chain"); 3668 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", 3669 CTLFLAG_RD, &stats->rx_enq_replacement_failed, 3670 "Enqueuing the replacement receive mbuf failed"); 3671 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", 3672 CTLFLAG_RD, &stats->rx_mergeable_failed, 3673 "Mergeable buffers receive failures"); 3674 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", 3675 CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 3676 "Received checksum offloaded buffer with unsupported " 3677 "Ethernet type"); 3678 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", 3679 CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 3680 "Received checksum offloaded buffer with incorrect IP protocol"); 3681 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset", 3682 CTLFLAG_RD, &stats->rx_csum_bad_offset, 3683 "Received checksum offloaded buffer with incorrect offset"); 3684 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto", 3685 CTLFLAG_RD, &stats->rx_csum_bad_proto, 3686 "Received checksum offloaded buffer with incorrect protocol"); 3687 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed", 3688 CTLFLAG_RD, &stats->rx_csum_failed, 3689 "Received buffer checksum offload failed"); 3690 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded", 3691 CTLFLAG_RD, &stats->rx_csum_offloaded, 3692 "Received buffer checksum offload succeeded"); 3693 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled", 3694 CTLFLAG_RD, &stats->rx_task_rescheduled, 3695 "Times the receive interrupt task rescheduled itself"); 3696 3697 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype", 3698 CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 3699 "Aborted transmit of checksum offloaded buffer with unknown " 3700 "Ethernet type"); 3701 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype", 3702 CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 3703 "Aborted transmit of TSO buffer with unknown Ethernet type"); 3704 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", 3705 CTLFLAG_RD, &stats->tx_tso_not_tcp, 3706 "Aborted transmit of TSO buffer with non TCP protocol"); 3707 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", 3708 CTLFLAG_RD, &stats->tx_defragged, 3709 "Transmit mbufs defragged"); 3710 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", 3711 CTLFLAG_RD, &stats->tx_defrag_failed, 3712 "Aborted transmit of buffer because defrag failed"); 3713 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded", 3714 CTLFLAG_RD, &stats->tx_csum_offloaded, 3715 "Offloaded checksum of transmitted buffer"); 3716 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded", 3717 CTLFLAG_RD, &stats->tx_tso_offloaded, 3718 "Segmentation offload of transmitted buffer"); 3719 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled", 3720 CTLFLAG_RD, &stats->tx_task_rescheduled, 3721 "Times the transmit interrupt task rescheduled itself"); 3722} 3723 3724static void 3725vtnet_setup_sysctl(struct vtnet_softc *sc) 3726{ 3727 device_t dev; 3728 struct sysctl_ctx_list *ctx; 3729 struct sysctl_oid *tree; 3730 struct sysctl_oid_list *child; 3731 3732 dev = sc->vtnet_dev; 3733 ctx = device_get_sysctl_ctx(dev); 3734 tree = device_get_sysctl_tree(dev); 3735 child = SYSCTL_CHILDREN(tree); 3736 3737 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", 3738 CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, 3739 "Maximum number of supported virtqueue pairs"); 3740 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", 3741 CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, 3742 "Number of active virtqueue pairs"); 3743 3744 vtnet_setup_stat_sysctl(ctx, child, sc); 3745} 3746 3747static int 3748vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) 3749{ 3750 3751 return (virtqueue_enable_intr(rxq->vtnrx_vq)); 3752} 3753 3754static void 3755vtnet_rxq_disable_intr(struct vtnet_rxq *rxq) 3756{ 3757 3758 virtqueue_disable_intr(rxq->vtnrx_vq); 3759} 3760 3761static int 3762vtnet_txq_enable_intr(struct vtnet_txq *txq) 3763{ 3764 3765 return (virtqueue_postpone_intr(txq->vtntx_vq, VQ_POSTPONE_LONG)); 3766} 3767 3768static void 3769vtnet_txq_disable_intr(struct vtnet_txq *txq) 3770{ 3771 3772 virtqueue_disable_intr(txq->vtntx_vq); 3773} 3774 3775static void 3776vtnet_enable_rx_interrupts(struct vtnet_softc *sc) 3777{ 3778 int i; 3779 3780 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3781 vtnet_rxq_enable_intr(&sc->vtnet_rxqs[i]); 3782} 3783 3784static void 3785vtnet_enable_tx_interrupts(struct vtnet_softc *sc) 3786{ 3787 int i; 3788 3789 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3790 vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); 3791} 3792 3793static void 3794vtnet_enable_interrupts(struct vtnet_softc *sc) 3795{ 3796 3797 vtnet_enable_rx_interrupts(sc); 3798 vtnet_enable_tx_interrupts(sc); 3799} 3800 3801static void 3802vtnet_disable_rx_interrupts(struct vtnet_softc *sc) 3803{ 3804 int i; 3805 3806 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3807 vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); 3808} 3809 3810static void 3811vtnet_disable_tx_interrupts(struct vtnet_softc *sc) 3812{ 3813 int i; 3814 3815 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3816 vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); 3817} 3818 3819static void 3820vtnet_disable_interrupts(struct vtnet_softc *sc) 3821{ 3822 3823 vtnet_disable_rx_interrupts(sc); 3824 vtnet_disable_tx_interrupts(sc); 3825} 3826 3827static int 3828vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def) 3829{ 3830 char path[64]; 3831 3832 snprintf(path, sizeof(path), 3833 "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); 3834 TUNABLE_INT_FETCH(path, &def); 3835 3836 return (def); 3837}
| 3332 int error; 3333 3334 s.hdr.class = VIRTIO_NET_CTRL_VLAN; 3335 s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 3336 s.tag = tag; 3337 s.ack = VIRTIO_NET_ERR; 3338 3339 sglist_init(&sg, 3, segs); 3340 error = 0; 3341 error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); 3342 error |= sglist_append(&sg, &s.tag, sizeof(uint16_t)); 3343 error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); 3344 KASSERT(error == 0 && sg.sg_nseg == 3, 3345 ("%s: error %d adding VLAN message to sglist", __func__, error)); 3346 3347 vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); 3348 3349 return (s.ack == VIRTIO_NET_OK ? 0 : EIO); 3350} 3351 3352static void 3353vtnet_rx_filter_vlan(struct vtnet_softc *sc) 3354{ 3355 uint32_t w; 3356 uint16_t tag; 3357 int i, bit; 3358 3359 VTNET_CORE_LOCK_ASSERT(sc); 3360 KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER, 3361 ("%s: VLAN_FILTER feature not negotiated", __func__)); 3362 3363 /* Enable the filter for each configured VLAN. */ 3364 for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) { 3365 w = sc->vtnet_vlan_filter[i]; 3366 3367 while ((bit = ffs(w) - 1) != -1) { 3368 w &= ~(1 << bit); 3369 tag = sizeof(w) * CHAR_BIT * i + bit; 3370 3371 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) { 3372 device_printf(sc->vtnet_dev, 3373 "cannot enable VLAN %d filter\n", tag); 3374 } 3375 } 3376 } 3377} 3378 3379static void 3380vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 3381{ 3382 struct ifnet *ifp; 3383 int idx, bit; 3384 3385 ifp = sc->vtnet_ifp; 3386 idx = (tag >> 5) & 0x7F; 3387 bit = tag & 0x1F; 3388 3389 if (tag == 0 || tag > 4095) 3390 return; 3391 3392 VTNET_CORE_LOCK(sc); 3393 3394 if (add) 3395 sc->vtnet_vlan_filter[idx] |= (1 << bit); 3396 else 3397 sc->vtnet_vlan_filter[idx] &= ~(1 << bit); 3398 3399 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER && 3400 vtnet_exec_vlan_filter(sc, add, tag) != 0) { 3401 device_printf(sc->vtnet_dev, 3402 "cannot %s VLAN %d %s the host filter table\n", 3403 add ? "add" : "remove", tag, add ? "to" : "from"); 3404 } 3405 3406 VTNET_CORE_UNLOCK(sc); 3407} 3408 3409static void 3410vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3411{ 3412 3413 if (ifp->if_softc != arg) 3414 return; 3415 3416 vtnet_update_vlan_filter(arg, 1, tag); 3417} 3418 3419static void 3420vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3421{ 3422 3423 if (ifp->if_softc != arg) 3424 return; 3425 3426 vtnet_update_vlan_filter(arg, 0, tag); 3427} 3428 3429static int 3430vtnet_is_link_up(struct vtnet_softc *sc) 3431{ 3432 device_t dev; 3433 struct ifnet *ifp; 3434 uint16_t status; 3435 3436 dev = sc->vtnet_dev; 3437 ifp = sc->vtnet_ifp; 3438 3439 if ((ifp->if_capabilities & IFCAP_LINKSTATE) == 0) 3440 status = VIRTIO_NET_S_LINK_UP; 3441 else 3442 status = virtio_read_dev_config_2(dev, 3443 offsetof(struct virtio_net_config, status)); 3444 3445 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 3446} 3447 3448static void 3449vtnet_update_link_status(struct vtnet_softc *sc) 3450{ 3451 struct ifnet *ifp; 3452 int link; 3453 3454 ifp = sc->vtnet_ifp; 3455 3456 VTNET_CORE_LOCK_ASSERT(sc); 3457 link = vtnet_is_link_up(sc); 3458 3459 /* Notify if the link status has changed. */ 3460 if (link != 0 && sc->vtnet_link_active == 0) { 3461 sc->vtnet_link_active = 1; 3462 if_link_state_change(ifp, LINK_STATE_UP); 3463 } else if (link == 0 && sc->vtnet_link_active != 0) { 3464 sc->vtnet_link_active = 0; 3465 if_link_state_change(ifp, LINK_STATE_DOWN); 3466 } 3467} 3468 3469static int 3470vtnet_ifmedia_upd(struct ifnet *ifp) 3471{ 3472 struct vtnet_softc *sc; 3473 struct ifmedia *ifm; 3474 3475 sc = ifp->if_softc; 3476 ifm = &sc->vtnet_media; 3477 3478 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3479 return (EINVAL); 3480 3481 return (0); 3482} 3483 3484static void 3485vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 3486{ 3487 struct vtnet_softc *sc; 3488 3489 sc = ifp->if_softc; 3490 3491 ifmr->ifm_status = IFM_AVALID; 3492 ifmr->ifm_active = IFM_ETHER; 3493 3494 VTNET_CORE_LOCK(sc); 3495 if (vtnet_is_link_up(sc) != 0) { 3496 ifmr->ifm_status |= IFM_ACTIVE; 3497 ifmr->ifm_active |= VTNET_MEDIATYPE; 3498 } else 3499 ifmr->ifm_active |= IFM_NONE; 3500 VTNET_CORE_UNLOCK(sc); 3501} 3502 3503static void 3504vtnet_set_hwaddr(struct vtnet_softc *sc) 3505{ 3506 device_t dev; 3507 int i; 3508 3509 dev = sc->vtnet_dev; 3510 3511 if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { 3512 if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0) 3513 device_printf(dev, "unable to set MAC address\n"); 3514 } else if (sc->vtnet_flags & VTNET_FLAG_MAC) { 3515 for (i = 0; i < ETHER_ADDR_LEN; i++) { 3516 virtio_write_dev_config_1(dev, 3517 offsetof(struct virtio_net_config, mac) + i, 3518 sc->vtnet_hwaddr[i]); 3519 } 3520 } 3521} 3522 3523static void 3524vtnet_get_hwaddr(struct vtnet_softc *sc) 3525{ 3526 device_t dev; 3527 int i; 3528 3529 dev = sc->vtnet_dev; 3530 3531 if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) { 3532 /* 3533 * Generate a random locally administered unicast address. 3534 * 3535 * It would be nice to generate the same MAC address across 3536 * reboots, but it seems all the hosts currently available 3537 * support the MAC feature, so this isn't too important. 3538 */ 3539 sc->vtnet_hwaddr[0] = 0xB2; 3540 arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); 3541 vtnet_set_hwaddr(sc); 3542 return; 3543 } 3544 3545 for (i = 0; i < ETHER_ADDR_LEN; i++) { 3546 sc->vtnet_hwaddr[i] = virtio_read_dev_config_1(dev, 3547 offsetof(struct virtio_net_config, mac) + i); 3548 } 3549} 3550 3551static void 3552vtnet_vlan_tag_remove(struct mbuf *m) 3553{ 3554 struct ether_vlan_header *evh; 3555 3556 evh = mtod(m, struct ether_vlan_header *); 3557 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 3558 m->m_flags |= M_VLANTAG; 3559 3560 /* Strip the 802.1Q header. */ 3561 bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 3562 ETHER_HDR_LEN - ETHER_TYPE_LEN); 3563 m_adj(m, ETHER_VLAN_ENCAP_LEN); 3564} 3565 3566static void 3567vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, 3568 struct sysctl_oid_list *child, struct vtnet_rxq *rxq) 3569{ 3570 struct sysctl_oid *node; 3571 struct sysctl_oid_list *list; 3572 struct vtnet_rxq_stats *stats; 3573 char namebuf[16]; 3574 3575 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); 3576 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3577 CTLFLAG_RD, NULL, "Receive Queue"); 3578 list = SYSCTL_CHILDREN(node); 3579 3580 stats = &rxq->vtnrx_stats; 3581 3582 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3583 &stats->vrxs_ipackets, "Receive packets"); 3584 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3585 &stats->vrxs_ibytes, "Receive bytes"); 3586 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3587 &stats->vrxs_iqdrops, "Receive drops"); 3588 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3589 &stats->vrxs_ierrors, "Receive errors"); 3590 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3591 &stats->vrxs_csum, "Receive checksum offloaded"); 3592 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD, 3593 &stats->vrxs_csum_failed, "Receive checksum offload failed"); 3594 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3595 &stats->vrxs_rescheduled, 3596 "Receive interrupt handler rescheduled"); 3597} 3598 3599static void 3600vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, 3601 struct sysctl_oid_list *child, struct vtnet_txq *txq) 3602{ 3603 struct sysctl_oid *node; 3604 struct sysctl_oid_list *list; 3605 struct vtnet_txq_stats *stats; 3606 char namebuf[16]; 3607 3608 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); 3609 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3610 CTLFLAG_RD, NULL, "Transmit Queue"); 3611 list = SYSCTL_CHILDREN(node); 3612 3613 stats = &txq->vtntx_stats; 3614 3615 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3616 &stats->vtxs_opackets, "Transmit packets"); 3617 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3618 &stats->vtxs_obytes, "Transmit bytes"); 3619 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3620 &stats->vtxs_omcasts, "Transmit multicasts"); 3621 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3622 &stats->vtxs_csum, "Transmit checksum offloaded"); 3623 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3624 &stats->vtxs_tso, "Transmit segmentation offloaded"); 3625 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, 3626 &stats->vtxs_rescheduled, 3627 "Transmit interrupt handler rescheduled"); 3628} 3629 3630static void 3631vtnet_setup_queue_sysctl(struct vtnet_softc *sc) 3632{ 3633 device_t dev; 3634 struct sysctl_ctx_list *ctx; 3635 struct sysctl_oid *tree; 3636 struct sysctl_oid_list *child; 3637 int i; 3638 3639 dev = sc->vtnet_dev; 3640 ctx = device_get_sysctl_ctx(dev); 3641 tree = device_get_sysctl_tree(dev); 3642 child = SYSCTL_CHILDREN(tree); 3643 3644 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3645 vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); 3646 vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); 3647 } 3648} 3649 3650static void 3651vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, 3652 struct sysctl_oid_list *child, struct vtnet_softc *sc) 3653{ 3654 struct vtnet_statistics *stats; 3655 3656 stats = &sc->vtnet_stats; 3657 3658 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", 3659 CTLFLAG_RD, &stats->mbuf_alloc_failed, 3660 "Mbuf cluster allocation failures"); 3661 3662 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", 3663 CTLFLAG_RD, &stats->rx_frame_too_large, 3664 "Received frame larger than the mbuf chain"); 3665 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", 3666 CTLFLAG_RD, &stats->rx_enq_replacement_failed, 3667 "Enqueuing the replacement receive mbuf failed"); 3668 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", 3669 CTLFLAG_RD, &stats->rx_mergeable_failed, 3670 "Mergeable buffers receive failures"); 3671 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", 3672 CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 3673 "Received checksum offloaded buffer with unsupported " 3674 "Ethernet type"); 3675 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", 3676 CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 3677 "Received checksum offloaded buffer with incorrect IP protocol"); 3678 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset", 3679 CTLFLAG_RD, &stats->rx_csum_bad_offset, 3680 "Received checksum offloaded buffer with incorrect offset"); 3681 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto", 3682 CTLFLAG_RD, &stats->rx_csum_bad_proto, 3683 "Received checksum offloaded buffer with incorrect protocol"); 3684 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed", 3685 CTLFLAG_RD, &stats->rx_csum_failed, 3686 "Received buffer checksum offload failed"); 3687 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded", 3688 CTLFLAG_RD, &stats->rx_csum_offloaded, 3689 "Received buffer checksum offload succeeded"); 3690 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled", 3691 CTLFLAG_RD, &stats->rx_task_rescheduled, 3692 "Times the receive interrupt task rescheduled itself"); 3693 3694 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype", 3695 CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 3696 "Aborted transmit of checksum offloaded buffer with unknown " 3697 "Ethernet type"); 3698 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype", 3699 CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 3700 "Aborted transmit of TSO buffer with unknown Ethernet type"); 3701 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", 3702 CTLFLAG_RD, &stats->tx_tso_not_tcp, 3703 "Aborted transmit of TSO buffer with non TCP protocol"); 3704 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", 3705 CTLFLAG_RD, &stats->tx_defragged, 3706 "Transmit mbufs defragged"); 3707 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", 3708 CTLFLAG_RD, &stats->tx_defrag_failed, 3709 "Aborted transmit of buffer because defrag failed"); 3710 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded", 3711 CTLFLAG_RD, &stats->tx_csum_offloaded, 3712 "Offloaded checksum of transmitted buffer"); 3713 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded", 3714 CTLFLAG_RD, &stats->tx_tso_offloaded, 3715 "Segmentation offload of transmitted buffer"); 3716 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled", 3717 CTLFLAG_RD, &stats->tx_task_rescheduled, 3718 "Times the transmit interrupt task rescheduled itself"); 3719} 3720 3721static void 3722vtnet_setup_sysctl(struct vtnet_softc *sc) 3723{ 3724 device_t dev; 3725 struct sysctl_ctx_list *ctx; 3726 struct sysctl_oid *tree; 3727 struct sysctl_oid_list *child; 3728 3729 dev = sc->vtnet_dev; 3730 ctx = device_get_sysctl_ctx(dev); 3731 tree = device_get_sysctl_tree(dev); 3732 child = SYSCTL_CHILDREN(tree); 3733 3734 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", 3735 CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, 3736 "Maximum number of supported virtqueue pairs"); 3737 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", 3738 CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, 3739 "Number of active virtqueue pairs"); 3740 3741 vtnet_setup_stat_sysctl(ctx, child, sc); 3742} 3743 3744static int 3745vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) 3746{ 3747 3748 return (virtqueue_enable_intr(rxq->vtnrx_vq)); 3749} 3750 3751static void 3752vtnet_rxq_disable_intr(struct vtnet_rxq *rxq) 3753{ 3754 3755 virtqueue_disable_intr(rxq->vtnrx_vq); 3756} 3757 3758static int 3759vtnet_txq_enable_intr(struct vtnet_txq *txq) 3760{ 3761 3762 return (virtqueue_postpone_intr(txq->vtntx_vq, VQ_POSTPONE_LONG)); 3763} 3764 3765static void 3766vtnet_txq_disable_intr(struct vtnet_txq *txq) 3767{ 3768 3769 virtqueue_disable_intr(txq->vtntx_vq); 3770} 3771 3772static void 3773vtnet_enable_rx_interrupts(struct vtnet_softc *sc) 3774{ 3775 int i; 3776 3777 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3778 vtnet_rxq_enable_intr(&sc->vtnet_rxqs[i]); 3779} 3780 3781static void 3782vtnet_enable_tx_interrupts(struct vtnet_softc *sc) 3783{ 3784 int i; 3785 3786 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3787 vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); 3788} 3789 3790static void 3791vtnet_enable_interrupts(struct vtnet_softc *sc) 3792{ 3793 3794 vtnet_enable_rx_interrupts(sc); 3795 vtnet_enable_tx_interrupts(sc); 3796} 3797 3798static void 3799vtnet_disable_rx_interrupts(struct vtnet_softc *sc) 3800{ 3801 int i; 3802 3803 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3804 vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); 3805} 3806 3807static void 3808vtnet_disable_tx_interrupts(struct vtnet_softc *sc) 3809{ 3810 int i; 3811 3812 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3813 vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); 3814} 3815 3816static void 3817vtnet_disable_interrupts(struct vtnet_softc *sc) 3818{ 3819 3820 vtnet_disable_rx_interrupts(sc); 3821 vtnet_disable_tx_interrupts(sc); 3822} 3823 3824static int 3825vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def) 3826{ 3827 char path[64]; 3828 3829 snprintf(path, sizeof(path), 3830 "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); 3831 TUNABLE_INT_FETCH(path, &def); 3832 3833 return (def); 3834}
|