if_vioif.c revision 1.37
1/* $NetBSD: if_vioif.c,v 1.37 2017/05/17 20:13:02 jdolecek Exp $ */ 2 3/* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.37 2017/05/17 20:13:02 jdolecek Exp $"); 30 31#ifdef _KERNEL_OPT 32#include "opt_net_mpsafe.h" 33#endif 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/condvar.h> 40#include <sys/device.h> 41#include <sys/intr.h> 42#include <sys/kmem.h> 43#include <sys/mbuf.h> 44#include <sys/mutex.h> 45#include <sys/sockio.h> 46#include <sys/cpu.h> 47#include <sys/module.h> 48 49#include <dev/pci/pcidevs.h> 50#include <dev/pci/pcireg.h> 51#include <dev/pci/pcivar.h> 52#include <dev/pci/virtioreg.h> 53#include <dev/pci/virtiovar.h> 54 55#include <net/if.h> 56#include <net/if_media.h> 57#include <net/if_ether.h> 58 59#include <net/bpf.h> 60 61#include "ioconf.h" 62 63#ifdef NET_MPSAFE 64#define VIOIF_MPSAFE 1 65#endif 66 67#ifdef SOFTINT_INTR 68#define VIOIF_SOFTINT_INTR 1 69#endif 70 71/* 72 * if_vioifreg.h: 73 */ 74/* Configuration registers */ 75#define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 76#define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 77 78/* Feature bits */ 79#define VIRTIO_NET_F_CSUM (1<<0) 80#define VIRTIO_NET_F_GUEST_CSUM (1<<1) 81#define VIRTIO_NET_F_MAC (1<<5) 82#define VIRTIO_NET_F_GSO (1<<6) 83#define VIRTIO_NET_F_GUEST_TSO4 (1<<7) 84#define VIRTIO_NET_F_GUEST_TSO6 (1<<8) 85#define VIRTIO_NET_F_GUEST_ECN (1<<9) 86#define VIRTIO_NET_F_GUEST_UFO (1<<10) 87#define VIRTIO_NET_F_HOST_TSO4 (1<<11) 88#define VIRTIO_NET_F_HOST_TSO6 (1<<12) 89#define VIRTIO_NET_F_HOST_ECN (1<<13) 90#define VIRTIO_NET_F_HOST_UFO (1<<14) 91#define VIRTIO_NET_F_MRG_RXBUF (1<<15) 92#define VIRTIO_NET_F_STATUS (1<<16) 93#define VIRTIO_NET_F_CTRL_VQ (1<<17) 94#define VIRTIO_NET_F_CTRL_RX (1<<18) 95#define VIRTIO_NET_F_CTRL_VLAN (1<<19) 96 97#define VIRTIO_NET_FLAG_BITS \ 98 VIRTIO_COMMON_FLAG_BITS \ 99 "\x14""CTRL_VLAN" \ 100 "\x13""CTRL_RX" \ 101 "\x12""CTRL_VQ" \ 102 "\x11""STATUS" \ 103 "\x10""MRG_RXBUF" \ 104 "\x0f""HOST_UFO" \ 105 "\x0e""HOST_ECN" \ 106 "\x0d""HOST_TSO6" \ 107 "\x0c""HOST_TSO4" \ 108 "\x0b""GUEST_UFO" \ 109 "\x0a""GUEST_ECN" \ 110 "\x09""GUEST_TSO6" \ 111 "\x08""GUEST_TSO4" \ 112 "\x07""GSO" \ 113 "\x06""MAC" \ 114 "\x02""GUEST_CSUM" \ 115 "\x01""CSUM" 116 117/* Status */ 118#define VIRTIO_NET_S_LINK_UP 1 119 120/* Packet header structure */ 121struct virtio_net_hdr { 122 uint8_t flags; 123 uint8_t gso_type; 124 uint16_t hdr_len; 125 uint16_t gso_size; 126 uint16_t csum_start; 127 uint16_t csum_offset; 128#if 0 129 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 130#endif 131} __packed; 132 133#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 134#define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 135#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 136#define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 137#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 138#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 139 140#define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 141 142/* Control virtqueue */ 143struct virtio_net_ctrl_cmd { 144 uint8_t class; 145 uint8_t command; 146} __packed; 147#define VIRTIO_NET_CTRL_RX 0 148# define VIRTIO_NET_CTRL_RX_PROMISC 0 149# define VIRTIO_NET_CTRL_RX_ALLMULTI 1 150 151#define VIRTIO_NET_CTRL_MAC 1 152# define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 153 154#define VIRTIO_NET_CTRL_VLAN 2 155# define VIRTIO_NET_CTRL_VLAN_ADD 0 156# define VIRTIO_NET_CTRL_VLAN_DEL 1 157 158struct virtio_net_ctrl_status { 159 uint8_t ack; 160} __packed; 161#define VIRTIO_NET_OK 0 162#define VIRTIO_NET_ERR 1 163 164struct virtio_net_ctrl_rx { 165 uint8_t onoff; 166} __packed; 167 168struct virtio_net_ctrl_mac_tbl { 169 uint32_t nentries; 170 uint8_t macs[][ETHER_ADDR_LEN]; 171} __packed; 172 173struct virtio_net_ctrl_vlan { 174 uint16_t id; 175} __packed; 176 177 178/* 179 * if_vioifvar.h: 180 */ 181struct vioif_softc { 182 device_t sc_dev; 183 184 struct virtio_softc *sc_virtio; 185 struct virtqueue sc_vq[3]; 186#define VQ_RX 0 187#define VQ_TX 1 188#define VQ_CTRL 2 189 190 uint8_t sc_mac[ETHER_ADDR_LEN]; 191 struct ethercom sc_ethercom; 192 short sc_deferred_init_done; 193 bool sc_link_active; 194 195 /* bus_dmamem */ 196 bus_dma_segment_t sc_hdr_segs[1]; 197 struct virtio_net_hdr *sc_hdrs; 198#define sc_rx_hdrs sc_hdrs 199 struct virtio_net_hdr *sc_tx_hdrs; 200 struct virtio_net_ctrl_cmd *sc_ctrl_cmd; 201 struct virtio_net_ctrl_status *sc_ctrl_status; 202 struct virtio_net_ctrl_rx *sc_ctrl_rx; 203 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; 204 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; 205 206 /* kmem */ 207 bus_dmamap_t *sc_arrays; 208#define sc_rxhdr_dmamaps sc_arrays 209 bus_dmamap_t *sc_txhdr_dmamaps; 210 bus_dmamap_t *sc_rx_dmamaps; 211 bus_dmamap_t *sc_tx_dmamaps; 212 struct mbuf **sc_rx_mbufs; 213 struct mbuf **sc_tx_mbufs; 214 215 bus_dmamap_t sc_ctrl_cmd_dmamap; 216 bus_dmamap_t sc_ctrl_status_dmamap; 217 bus_dmamap_t sc_ctrl_rx_dmamap; 218 bus_dmamap_t sc_ctrl_tbl_uc_dmamap; 219 bus_dmamap_t sc_ctrl_tbl_mc_dmamap; 220 221 void *sc_rx_softint; 222 void *sc_ctl_softint; 223 224 enum { 225 FREE, INUSE, DONE 226 } sc_ctrl_inuse; 227 kcondvar_t sc_ctrl_wait; 228 kmutex_t sc_ctrl_wait_lock; 229 kmutex_t sc_tx_lock; 230 kmutex_t sc_rx_lock; 231 bool sc_stopping; 232 233 bool sc_has_ctrl; 234}; 235#define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 236#define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 237 238#define VIOIF_TX_LOCK(_sc) mutex_enter(&(_sc)->sc_tx_lock) 239#define VIOIF_TX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_tx_lock) 240#define VIOIF_TX_LOCKED(_sc) mutex_owned(&(_sc)->sc_tx_lock) 241#define VIOIF_RX_LOCK(_sc) mutex_enter(&(_sc)->sc_rx_lock) 242#define VIOIF_RX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_rx_lock) 243#define VIOIF_RX_LOCKED(_sc) mutex_owned(&(_sc)->sc_rx_lock) 244 245/* cfattach interface functions */ 246static int vioif_match(device_t, cfdata_t, void *); 247static void vioif_attach(device_t, device_t, void *); 248static void vioif_deferred_init(device_t); 249 250/* ifnet interface functions */ 251static int vioif_init(struct ifnet *); 252static void vioif_stop(struct ifnet *, int); 253static void vioif_start(struct ifnet *); 254static int vioif_ioctl(struct ifnet *, u_long, void *); 255static void vioif_watchdog(struct ifnet *); 256 257/* rx */ 258static int vioif_add_rx_mbuf(struct vioif_softc *, int); 259static void vioif_free_rx_mbuf(struct vioif_softc *, int); 260static void vioif_populate_rx_mbufs(struct vioif_softc *); 261static void vioif_populate_rx_mbufs_locked(struct vioif_softc *); 262static int vioif_rx_deq(struct vioif_softc *); 263static int vioif_rx_deq_locked(struct vioif_softc *); 264static int vioif_rx_vq_done(struct virtqueue *); 265static void vioif_rx_softint(void *); 266static void vioif_rx_drain(struct vioif_softc *); 267 268/* tx */ 269static int vioif_tx_vq_done(struct virtqueue *); 270static int vioif_tx_vq_done_locked(struct virtqueue *); 271static void vioif_tx_drain(struct vioif_softc *); 272 273/* other control */ 274static bool vioif_is_link_up(struct vioif_softc *); 275static void vioif_update_link_status(struct vioif_softc *); 276static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 277static int vioif_set_promisc(struct vioif_softc *, bool); 278static int vioif_set_allmulti(struct vioif_softc *, bool); 279static int vioif_set_rx_filter(struct vioif_softc *); 280static int vioif_rx_filter(struct vioif_softc *); 281static int vioif_ctrl_vq_done(struct virtqueue *); 282static int vioif_config_change(struct virtio_softc *); 283static void vioif_ctl_softint(void *); 284 285CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 286 vioif_match, vioif_attach, NULL, NULL); 287 288static int 289vioif_match(device_t parent, cfdata_t match, void *aux) 290{ 291 struct virtio_attach_args *va = aux; 292 293 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 294 return 1; 295 296 return 0; 297} 298 299/* allocate memory */ 300/* 301 * dma memory is used for: 302 * sc_rx_hdrs[slot]: metadata array for recieved frames (READ) 303 * sc_tx_hdrs[slot]: metadata array for frames to be sent (WRITE) 304 * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) 305 * sc_ctrl_status: return value for a command via ctrl vq (READ) 306 * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command 307 * (WRITE) 308 * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 309 * class command (WRITE) 310 * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 311 * class command (WRITE) 312 * sc_ctrl_* structures are allocated only one each; they are protected by 313 * sc_ctrl_inuse variable and sc_ctrl_wait condvar. 314 */ 315/* 316 * dynamically allocated memory is used for: 317 * sc_rxhdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 318 * sc_txhdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 319 * sc_rx_dmamaps[slot]: bus_dmamap_t array for recieved payload 320 * sc_tx_dmamaps[slot]: bus_dmamap_t array for sent payload 321 * sc_rx_mbufs[slot]: mbuf pointer array for recieved frames 322 * sc_tx_mbufs[slot]: mbuf pointer array for sent frames 323 */ 324static int 325vioif_alloc_mems(struct vioif_softc *sc) 326{ 327 struct virtio_softc *vsc = sc->sc_virtio; 328 int allocsize, allocsize2, r, rsegs, i; 329 void *vaddr; 330 intptr_t p; 331 int rxqsize, txqsize; 332 333 rxqsize = sc->sc_vq[VQ_RX].vq_num; 334 txqsize = sc->sc_vq[VQ_TX].vq_num; 335 336 allocsize = sizeof(struct virtio_net_hdr) * rxqsize; 337 allocsize += sizeof(struct virtio_net_hdr) * txqsize; 338 if (sc->sc_has_ctrl) { 339 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 340 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 341 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 342 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 343 + sizeof(struct virtio_net_ctrl_mac_tbl) 344 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 345 } 346 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 347 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 348 if (r != 0) { 349 aprint_error_dev(sc->sc_dev, 350 "DMA memory allocation failed, size %d, " 351 "error code %d\n", allocsize, r); 352 goto err_none; 353 } 354 r = bus_dmamem_map(virtio_dmat(vsc), 355 &sc->sc_hdr_segs[0], 1, allocsize, 356 &vaddr, BUS_DMA_NOWAIT); 357 if (r != 0) { 358 aprint_error_dev(sc->sc_dev, 359 "DMA memory map failed, " 360 "error code %d\n", r); 361 goto err_dmamem_alloc; 362 } 363 sc->sc_hdrs = vaddr; 364 memset(vaddr, 0, allocsize); 365 p = (intptr_t) vaddr; 366 p += sizeof(struct virtio_net_hdr) * rxqsize; 367#define P(name,size) do { sc->sc_ ##name = (void*) p; \ 368 p += size; } while (0) 369 P(tx_hdrs, sizeof(struct virtio_net_hdr) * txqsize); 370 if (sc->sc_has_ctrl) { 371 P(ctrl_cmd, sizeof(struct virtio_net_ctrl_cmd)); 372 P(ctrl_status, sizeof(struct virtio_net_ctrl_status)); 373 P(ctrl_rx, sizeof(struct virtio_net_ctrl_rx)); 374 P(ctrl_mac_tbl_uc, sizeof(struct virtio_net_ctrl_mac_tbl)); 375 P(ctrl_mac_tbl_mc, 376 (sizeof(struct virtio_net_ctrl_mac_tbl) 377 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); 378 } 379#undef P 380 381 allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize); 382 allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize); 383 allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize); 384 sc->sc_arrays = kmem_zalloc(allocsize2, KM_SLEEP); 385 if (sc->sc_arrays == NULL) 386 goto err_dmamem_map; 387 sc->sc_txhdr_dmamaps = sc->sc_arrays + rxqsize; 388 sc->sc_rx_dmamaps = sc->sc_txhdr_dmamaps + txqsize; 389 sc->sc_tx_dmamaps = sc->sc_rx_dmamaps + rxqsize; 390 sc->sc_rx_mbufs = (void*) (sc->sc_tx_dmamaps + txqsize); 391 sc->sc_tx_mbufs = sc->sc_rx_mbufs + rxqsize; 392 393#define C(map, buf, size, nsegs, rw, usage) \ 394 do { \ 395 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ 396 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ 397 &sc->sc_ ##map); \ 398 if (r != 0) { \ 399 aprint_error_dev(sc->sc_dev, \ 400 usage " dmamap creation failed, " \ 401 "error code %d\n", r); \ 402 goto err_reqs; \ 403 } \ 404 } while (0) 405#define C_L1(map, buf, size, nsegs, rw, usage) \ 406 C(map, buf, size, nsegs, rw, usage); \ 407 do { \ 408 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 409 &sc->sc_ ##buf, size, NULL, \ 410 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 411 if (r != 0) { \ 412 aprint_error_dev(sc->sc_dev, \ 413 usage " dmamap load failed, " \ 414 "error code %d\n", r); \ 415 goto err_reqs; \ 416 } \ 417 } while (0) 418#define C_L2(map, buf, size, nsegs, rw, usage) \ 419 C(map, buf, size, nsegs, rw, usage); \ 420 do { \ 421 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 422 sc->sc_ ##buf, size, NULL, \ 423 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 424 if (r != 0) { \ 425 aprint_error_dev(sc->sc_dev, \ 426 usage " dmamap load failed, " \ 427 "error code %d\n", r); \ 428 goto err_reqs; \ 429 } \ 430 } while (0) 431 for (i = 0; i < rxqsize; i++) { 432 C_L1(rxhdr_dmamaps[i], rx_hdrs[i], 433 sizeof(struct virtio_net_hdr), 1, 434 READ, "rx header"); 435 C(rx_dmamaps[i], NULL, MCLBYTES, 1, 0, "rx payload"); 436 } 437 438 for (i = 0; i < txqsize; i++) { 439 C_L1(txhdr_dmamaps[i], tx_hdrs[i], 440 sizeof(struct virtio_net_hdr), 1, 441 WRITE, "tx header"); 442 C(tx_dmamaps[i], NULL, ETHER_MAX_LEN, VIRTIO_NET_TX_MAXNSEGS, 0, 443 "tx payload"); 444 } 445 446 if (sc->sc_has_ctrl) { 447 /* control vq class & command */ 448 C_L2(ctrl_cmd_dmamap, ctrl_cmd, 449 sizeof(struct virtio_net_ctrl_cmd), 1, WRITE, 450 "control command"); 451 452 /* control vq status */ 453 C_L2(ctrl_status_dmamap, ctrl_status, 454 sizeof(struct virtio_net_ctrl_status), 1, READ, 455 "control status"); 456 457 /* control vq rx mode command parameter */ 458 C_L2(ctrl_rx_dmamap, ctrl_rx, 459 sizeof(struct virtio_net_ctrl_rx), 1, WRITE, 460 "rx mode control command"); 461 462 /* control vq MAC filter table for unicast */ 463 /* do not load now since its length is variable */ 464 C(ctrl_tbl_uc_dmamap, NULL, 465 sizeof(struct virtio_net_ctrl_mac_tbl) + 0, 1, WRITE, 466 "unicast MAC address filter command"); 467 468 /* control vq MAC filter table for multicast */ 469 C(ctrl_tbl_mc_dmamap, NULL, 470 (sizeof(struct virtio_net_ctrl_mac_tbl) 471 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES), 472 1, WRITE, "multicast MAC address filter command"); 473 } 474#undef C_L2 475#undef C_L1 476#undef C 477 478 return 0; 479 480err_reqs: 481#define D(map) \ 482 do { \ 483 if (sc->sc_ ##map) { \ 484 bus_dmamap_destroy(virtio_dmat(vsc), sc->sc_ ##map); \ 485 sc->sc_ ##map = NULL; \ 486 } \ 487 } while (0) 488 D(ctrl_tbl_mc_dmamap); 489 D(ctrl_tbl_uc_dmamap); 490 D(ctrl_rx_dmamap); 491 D(ctrl_status_dmamap); 492 D(ctrl_cmd_dmamap); 493 for (i = 0; i < txqsize; i++) { 494 D(tx_dmamaps[i]); 495 D(txhdr_dmamaps[i]); 496 } 497 for (i = 0; i < rxqsize; i++) { 498 D(rx_dmamaps[i]); 499 D(rxhdr_dmamaps[i]); 500 } 501#undef D 502 if (sc->sc_arrays) { 503 kmem_free(sc->sc_arrays, allocsize2); 504 sc->sc_arrays = 0; 505 } 506err_dmamem_map: 507 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_hdrs, allocsize); 508err_dmamem_alloc: 509 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 510err_none: 511 return -1; 512} 513 514static void 515vioif_attach(device_t parent, device_t self, void *aux) 516{ 517 struct vioif_softc *sc = device_private(self); 518 struct virtio_softc *vsc = device_private(parent); 519 uint32_t features; 520 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 521 u_int flags; 522 int r, nvqs=0, req_flags; 523 524 if (virtio_child(vsc) != NULL) { 525 aprint_normal(": child already attached for %s; " 526 "something wrong...\n", 527 device_xname(parent)); 528 return; 529 } 530 531 sc->sc_dev = self; 532 sc->sc_virtio = vsc; 533 sc->sc_link_active = false; 534 535 req_flags = 0; 536 537#ifdef VIOIF_MPSAFE 538 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 539#endif 540#ifdef VIOIF_SOFTINT_INTR 541 req_flags |= VIRTIO_F_PCI_INTR_SOFTINT; 542#endif 543 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 544 545 virtio_child_attach_start(vsc, self, IPL_NET, sc->sc_vq, 546 vioif_config_change, virtio_vq_intr, req_flags, 547 (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 548 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY), 549 VIRTIO_NET_FLAG_BITS); 550 551 features = virtio_features(vsc); 552 553 if (features & VIRTIO_NET_F_MAC) { 554 sc->sc_mac[0] = virtio_read_device_config_1(vsc, 555 VIRTIO_NET_CONFIG_MAC+0); 556 sc->sc_mac[1] = virtio_read_device_config_1(vsc, 557 VIRTIO_NET_CONFIG_MAC+1); 558 sc->sc_mac[2] = virtio_read_device_config_1(vsc, 559 VIRTIO_NET_CONFIG_MAC+2); 560 sc->sc_mac[3] = virtio_read_device_config_1(vsc, 561 VIRTIO_NET_CONFIG_MAC+3); 562 sc->sc_mac[4] = virtio_read_device_config_1(vsc, 563 VIRTIO_NET_CONFIG_MAC+4); 564 sc->sc_mac[5] = virtio_read_device_config_1(vsc, 565 VIRTIO_NET_CONFIG_MAC+5); 566 } else { 567 /* code stolen from sys/net/if_tap.c */ 568 struct timeval tv; 569 uint32_t ui; 570 getmicrouptime(&tv); 571 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 572 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 573 virtio_write_device_config_1(vsc, 574 VIRTIO_NET_CONFIG_MAC+0, 575 sc->sc_mac[0]); 576 virtio_write_device_config_1(vsc, 577 VIRTIO_NET_CONFIG_MAC+1, 578 sc->sc_mac[1]); 579 virtio_write_device_config_1(vsc, 580 VIRTIO_NET_CONFIG_MAC+2, 581 sc->sc_mac[2]); 582 virtio_write_device_config_1(vsc, 583 VIRTIO_NET_CONFIG_MAC+3, 584 sc->sc_mac[3]); 585 virtio_write_device_config_1(vsc, 586 VIRTIO_NET_CONFIG_MAC+4, 587 sc->sc_mac[4]); 588 virtio_write_device_config_1(vsc, 589 VIRTIO_NET_CONFIG_MAC+5, 590 sc->sc_mac[5]); 591 } 592 593 aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); 594 595 mutex_init(&sc->sc_tx_lock, MUTEX_DEFAULT, IPL_NET); 596 mutex_init(&sc->sc_rx_lock, MUTEX_DEFAULT, IPL_NET); 597 sc->sc_stopping = false; 598 599 /* 600 * Allocating a virtqueue for Rx 601 */ 602 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_RX], VQ_RX, 603 MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); 604 if (r != 0) 605 goto err; 606 nvqs = 1; 607 sc->sc_vq[VQ_RX].vq_done = vioif_rx_vq_done; 608 609 /* 610 * Allocating a virtqueue for Tx 611 */ 612 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_TX], VQ_TX, 613 (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), 614 VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); 615 if (r != 0) 616 goto err; 617 nvqs = 2; 618 sc->sc_vq[VQ_TX].vq_done = vioif_tx_vq_done; 619 620 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 621 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); /* not urgent; do it later */ 622 623 if ((features & VIRTIO_NET_F_CTRL_VQ) && 624 (features & VIRTIO_NET_F_CTRL_RX)) { 625 /* 626 * Allocating a virtqueue for control channel 627 */ 628 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_CTRL], VQ_CTRL, 629 NBPG, 1, "control"); 630 if (r != 0) { 631 aprint_error_dev(self, "failed to allocate " 632 "a virtqueue for control channel\n"); 633 goto skip; 634 } 635 636 sc->sc_vq[VQ_CTRL].vq_done = vioif_ctrl_vq_done; 637 cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); 638 mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); 639 sc->sc_ctrl_inuse = FREE; 640 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 641 sc->sc_has_ctrl = true; 642 nvqs = 3; 643 } 644skip: 645 646#ifdef VIOIF_MPSAFE 647 flags = SOFTINT_NET | SOFTINT_MPSAFE; 648#else 649 flags = SOFTINT_NET; 650#endif 651 sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc); 652 if (sc->sc_rx_softint == NULL) { 653 aprint_error_dev(self, "cannot establish rx softint\n"); 654 goto err; 655 } 656 657 sc->sc_ctl_softint = softint_establish(flags, vioif_ctl_softint, sc); 658 if (sc->sc_ctl_softint == NULL) { 659 aprint_error_dev(self, "cannot establish ctl softint\n"); 660 goto err; 661 } 662 663 if (vioif_alloc_mems(sc) < 0) 664 goto err; 665 666 if (virtio_child_attach_finish(vsc) != 0) 667 goto err; 668 669 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 670 ifp->if_softc = sc; 671 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 672 ifp->if_start = vioif_start; 673 ifp->if_ioctl = vioif_ioctl; 674 ifp->if_init = vioif_init; 675 ifp->if_stop = vioif_stop; 676 ifp->if_capabilities = 0; 677 ifp->if_watchdog = vioif_watchdog; 678 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(sc->sc_vq[VQ_TX].vq_num, IFQ_MAXLEN)); 679 IFQ_SET_READY(&ifp->if_snd); 680 681 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 682 683 if_attach(ifp); 684 if_deferred_start_init(ifp, NULL); 685 ether_ifattach(ifp, sc->sc_mac); 686 687 return; 688 689err: 690 mutex_destroy(&sc->sc_tx_lock); 691 mutex_destroy(&sc->sc_rx_lock); 692 693 if (sc->sc_has_ctrl) { 694 cv_destroy(&sc->sc_ctrl_wait); 695 mutex_destroy(&sc->sc_ctrl_wait_lock); 696 } 697 698 while (nvqs > 0) 699 virtio_free_vq(vsc, &sc->sc_vq[--nvqs]); 700 701 virtio_child_attach_failed(vsc); 702 return; 703} 704 705/* we need interrupts to make promiscuous mode off */ 706static void 707vioif_deferred_init(device_t self) 708{ 709 struct vioif_softc *sc = device_private(self); 710 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 711 int r; 712 713 if (ifp->if_flags & IFF_PROMISC) 714 return; 715 716 r = vioif_set_promisc(sc, false); 717 if (r != 0) 718 aprint_error_dev(self, "resetting promisc mode failed, " 719 "errror code %d\n", r); 720} 721 722/* 723 * Interface functions for ifnet 724 */ 725static int 726vioif_init(struct ifnet *ifp) 727{ 728 struct vioif_softc *sc = ifp->if_softc; 729 struct virtio_softc *vsc = sc->sc_virtio; 730 731 vioif_stop(ifp, 0); 732 733 virtio_reinit_start(vsc); 734 virtio_negotiate_features(vsc, virtio_features(vsc)); 735 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 736 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 737 if (sc->sc_has_ctrl) 738 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 739 virtio_reinit_end(vsc); 740 741 if (!sc->sc_deferred_init_done) { 742 sc->sc_deferred_init_done = 1; 743 if (sc->sc_has_ctrl) 744 vioif_deferred_init(sc->sc_dev); 745 } 746 747 /* Have to set false before vioif_populate_rx_mbufs */ 748 sc->sc_stopping = false; 749 750 vioif_populate_rx_mbufs(sc); 751 752 vioif_update_link_status(sc); 753 ifp->if_flags |= IFF_RUNNING; 754 ifp->if_flags &= ~IFF_OACTIVE; 755 vioif_rx_filter(sc); 756 757 return 0; 758} 759 760static void 761vioif_stop(struct ifnet *ifp, int disable) 762{ 763 struct vioif_softc *sc = ifp->if_softc; 764 struct virtio_softc *vsc = sc->sc_virtio; 765 766 /* Take the locks to ensure that ongoing TX/RX finish */ 767 VIOIF_TX_LOCK(sc); 768 VIOIF_RX_LOCK(sc); 769 sc->sc_stopping = true; 770 VIOIF_RX_UNLOCK(sc); 771 VIOIF_TX_UNLOCK(sc); 772 773 /* disable interrupts */ 774 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 775 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 776 if (sc->sc_has_ctrl) 777 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 778 779 /* only way to stop I/O and DMA is resetting... */ 780 virtio_reset(vsc); 781 vioif_rx_deq(sc); 782 vioif_tx_drain(sc); 783 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 784 sc->sc_link_active = false; 785 786 if (disable) 787 vioif_rx_drain(sc); 788} 789 790static void 791vioif_start(struct ifnet *ifp) 792{ 793 struct vioif_softc *sc = ifp->if_softc; 794 struct virtio_softc *vsc = sc->sc_virtio; 795 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 796 struct mbuf *m; 797 int queued = 0; 798 799 VIOIF_TX_LOCK(sc); 800 801 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING || 802 !sc->sc_link_active) 803 goto out; 804 805 if (sc->sc_stopping) 806 goto out; 807 808 for (;;) { 809 int slot, r; 810 811 IFQ_DEQUEUE(&ifp->if_snd, m); 812 if (m == NULL) 813 break; 814 815 r = virtio_enqueue_prep(vsc, vq, &slot); 816 if (r == EAGAIN) { 817 ifp->if_flags |= IFF_OACTIVE; 818 m_freem(m); 819 break; 820 } 821 if (r != 0) 822 panic("enqueue_prep for a tx buffer"); 823 824 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 825 sc->sc_tx_dmamaps[slot], 826 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 827 if (r != 0) { 828 /* maybe just too fragmented */ 829 struct mbuf *newm; 830 831 newm = m_defrag(m, M_NOWAIT); 832 if (newm == NULL) { 833 aprint_error_dev(sc->sc_dev, 834 "m_defrag() failed\n"); 835 goto skip; 836 } 837 838 m = newm; 839 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 840 sc->sc_tx_dmamaps[slot], 841 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 842 if (r != 0) { 843 aprint_error_dev(sc->sc_dev, 844 "tx dmamap load failed, error code %d\n", 845 r); 846skip: 847 m_freem(m); 848 virtio_enqueue_abort(vsc, vq, slot); 849 continue; 850 } 851 } 852 853 /* This should actually never fail */ 854 r = virtio_enqueue_reserve(vsc, vq, slot, 855 sc->sc_tx_dmamaps[slot]->dm_nsegs + 1); 856 if (r != 0) { 857 aprint_error_dev(sc->sc_dev, 858 "virtio_enqueue_reserve failed, error code %d\n", 859 r); 860 bus_dmamap_unload(virtio_dmat(vsc), 861 sc->sc_tx_dmamaps[slot]); 862 /* slot already freed by virtio_enqueue_reserve */ 863 m_freem(m); 864 continue; 865 } 866 867 sc->sc_tx_mbufs[slot] = m; 868 869 memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 870 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 871 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 872 BUS_DMASYNC_PREWRITE); 873 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 874 0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize, 875 BUS_DMASYNC_PREWRITE); 876 virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true); 877 virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true); 878 virtio_enqueue_commit(vsc, vq, slot, false); 879 880 queued++; 881 bpf_mtap(ifp, m); 882 } 883 884 if (queued > 0) { 885 virtio_enqueue_commit(vsc, vq, -1, true); 886 ifp->if_timer = 5; 887 } 888 889out: 890 VIOIF_TX_UNLOCK(sc); 891} 892 893static int 894vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 895{ 896 int s, r; 897 898 s = splnet(); 899 900 r = ether_ioctl(ifp, cmd, data); 901 if ((r == 0 && cmd == SIOCSIFFLAGS) || 902 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 903 if (ifp->if_flags & IFF_RUNNING) 904 r = vioif_rx_filter(ifp->if_softc); 905 else 906 r = 0; 907 } 908 909 splx(s); 910 911 return r; 912} 913 914void 915vioif_watchdog(struct ifnet *ifp) 916{ 917 struct vioif_softc *sc = ifp->if_softc; 918 919 if (ifp->if_flags & IFF_RUNNING) 920 vioif_tx_vq_done(&sc->sc_vq[VQ_TX]); 921} 922 923 924/* 925 * Recieve implementation 926 */ 927/* allocate and initialize a mbuf for recieve */ 928static int 929vioif_add_rx_mbuf(struct vioif_softc *sc, int i) 930{ 931 struct mbuf *m; 932 int r; 933 934 MGETHDR(m, M_DONTWAIT, MT_DATA); 935 if (m == NULL) 936 return ENOBUFS; 937 MCLGET(m, M_DONTWAIT); 938 if ((m->m_flags & M_EXT) == 0) { 939 m_freem(m); 940 return ENOBUFS; 941 } 942 sc->sc_rx_mbufs[i] = m; 943 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 944 r = bus_dmamap_load_mbuf(virtio_dmat(sc->sc_virtio), 945 sc->sc_rx_dmamaps[i], 946 m, BUS_DMA_READ|BUS_DMA_NOWAIT); 947 if (r) { 948 m_freem(m); 949 sc->sc_rx_mbufs[i] = 0; 950 return r; 951 } 952 953 return 0; 954} 955 956/* free a mbuf for recieve */ 957static void 958vioif_free_rx_mbuf(struct vioif_softc *sc, int i) 959{ 960 bus_dmamap_unload(virtio_dmat(sc->sc_virtio), sc->sc_rx_dmamaps[i]); 961 m_freem(sc->sc_rx_mbufs[i]); 962 sc->sc_rx_mbufs[i] = NULL; 963} 964 965/* add mbufs for all the empty recieve slots */ 966static void 967vioif_populate_rx_mbufs(struct vioif_softc *sc) 968{ 969 VIOIF_RX_LOCK(sc); 970 vioif_populate_rx_mbufs_locked(sc); 971 VIOIF_RX_UNLOCK(sc); 972} 973 974static void 975vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) 976{ 977 struct virtio_softc *vsc = sc->sc_virtio; 978 int i, r, ndone = 0; 979 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 980 981 KASSERT(VIOIF_RX_LOCKED(sc)); 982 983 if (sc->sc_stopping) 984 return; 985 986 for (i = 0; i < vq->vq_num; i++) { 987 int slot; 988 r = virtio_enqueue_prep(vsc, vq, &slot); 989 if (r == EAGAIN) 990 break; 991 if (r != 0) 992 panic("enqueue_prep for rx buffers"); 993 if (sc->sc_rx_mbufs[slot] == NULL) { 994 r = vioif_add_rx_mbuf(sc, slot); 995 if (r != 0) { 996 printf("%s: rx mbuf allocation failed, " 997 "error code %d\n", 998 device_xname(sc->sc_dev), r); 999 break; 1000 } 1001 } 1002 r = virtio_enqueue_reserve(vsc, vq, slot, 1003 sc->sc_rx_dmamaps[slot]->dm_nsegs + 1); 1004 if (r != 0) { 1005 vioif_free_rx_mbuf(sc, slot); 1006 break; 1007 } 1008 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1009 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 1010 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1011 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1012 virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false); 1013 virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false); 1014 virtio_enqueue_commit(vsc, vq, slot, false); 1015 ndone++; 1016 } 1017 if (ndone > 0) 1018 virtio_enqueue_commit(vsc, vq, -1, true); 1019} 1020 1021/* dequeue recieved packets */ 1022static int 1023vioif_rx_deq(struct vioif_softc *sc) 1024{ 1025 int r; 1026 1027 KASSERT(sc->sc_stopping); 1028 1029 VIOIF_RX_LOCK(sc); 1030 r = vioif_rx_deq_locked(sc); 1031 VIOIF_RX_UNLOCK(sc); 1032 1033 return r; 1034} 1035 1036/* dequeue recieved packets */ 1037static int 1038vioif_rx_deq_locked(struct vioif_softc *sc) 1039{ 1040 struct virtio_softc *vsc = sc->sc_virtio; 1041 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1042 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1043 struct mbuf *m; 1044 int r = 0; 1045 int slot, len; 1046 1047 KASSERT(VIOIF_RX_LOCKED(sc)); 1048 1049 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1050 len -= sizeof(struct virtio_net_hdr); 1051 r = 1; 1052 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1053 0, sizeof(struct virtio_net_hdr), 1054 BUS_DMASYNC_POSTREAD); 1055 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1056 0, MCLBYTES, 1057 BUS_DMASYNC_POSTREAD); 1058 m = sc->sc_rx_mbufs[slot]; 1059 KASSERT(m != NULL); 1060 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot]); 1061 sc->sc_rx_mbufs[slot] = 0; 1062 virtio_dequeue_commit(vsc, vq, slot); 1063 m_set_rcvif(m, ifp); 1064 m->m_len = m->m_pkthdr.len = len; 1065 1066 VIOIF_RX_UNLOCK(sc); 1067 if_percpuq_enqueue(ifp->if_percpuq, m); 1068 VIOIF_RX_LOCK(sc); 1069 1070 if (sc->sc_stopping) 1071 break; 1072 } 1073 1074 return r; 1075} 1076 1077/* rx interrupt; call _dequeue above and schedule a softint */ 1078static int 1079vioif_rx_vq_done(struct virtqueue *vq) 1080{ 1081 struct virtio_softc *vsc = vq->vq_owner; 1082 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1083 int r = 0; 1084 1085#ifdef VIOIF_SOFTINT_INTR 1086 KASSERT(!cpu_intr_p()); 1087#endif 1088 1089 VIOIF_RX_LOCK(sc); 1090 1091 if (sc->sc_stopping) 1092 goto out; 1093 1094 r = vioif_rx_deq_locked(sc); 1095 if (r) 1096#ifdef VIOIF_SOFTINT_INTR 1097 vioif_populate_rx_mbufs_locked(sc); 1098#else 1099 softint_schedule(sc->sc_rx_softint); 1100#endif 1101 1102out: 1103 VIOIF_RX_UNLOCK(sc); 1104 return r; 1105} 1106 1107/* softint: enqueue recieve requests for new incoming packets */ 1108static void 1109vioif_rx_softint(void *arg) 1110{ 1111 struct vioif_softc *sc = arg; 1112 1113 vioif_populate_rx_mbufs(sc); 1114} 1115 1116/* free all the mbufs; called from if_stop(disable) */ 1117static void 1118vioif_rx_drain(struct vioif_softc *sc) 1119{ 1120 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1121 int i; 1122 1123 for (i = 0; i < vq->vq_num; i++) { 1124 if (sc->sc_rx_mbufs[i] == NULL) 1125 continue; 1126 vioif_free_rx_mbuf(sc, i); 1127 } 1128} 1129 1130 1131/* 1132 * Transmition implementation 1133 */ 1134/* actual transmission is done in if_start */ 1135/* tx interrupt; dequeue and free mbufs */ 1136/* 1137 * tx interrupt is actually disabled; this should be called upon 1138 * tx vq full and watchdog 1139 */ 1140static int 1141vioif_tx_vq_done(struct virtqueue *vq) 1142{ 1143 struct virtio_softc *vsc = vq->vq_owner; 1144 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1145 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1146 int r = 0; 1147 1148 VIOIF_TX_LOCK(sc); 1149 1150 if (sc->sc_stopping) 1151 goto out; 1152 1153 r = vioif_tx_vq_done_locked(vq); 1154 1155out: 1156 VIOIF_TX_UNLOCK(sc); 1157 if (r) 1158 if_schedule_deferred_start(ifp); 1159 return r; 1160} 1161 1162static int 1163vioif_tx_vq_done_locked(struct virtqueue *vq) 1164{ 1165 struct virtio_softc *vsc = vq->vq_owner; 1166 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1167 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1168 struct mbuf *m; 1169 int r = 0; 1170 int slot, len; 1171 1172 KASSERT(VIOIF_TX_LOCKED(sc)); 1173 1174 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1175 r++; 1176 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 1177 0, sizeof(struct virtio_net_hdr), 1178 BUS_DMASYNC_POSTWRITE); 1179 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 1180 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 1181 BUS_DMASYNC_POSTWRITE); 1182 m = sc->sc_tx_mbufs[slot]; 1183 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot]); 1184 sc->sc_tx_mbufs[slot] = 0; 1185 virtio_dequeue_commit(vsc, vq, slot); 1186 ifp->if_opackets++; 1187 m_freem(m); 1188 } 1189 1190 if (r) 1191 ifp->if_flags &= ~IFF_OACTIVE; 1192 return r; 1193} 1194 1195/* free all the mbufs already put on vq; called from if_stop(disable) */ 1196static void 1197vioif_tx_drain(struct vioif_softc *sc) 1198{ 1199 struct virtio_softc *vsc = sc->sc_virtio; 1200 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 1201 int i; 1202 1203 KASSERT(sc->sc_stopping); 1204 1205 for (i = 0; i < vq->vq_num; i++) { 1206 if (sc->sc_tx_mbufs[i] == NULL) 1207 continue; 1208 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[i]); 1209 m_freem(sc->sc_tx_mbufs[i]); 1210 sc->sc_tx_mbufs[i] = NULL; 1211 } 1212} 1213 1214/* 1215 * Control vq 1216 */ 1217/* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1218static int 1219vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 1220{ 1221 struct virtio_softc *vsc = sc->sc_virtio; 1222 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1223 int r, slot; 1224 1225 if (!sc->sc_has_ctrl) 1226 return ENOTSUP; 1227 1228 mutex_enter(&sc->sc_ctrl_wait_lock); 1229 while (sc->sc_ctrl_inuse != FREE) 1230 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1231 sc->sc_ctrl_inuse = INUSE; 1232 mutex_exit(&sc->sc_ctrl_wait_lock); 1233 1234 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX; 1235 sc->sc_ctrl_cmd->command = cmd; 1236 sc->sc_ctrl_rx->onoff = onoff; 1237 1238 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1239 0, sizeof(struct virtio_net_ctrl_cmd), 1240 BUS_DMASYNC_PREWRITE); 1241 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 1242 0, sizeof(struct virtio_net_ctrl_rx), 1243 BUS_DMASYNC_PREWRITE); 1244 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1245 0, sizeof(struct virtio_net_ctrl_status), 1246 BUS_DMASYNC_PREREAD); 1247 1248 r = virtio_enqueue_prep(vsc, vq, &slot); 1249 if (r != 0) 1250 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1251 r = virtio_enqueue_reserve(vsc, vq, slot, 3); 1252 if (r != 0) 1253 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1254 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1255 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true); 1256 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1257 virtio_enqueue_commit(vsc, vq, slot, true); 1258 1259 /* wait for done */ 1260 mutex_enter(&sc->sc_ctrl_wait_lock); 1261 while (sc->sc_ctrl_inuse != DONE) 1262 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1263 mutex_exit(&sc->sc_ctrl_wait_lock); 1264 /* already dequeueued */ 1265 1266 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1267 sizeof(struct virtio_net_ctrl_cmd), 1268 BUS_DMASYNC_POSTWRITE); 1269 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 0, 1270 sizeof(struct virtio_net_ctrl_rx), 1271 BUS_DMASYNC_POSTWRITE); 1272 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1273 sizeof(struct virtio_net_ctrl_status), 1274 BUS_DMASYNC_POSTREAD); 1275 1276 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1277 r = 0; 1278 else { 1279 printf("%s: failed setting rx mode\n", 1280 device_xname(sc->sc_dev)); 1281 r = EIO; 1282 } 1283 1284 mutex_enter(&sc->sc_ctrl_wait_lock); 1285 sc->sc_ctrl_inuse = FREE; 1286 cv_signal(&sc->sc_ctrl_wait); 1287 mutex_exit(&sc->sc_ctrl_wait_lock); 1288 1289 return r; 1290} 1291 1292static int 1293vioif_set_promisc(struct vioif_softc *sc, bool onoff) 1294{ 1295 int r; 1296 1297 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 1298 1299 return r; 1300} 1301 1302static int 1303vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 1304{ 1305 int r; 1306 1307 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 1308 1309 return r; 1310} 1311 1312/* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 1313static int 1314vioif_set_rx_filter(struct vioif_softc *sc) 1315{ 1316 /* filter already set in sc_ctrl_mac_tbl */ 1317 struct virtio_softc *vsc = sc->sc_virtio; 1318 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1319 int r, slot; 1320 1321 if (!sc->sc_has_ctrl) 1322 return ENOTSUP; 1323 1324 mutex_enter(&sc->sc_ctrl_wait_lock); 1325 while (sc->sc_ctrl_inuse != FREE) 1326 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1327 sc->sc_ctrl_inuse = INUSE; 1328 mutex_exit(&sc->sc_ctrl_wait_lock); 1329 1330 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC; 1331 sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1332 1333 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 1334 sc->sc_ctrl_mac_tbl_uc, 1335 (sizeof(struct virtio_net_ctrl_mac_tbl) 1336 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1337 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1338 if (r) { 1339 printf("%s: control command dmamap load failed, " 1340 "error code %d\n", device_xname(sc->sc_dev), r); 1341 goto out; 1342 } 1343 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 1344 sc->sc_ctrl_mac_tbl_mc, 1345 (sizeof(struct virtio_net_ctrl_mac_tbl) 1346 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1347 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1348 if (r) { 1349 printf("%s: control command dmamap load failed, " 1350 "error code %d\n", device_xname(sc->sc_dev), r); 1351 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1352 goto out; 1353 } 1354 1355 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1356 0, sizeof(struct virtio_net_ctrl_cmd), 1357 BUS_DMASYNC_PREWRITE); 1358 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1359 (sizeof(struct virtio_net_ctrl_mac_tbl) 1360 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1361 BUS_DMASYNC_PREWRITE); 1362 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1363 (sizeof(struct virtio_net_ctrl_mac_tbl) 1364 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1365 BUS_DMASYNC_PREWRITE); 1366 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1367 0, sizeof(struct virtio_net_ctrl_status), 1368 BUS_DMASYNC_PREREAD); 1369 1370 r = virtio_enqueue_prep(vsc, vq, &slot); 1371 if (r != 0) 1372 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1373 r = virtio_enqueue_reserve(vsc, vq, slot, 4); 1374 if (r != 0) 1375 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1376 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1377 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true); 1378 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true); 1379 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1380 virtio_enqueue_commit(vsc, vq, slot, true); 1381 1382 /* wait for done */ 1383 mutex_enter(&sc->sc_ctrl_wait_lock); 1384 while (sc->sc_ctrl_inuse != DONE) 1385 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1386 mutex_exit(&sc->sc_ctrl_wait_lock); 1387 /* already dequeueued */ 1388 1389 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1390 sizeof(struct virtio_net_ctrl_cmd), 1391 BUS_DMASYNC_POSTWRITE); 1392 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1393 (sizeof(struct virtio_net_ctrl_mac_tbl) 1394 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1395 BUS_DMASYNC_POSTWRITE); 1396 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1397 (sizeof(struct virtio_net_ctrl_mac_tbl) 1398 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1399 BUS_DMASYNC_POSTWRITE); 1400 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1401 sizeof(struct virtio_net_ctrl_status), 1402 BUS_DMASYNC_POSTREAD); 1403 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1404 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap); 1405 1406 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1407 r = 0; 1408 else { 1409 printf("%s: failed setting rx filter\n", 1410 device_xname(sc->sc_dev)); 1411 r = EIO; 1412 } 1413 1414out: 1415 mutex_enter(&sc->sc_ctrl_wait_lock); 1416 sc->sc_ctrl_inuse = FREE; 1417 cv_signal(&sc->sc_ctrl_wait); 1418 mutex_exit(&sc->sc_ctrl_wait_lock); 1419 1420 return r; 1421} 1422 1423/* ctrl vq interrupt; wake up the command issuer */ 1424static int 1425vioif_ctrl_vq_done(struct virtqueue *vq) 1426{ 1427 struct virtio_softc *vsc = vq->vq_owner; 1428 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1429 int r, slot; 1430 1431 r = virtio_dequeue(vsc, vq, &slot, NULL); 1432 if (r == ENOENT) 1433 return 0; 1434 virtio_dequeue_commit(vsc, vq, slot); 1435 1436 mutex_enter(&sc->sc_ctrl_wait_lock); 1437 sc->sc_ctrl_inuse = DONE; 1438 cv_signal(&sc->sc_ctrl_wait); 1439 mutex_exit(&sc->sc_ctrl_wait_lock); 1440 1441 return 1; 1442} 1443 1444/* 1445 * If IFF_PROMISC requested, set promiscuous 1446 * If multicast filter small enough (<=MAXENTRIES) set rx filter 1447 * If large multicast filter exist use ALLMULTI 1448 */ 1449/* 1450 * If setting rx filter fails fall back to ALLMULTI 1451 * If ALLMULTI fails fall back to PROMISC 1452 */ 1453static int 1454vioif_rx_filter(struct vioif_softc *sc) 1455{ 1456 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1457 struct ether_multi *enm; 1458 struct ether_multistep step; 1459 int nentries; 1460 int promisc = 0, allmulti = 0, rxfilter = 0; 1461 int r; 1462 1463 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 1464 ifp->if_flags |= IFF_PROMISC; 1465 return 0; 1466 } 1467 1468 if (ifp->if_flags & IFF_PROMISC) { 1469 promisc = 1; 1470 goto set; 1471 } 1472 1473 nentries = -1; 1474 ETHER_LOCK(&sc->sc_ethercom); 1475 ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm); 1476 while (nentries++, enm != NULL) { 1477 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 1478 allmulti = 1; 1479 goto set_unlock; 1480 } 1481 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1482 ETHER_ADDR_LEN)) { 1483 allmulti = 1; 1484 goto set_unlock; 1485 } 1486 memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries], 1487 enm->enm_addrlo, ETHER_ADDR_LEN); 1488 ETHER_NEXT_MULTI(step, enm); 1489 } 1490 rxfilter = 1; 1491 1492set_unlock: 1493 ETHER_UNLOCK(&sc->sc_ethercom); 1494 1495set: 1496 if (rxfilter) { 1497 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1498 sc->sc_ctrl_mac_tbl_mc->nentries = nentries; 1499 r = vioif_set_rx_filter(sc); 1500 if (r != 0) { 1501 rxfilter = 0; 1502 allmulti = 1; /* fallback */ 1503 } 1504 } else { 1505 /* remove rx filter */ 1506 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1507 sc->sc_ctrl_mac_tbl_mc->nentries = 0; 1508 r = vioif_set_rx_filter(sc); 1509 /* what to do on failure? */ 1510 } 1511 if (allmulti) { 1512 r = vioif_set_allmulti(sc, true); 1513 if (r != 0) { 1514 allmulti = 0; 1515 promisc = 1; /* fallback */ 1516 } 1517 } else { 1518 r = vioif_set_allmulti(sc, false); 1519 /* what to do on failure? */ 1520 } 1521 if (promisc) { 1522 r = vioif_set_promisc(sc, true); 1523 } else { 1524 r = vioif_set_promisc(sc, false); 1525 } 1526 1527 return r; 1528} 1529 1530static bool 1531vioif_is_link_up(struct vioif_softc *sc) 1532{ 1533 struct virtio_softc *vsc = sc->sc_virtio; 1534 uint16_t status; 1535 1536 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 1537 status = virtio_read_device_config_2(vsc, 1538 VIRTIO_NET_CONFIG_STATUS); 1539 else 1540 status = VIRTIO_NET_S_LINK_UP; 1541 1542 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 1543} 1544 1545/* change link status */ 1546static void 1547vioif_update_link_status(struct vioif_softc *sc) 1548{ 1549 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1550 bool active, changed; 1551 int link; 1552 1553 active = vioif_is_link_up(sc); 1554 changed = false; 1555 1556 VIOIF_TX_LOCK(sc); 1557 if (active) { 1558 if (!sc->sc_link_active) 1559 changed = true; 1560 1561 link = LINK_STATE_UP; 1562 sc->sc_link_active = true; 1563 } else { 1564 if (sc->sc_link_active) 1565 changed = true; 1566 1567 link = LINK_STATE_DOWN; 1568 sc->sc_link_active = false; 1569 } 1570 VIOIF_TX_UNLOCK(sc); 1571 1572 if (changed) 1573 if_link_state_change(ifp, link); 1574} 1575 1576static int 1577vioif_config_change(struct virtio_softc *vsc) 1578{ 1579 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1580 1581#ifdef VIOIF_SOFTINT_INTR 1582 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1583#endif 1584 1585#ifdef VIOIF_SOFTINT_INTR 1586 KASSERT(!cpu_intr_p()); 1587 vioif_update_link_status(sc); 1588 vioif_start(ifp); 1589#else 1590 softint_schedule(sc->sc_ctl_softint); 1591#endif 1592 1593 return 0; 1594} 1595 1596static void 1597vioif_ctl_softint(void *arg) 1598{ 1599 struct vioif_softc *sc = arg; 1600 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1601 1602 vioif_update_link_status(sc); 1603 vioif_start(ifp); 1604} 1605 1606MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 1607 1608#ifdef _MODULE 1609#include "ioconf.c" 1610#endif 1611 1612static int 1613if_vioif_modcmd(modcmd_t cmd, void *opaque) 1614{ 1615 int error = 0; 1616 1617#ifdef _MODULE 1618 switch (cmd) { 1619 case MODULE_CMD_INIT: 1620 error = config_init_component(cfdriver_ioconf_if_vioif, 1621 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1622 break; 1623 case MODULE_CMD_FINI: 1624 error = config_fini_component(cfdriver_ioconf_if_vioif, 1625 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1626 break; 1627 default: 1628 error = ENOTTY; 1629 break; 1630 } 1631#endif 1632 1633 return error; 1634} 1635