if_vioif.c revision 1.65
1/* $NetBSD: if_vioif.c,v 1.65 2020/05/28 23:25:17 riastradh Exp $ */ 2 3/* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.65 2020/05/28 23:25:17 riastradh Exp $"); 30 31#ifdef _KERNEL_OPT 32#include "opt_net_mpsafe.h" 33#endif 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/atomic.h> 39#include <sys/bus.h> 40#include <sys/condvar.h> 41#include <sys/device.h> 42#include <sys/evcnt.h> 43#include <sys/intr.h> 44#include <sys/kmem.h> 45#include <sys/mbuf.h> 46#include <sys/mutex.h> 47#include <sys/sockio.h> 48#include <sys/cpu.h> 49#include <sys/module.h> 50#include <sys/pcq.h> 51#include <sys/workqueue.h> 52 53#include <dev/pci/virtioreg.h> 54#include <dev/pci/virtiovar.h> 55 56#include <net/if.h> 57#include <net/if_media.h> 58#include <net/if_ether.h> 59 60#include <net/bpf.h> 61 62#include "ioconf.h" 63 64#ifdef NET_MPSAFE 65#define VIOIF_MPSAFE 1 66#define VIOIF_MULTIQ 1 67#endif 68 69/* 70 * if_vioifreg.h: 71 */ 72/* Configuration registers */ 73#define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 74#define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 75#define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ 76 77/* Feature bits */ 78#define VIRTIO_NET_F_CSUM __BIT(0) 79#define VIRTIO_NET_F_GUEST_CSUM __BIT(1) 80#define VIRTIO_NET_F_MAC __BIT(5) 81#define VIRTIO_NET_F_GSO __BIT(6) 82#define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) 83#define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) 84#define VIRTIO_NET_F_GUEST_ECN __BIT(9) 85#define VIRTIO_NET_F_GUEST_UFO __BIT(10) 86#define VIRTIO_NET_F_HOST_TSO4 __BIT(11) 87#define VIRTIO_NET_F_HOST_TSO6 __BIT(12) 88#define VIRTIO_NET_F_HOST_ECN __BIT(13) 89#define VIRTIO_NET_F_HOST_UFO __BIT(14) 90#define VIRTIO_NET_F_MRG_RXBUF __BIT(15) 91#define VIRTIO_NET_F_STATUS __BIT(16) 92#define VIRTIO_NET_F_CTRL_VQ __BIT(17) 93#define VIRTIO_NET_F_CTRL_RX __BIT(18) 94#define VIRTIO_NET_F_CTRL_VLAN __BIT(19) 95#define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) 96#define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) 97#define VIRTIO_NET_F_MQ __BIT(22) 98 99#define VIRTIO_NET_FLAG_BITS \ 100 VIRTIO_COMMON_FLAG_BITS \ 101 "\x17""MQ" \ 102 "\x16""GUEST_ANNOUNCE" \ 103 "\x15""CTRL_RX_EXTRA" \ 104 "\x14""CTRL_VLAN" \ 105 "\x13""CTRL_RX" \ 106 "\x12""CTRL_VQ" \ 107 "\x11""STATUS" \ 108 "\x10""MRG_RXBUF" \ 109 "\x0f""HOST_UFO" \ 110 "\x0e""HOST_ECN" \ 111 "\x0d""HOST_TSO6" \ 112 "\x0c""HOST_TSO4" \ 113 "\x0b""GUEST_UFO" \ 114 "\x0a""GUEST_ECN" \ 115 "\x09""GUEST_TSO6" \ 116 "\x08""GUEST_TSO4" \ 117 "\x07""GSO" \ 118 "\x06""MAC" \ 119 "\x02""GUEST_CSUM" \ 120 "\x01""CSUM" 121 122/* Status */ 123#define VIRTIO_NET_S_LINK_UP 1 124 125/* Packet header structure */ 126struct virtio_net_hdr { 127 uint8_t flags; 128 uint8_t gso_type; 129 uint16_t hdr_len; 130 uint16_t gso_size; 131 uint16_t csum_start; 132 uint16_t csum_offset; 133#if 0 134 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 135#endif 136} __packed; 137 138#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 139#define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 140#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 141#define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 142#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 143#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 144 145#define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 146 147/* Control virtqueue */ 148struct virtio_net_ctrl_cmd { 149 uint8_t class; 150 uint8_t command; 151} __packed; 152#define VIRTIO_NET_CTRL_RX 0 153# define VIRTIO_NET_CTRL_RX_PROMISC 0 154# define VIRTIO_NET_CTRL_RX_ALLMULTI 1 155 156#define VIRTIO_NET_CTRL_MAC 1 157# define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 158 159#define VIRTIO_NET_CTRL_VLAN 2 160# define VIRTIO_NET_CTRL_VLAN_ADD 0 161# define VIRTIO_NET_CTRL_VLAN_DEL 1 162 163#define VIRTIO_NET_CTRL_MQ 4 164# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 165# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 166# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 167 168struct virtio_net_ctrl_status { 169 uint8_t ack; 170} __packed; 171#define VIRTIO_NET_OK 0 172#define VIRTIO_NET_ERR 1 173 174struct virtio_net_ctrl_rx { 175 uint8_t onoff; 176} __packed; 177 178struct virtio_net_ctrl_mac_tbl { 179 uint32_t nentries; 180 uint8_t macs[][ETHER_ADDR_LEN]; 181} __packed; 182 183struct virtio_net_ctrl_vlan { 184 uint16_t id; 185} __packed; 186 187struct virtio_net_ctrl_mq { 188 uint16_t virtqueue_pairs; 189} __packed; 190 191struct vioif_ctrl_cmdspec { 192 bus_dmamap_t dmamap; 193 void *buf; 194 bus_size_t bufsize; 195}; 196 197/* 198 * if_vioifvar.h: 199 */ 200 201/* 202 * Locking notes: 203 * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and 204 * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex). 205 * - more than one lock cannot be held at onece 206 * + ctrlq_inuse is protected by ctrlq_wait_lock. 207 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse 208 * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock 209 * + fields in vioif_softc except queues are protected by 210 * sc->sc_lock(an adaptive mutex) 211 * - the lock is held before acquisition of other locks 212 */ 213 214struct vioif_work { 215 struct work cookie; 216 void (*func)(void *); 217 void *arg; 218 unsigned int added; 219}; 220 221struct vioif_txqueue { 222 kmutex_t *txq_lock; /* lock for tx operations */ 223 224 struct virtqueue *txq_vq; 225 bool txq_stopping; 226 bool txq_link_active; 227 pcq_t *txq_intrq; 228 229 struct virtio_net_hdr *txq_hdrs; 230 bus_dmamap_t *txq_hdr_dmamaps; 231 232 struct mbuf **txq_mbufs; 233 bus_dmamap_t *txq_dmamaps; 234 235 void *txq_deferred_transmit; 236 void *txq_handle_si; 237 struct vioif_work txq_work; 238 bool txq_workqueue; 239 bool txq_active; 240 241 char txq_evgroup[16]; 242 struct evcnt txq_defrag_failed; 243 struct evcnt txq_mbuf_load_failed; 244 struct evcnt txq_enqueue_reserve_failed; 245}; 246 247struct vioif_rxqueue { 248 kmutex_t *rxq_lock; /* lock for rx operations */ 249 250 struct virtqueue *rxq_vq; 251 bool rxq_stopping; 252 253 struct virtio_net_hdr *rxq_hdrs; 254 bus_dmamap_t *rxq_hdr_dmamaps; 255 256 struct mbuf **rxq_mbufs; 257 bus_dmamap_t *rxq_dmamaps; 258 259 void *rxq_handle_si; 260 struct vioif_work rxq_work; 261 bool rxq_workqueue; 262 bool rxq_active; 263 264 char rxq_evgroup[16]; 265 struct evcnt rxq_mbuf_add_failed; 266}; 267 268struct vioif_ctrlqueue { 269 struct virtqueue *ctrlq_vq; 270 enum { 271 FREE, INUSE, DONE 272 } ctrlq_inuse; 273 kcondvar_t ctrlq_wait; 274 kmutex_t ctrlq_wait_lock; 275 struct lwp *ctrlq_owner; 276 277 struct virtio_net_ctrl_cmd *ctrlq_cmd; 278 struct virtio_net_ctrl_status *ctrlq_status; 279 struct virtio_net_ctrl_rx *ctrlq_rx; 280 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; 281 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; 282 struct virtio_net_ctrl_mq *ctrlq_mq; 283 284 bus_dmamap_t ctrlq_cmd_dmamap; 285 bus_dmamap_t ctrlq_status_dmamap; 286 bus_dmamap_t ctrlq_rx_dmamap; 287 bus_dmamap_t ctrlq_tbl_uc_dmamap; 288 bus_dmamap_t ctrlq_tbl_mc_dmamap; 289 bus_dmamap_t ctrlq_mq_dmamap; 290 291 struct evcnt ctrlq_cmd_load_failed; 292 struct evcnt ctrlq_cmd_failed; 293}; 294 295struct vioif_softc { 296 device_t sc_dev; 297 kmutex_t sc_lock; 298 struct sysctllog *sc_sysctllog; 299 300 struct virtio_softc *sc_virtio; 301 struct virtqueue *sc_vqs; 302 303 int sc_max_nvq_pairs; 304 int sc_req_nvq_pairs; 305 int sc_act_nvq_pairs; 306 307 uint8_t sc_mac[ETHER_ADDR_LEN]; 308 struct ethercom sc_ethercom; 309 short sc_deferred_init_done; 310 bool sc_link_active; 311 312 struct vioif_txqueue *sc_txq; 313 struct vioif_rxqueue *sc_rxq; 314 315 bool sc_has_ctrl; 316 struct vioif_ctrlqueue sc_ctrlq; 317 318 bus_dma_segment_t sc_hdr_segs[1]; 319 void *sc_dmamem; 320 void *sc_kmem; 321 322 void *sc_ctl_softint; 323 324 struct workqueue *sc_txrx_workqueue; 325 bool sc_txrx_workqueue_sysctl; 326 u_int sc_tx_intr_process_limit; 327 u_int sc_tx_process_limit; 328 u_int sc_rx_intr_process_limit; 329 u_int sc_rx_process_limit; 330}; 331#define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 332#define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 333 334#define VIOIF_TX_INTR_PROCESS_LIMIT 256 335#define VIOIF_TX_PROCESS_LIMIT 256 336#define VIOIF_RX_INTR_PROCESS_LIMIT 0U 337#define VIOIF_RX_PROCESS_LIMIT 256 338 339#define VIOIF_WORKQUEUE_PRI PRI_SOFTNET 340 341/* cfattach interface functions */ 342static int vioif_match(device_t, cfdata_t, void *); 343static void vioif_attach(device_t, device_t, void *); 344static void vioif_deferred_init(device_t); 345static int vioif_finalize_teardown(device_t); 346 347/* ifnet interface functions */ 348static int vioif_init(struct ifnet *); 349static void vioif_stop(struct ifnet *, int); 350static void vioif_start(struct ifnet *); 351static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); 352static int vioif_transmit(struct ifnet *, struct mbuf *); 353static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); 354static int vioif_ioctl(struct ifnet *, u_long, void *); 355static void vioif_watchdog(struct ifnet *); 356 357/* rx */ 358static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); 359static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); 360static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *); 361static void vioif_rx_queue_clear(struct vioif_rxqueue *); 362static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, 363 struct vioif_rxqueue *, u_int); 364static int vioif_rx_intr(void *); 365static void vioif_rx_handle(void *); 366static void vioif_rx_sched_handle(struct vioif_softc *, 367 struct vioif_rxqueue *); 368static void vioif_rx_drain(struct vioif_rxqueue *); 369 370/* tx */ 371static int vioif_tx_intr(void *); 372static void vioif_tx_handle(void *); 373static void vioif_tx_sched_handle(struct vioif_softc *, 374 struct vioif_txqueue *); 375static void vioif_tx_queue_clear(struct vioif_txqueue *); 376static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, 377 struct vioif_txqueue *, u_int); 378static void vioif_tx_drain(struct vioif_txqueue *); 379static void vioif_deferred_transmit(void *); 380 381/* workqueue */ 382static struct workqueue* 383 vioif_workq_create(const char *, pri_t, int, int); 384static void vioif_workq_destroy(struct workqueue *); 385static void vioif_workq_work(struct work *, void *); 386static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); 387static void vioif_work_add(struct workqueue *, struct vioif_work *); 388static void vioif_work_wait(struct workqueue *, struct vioif_work *); 389 390/* other control */ 391static bool vioif_is_link_up(struct vioif_softc *); 392static void vioif_update_link_status(struct vioif_softc *); 393static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 394static int vioif_set_promisc(struct vioif_softc *, bool); 395static int vioif_set_allmulti(struct vioif_softc *, bool); 396static int vioif_set_rx_filter(struct vioif_softc *); 397static int vioif_rx_filter(struct vioif_softc *); 398static int vioif_ctrl_intr(void *); 399static int vioif_config_change(struct virtio_softc *); 400static void vioif_ctl_softint(void *); 401static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); 402static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); 403static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); 404static int vioif_setup_sysctl(struct vioif_softc *); 405static void vioif_setup_stats(struct vioif_softc *); 406 407CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 408 vioif_match, vioif_attach, NULL, NULL); 409 410static int 411vioif_match(device_t parent, cfdata_t match, void *aux) 412{ 413 struct virtio_attach_args *va = aux; 414 415 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 416 return 1; 417 418 return 0; 419} 420 421static int 422vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, 423 bus_size_t size, int nsegs, const char *usage) 424{ 425 int r; 426 427 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, 428 nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); 429 430 if (r != 0) { 431 aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " 432 "error code %d\n", usage, r); 433 } 434 435 return r; 436} 437 438static void 439vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) 440{ 441 442 if (*map) { 443 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); 444 *map = NULL; 445 } 446} 447 448static int 449vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, 450 void *buf, bus_size_t size, int nsegs, int rw, const char *usage) 451{ 452 int r; 453 454 r = vioif_dmamap_create(sc, map, size, nsegs, usage); 455 if (r != 0) 456 return 1; 457 458 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, 459 size, NULL, rw | BUS_DMA_NOWAIT); 460 if (r != 0) { 461 vioif_dmamap_destroy(sc, map); 462 aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " 463 "error code %d\n", usage, r); 464 } 465 466 return r; 467} 468 469static void * 470vioif_assign_mem(intptr_t *p, size_t size) 471{ 472 intptr_t rv; 473 474 rv = *p; 475 *p += size; 476 477 return (void *)rv; 478} 479 480static void 481vioif_alloc_queues(struct vioif_softc *sc) 482{ 483 int nvq_pairs = sc->sc_max_nvq_pairs; 484 int nvqs = nvq_pairs * 2; 485 int i; 486 487 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 488 489 sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, 490 KM_SLEEP); 491 sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, 492 KM_SLEEP); 493 494 if (sc->sc_has_ctrl) 495 nvqs++; 496 497 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); 498 nvqs = 0; 499 for (i = 0; i < nvq_pairs; i++) { 500 sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; 501 sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; 502 } 503 504 if (sc->sc_has_ctrl) 505 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; 506} 507 508static void 509vioif_free_queues(struct vioif_softc *sc) 510{ 511 int nvq_pairs = sc->sc_max_nvq_pairs; 512 int nvqs = nvq_pairs * 2; 513 514 if (sc->sc_ctrlq.ctrlq_vq) 515 nvqs++; 516 517 if (sc->sc_txq) { 518 kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); 519 sc->sc_txq = NULL; 520 } 521 522 if (sc->sc_rxq) { 523 kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); 524 sc->sc_rxq = NULL; 525 } 526 527 if (sc->sc_vqs) { 528 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); 529 sc->sc_vqs = NULL; 530 } 531} 532 533/* allocate memory */ 534/* 535 * dma memory is used for: 536 * rxq_hdrs[slot]: metadata array for received frames (READ) 537 * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) 538 * ctrlq_cmd: command to be sent via ctrl vq (WRITE) 539 * ctrlq_status: return value for a command via ctrl vq (READ) 540 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command 541 * (WRITE) 542 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 543 * class command (WRITE) 544 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 545 * class command (WRITE) 546 * ctrlq_* structures are allocated only one each; they are protected by 547 * ctrlq_inuse variable and ctrlq_wait condvar. 548 */ 549/* 550 * dynamically allocated memory is used for: 551 * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 552 * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 553 * rxq_dmamaps[slot]: bus_dmamap_t array for received payload 554 * txq_dmamaps[slot]: bus_dmamap_t array for sent payload 555 * rxq_mbufs[slot]: mbuf pointer array for received frames 556 * txq_mbufs[slot]: mbuf pointer array for sent frames 557 */ 558static int 559vioif_alloc_mems(struct vioif_softc *sc) 560{ 561 struct virtio_softc *vsc = sc->sc_virtio; 562 struct vioif_txqueue *txq; 563 struct vioif_rxqueue *rxq; 564 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 565 int allocsize, allocsize2, r, rsegs, i, qid; 566 void *vaddr; 567 intptr_t p; 568 569 allocsize = 0; 570 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 571 rxq = &sc->sc_rxq[qid]; 572 txq = &sc->sc_txq[qid]; 573 574 allocsize += 575 sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num; 576 allocsize += 577 sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num; 578 } 579 if (sc->sc_has_ctrl) { 580 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 581 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 582 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 583 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 584 + sizeof(struct virtio_net_ctrl_mac_tbl) 585 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 586 allocsize += sizeof(struct virtio_net_ctrl_mq) * 1; 587 } 588 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 589 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 590 if (r != 0) { 591 aprint_error_dev(sc->sc_dev, 592 "DMA memory allocation failed, size %d, " 593 "error code %d\n", allocsize, r); 594 goto err_none; 595 } 596 r = bus_dmamem_map(virtio_dmat(vsc), 597 &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT); 598 if (r != 0) { 599 aprint_error_dev(sc->sc_dev, 600 "DMA memory map failed, error code %d\n", r); 601 goto err_dmamem_alloc; 602 } 603 604 memset(vaddr, 0, allocsize); 605 sc->sc_dmamem = vaddr; 606 p = (intptr_t) vaddr; 607 608 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 609 rxq = &sc->sc_rxq[qid]; 610 txq = &sc->sc_txq[qid]; 611 612 rxq->rxq_hdrs = vioif_assign_mem(&p, 613 sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num); 614 txq->txq_hdrs = vioif_assign_mem(&p, 615 sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num); 616 } 617 if (sc->sc_has_ctrl) { 618 ctrlq->ctrlq_cmd = vioif_assign_mem(&p, 619 sizeof(*ctrlq->ctrlq_cmd)); 620 ctrlq->ctrlq_status = vioif_assign_mem(&p, 621 sizeof(*ctrlq->ctrlq_status)); 622 ctrlq->ctrlq_rx = vioif_assign_mem(&p, 623 sizeof(*ctrlq->ctrlq_rx)); 624 ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, 625 sizeof(*ctrlq->ctrlq_mac_tbl_uc)); 626 ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, 627 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 628 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); 629 ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); 630 } 631 632 allocsize2 = 0; 633 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 634 int rxqsize, txqsize; 635 636 rxq = &sc->sc_rxq[qid]; 637 txq = &sc->sc_txq[qid]; 638 rxqsize = rxq->rxq_vq->vq_num; 639 txqsize = txq->txq_vq->vq_num; 640 641 allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; 642 allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; 643 allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; 644 645 allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; 646 allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; 647 allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; 648 } 649 vaddr = kmem_zalloc(allocsize2, KM_SLEEP); 650 sc->sc_kmem = vaddr; 651 p = (intptr_t) vaddr; 652 653 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 654 int rxqsize, txqsize; 655 rxq = &sc->sc_rxq[qid]; 656 txq = &sc->sc_txq[qid]; 657 rxqsize = rxq->rxq_vq->vq_num; 658 txqsize = txq->txq_vq->vq_num; 659 660 rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p, 661 sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); 662 txq->txq_hdr_dmamaps = vioif_assign_mem(&p, 663 sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); 664 rxq->rxq_dmamaps = vioif_assign_mem(&p, 665 sizeof(rxq->rxq_dmamaps[0]) * rxqsize); 666 txq->txq_dmamaps = vioif_assign_mem(&p, 667 sizeof(txq->txq_dmamaps[0]) * txqsize); 668 rxq->rxq_mbufs = vioif_assign_mem(&p, 669 sizeof(rxq->rxq_mbufs[0]) * rxqsize); 670 txq->txq_mbufs = vioif_assign_mem(&p, 671 sizeof(txq->txq_mbufs[0]) * txqsize); 672 } 673 674 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 675 rxq = &sc->sc_rxq[qid]; 676 txq = &sc->sc_txq[qid]; 677 678 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 679 r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i], 680 &rxq->rxq_hdrs[i], sizeof(rxq->rxq_hdrs[0]), 1, 681 BUS_DMA_READ, "rx header"); 682 if (r != 0) 683 goto err_reqs; 684 685 r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i], 686 MCLBYTES, 1, "rx payload"); 687 if (r != 0) 688 goto err_reqs; 689 } 690 691 for (i = 0; i < txq->txq_vq->vq_num; i++) { 692 r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i], 693 &txq->txq_hdrs[i], sizeof(txq->txq_hdrs[0]), 1, 694 BUS_DMA_READ, "tx header"); 695 if (r != 0) 696 goto err_reqs; 697 698 r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN, 699 VIRTIO_NET_TX_MAXNSEGS, "tx payload"); 700 if (r != 0) 701 goto err_reqs; 702 } 703 } 704 705 if (sc->sc_has_ctrl) { 706 /* control vq class & command */ 707 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, 708 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, 709 BUS_DMA_WRITE, "control command"); 710 if (r != 0) 711 goto err_reqs; 712 713 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, 714 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, 715 BUS_DMA_READ, "control status"); 716 if (r != 0) 717 goto err_reqs; 718 719 /* control vq rx mode command parameter */ 720 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, 721 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, 722 BUS_DMA_WRITE, "rx mode control command"); 723 if (r != 0) 724 goto err_reqs; 725 726 /* multiqueue set command */ 727 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, 728 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, 729 BUS_DMA_WRITE, "multiqueue set command"); 730 if (r != 0) 731 goto err_reqs; 732 733 /* control vq MAC filter table for unicast */ 734 /* do not load now since its length is variable */ 735 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, 736 sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1, 737 "unicast MAC address filter command"); 738 if (r != 0) 739 goto err_reqs; 740 741 /* control vq MAC filter table for multicast */ 742 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, 743 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 744 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 745 "multicast MAC address filter command"); 746 } 747 748 return 0; 749 750err_reqs: 751 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); 752 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); 753 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); 754 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); 755 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); 756 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 757 rxq = &sc->sc_rxq[qid]; 758 txq = &sc->sc_txq[qid]; 759 760 for (i = 0; i < txq->txq_vq->vq_num; i++) { 761 vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]); 762 vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]); 763 } 764 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 765 vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]); 766 vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]); 767 } 768 } 769 if (sc->sc_kmem) { 770 kmem_free(sc->sc_kmem, allocsize2); 771 sc->sc_kmem = NULL; 772 } 773 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); 774err_dmamem_alloc: 775 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 776err_none: 777 return -1; 778} 779 780static void 781vioif_attach(device_t parent, device_t self, void *aux) 782{ 783 struct vioif_softc *sc = device_private(self); 784 struct virtio_softc *vsc = device_private(parent); 785 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 786 struct vioif_txqueue *txq; 787 struct vioif_rxqueue *rxq; 788 uint32_t features, req_features; 789 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 790 u_int softint_flags; 791 int r, i, nvqs=0, req_flags; 792 char xnamebuf[MAXCOMLEN]; 793 794 if (virtio_child(vsc) != NULL) { 795 aprint_normal(": child already attached for %s; " 796 "something wrong...\n", device_xname(parent)); 797 return; 798 } 799 800 sc->sc_dev = self; 801 sc->sc_virtio = vsc; 802 sc->sc_link_active = false; 803 804 sc->sc_max_nvq_pairs = 1; 805 sc->sc_req_nvq_pairs = 1; 806 sc->sc_act_nvq_pairs = 1; 807 sc->sc_txrx_workqueue_sysctl = true; 808 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; 809 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; 810 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; 811 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; 812 813 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); 814 815 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); 816 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, 817 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 818 if (sc->sc_txrx_workqueue == NULL) 819 goto err; 820 821 req_flags = 0; 822 823#ifdef VIOIF_MPSAFE 824 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 825#endif 826 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 827 828 req_features = 829 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 830 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; 831#ifdef VIOIF_MULTIQ 832 req_features |= VIRTIO_NET_F_MQ; 833#endif 834 virtio_child_attach_start(vsc, self, IPL_NET, NULL, 835 vioif_config_change, virtio_vq_intrhand, req_flags, 836 req_features, VIRTIO_NET_FLAG_BITS); 837 838 features = virtio_features(vsc); 839 840 if (features & VIRTIO_NET_F_MAC) { 841 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 842 sc->sc_mac[i] = virtio_read_device_config_1(vsc, 843 VIRTIO_NET_CONFIG_MAC + i); 844 } 845 } else { 846 /* code stolen from sys/net/if_tap.c */ 847 struct timeval tv; 848 uint32_t ui; 849 getmicrouptime(&tv); 850 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 851 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 852 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 853 virtio_write_device_config_1(vsc, 854 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 855 } 856 } 857 858 aprint_normal_dev(self, "Ethernet address %s\n", 859 ether_sprintf(sc->sc_mac)); 860 861 if ((features & VIRTIO_NET_F_CTRL_VQ) && 862 (features & VIRTIO_NET_F_CTRL_RX)) { 863 sc->sc_has_ctrl = true; 864 865 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); 866 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); 867 ctrlq->ctrlq_inuse = FREE; 868 } else { 869 sc->sc_has_ctrl = false; 870 } 871 872 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { 873 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, 874 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); 875 876 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 877 goto err; 878 879 /* Limit the number of queue pairs to use */ 880 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); 881 } 882 883 vioif_alloc_queues(sc); 884 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); 885 886#ifdef VIOIF_MPSAFE 887 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; 888#else 889 softint_flags = SOFTINT_NET; 890#endif 891 892 /* 893 * Allocating virtqueues 894 */ 895 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 896 rxq = &sc->sc_rxq[i]; 897 txq = &sc->sc_txq[i]; 898 char qname[32]; 899 900 rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 901 902 rxq->rxq_handle_si = softint_establish(softint_flags, 903 vioif_rx_handle, rxq); 904 if (rxq->rxq_handle_si == NULL) { 905 aprint_error_dev(self, "cannot establish rx softint\n"); 906 goto err; 907 } 908 909 snprintf(qname, sizeof(qname), "rx%d", i); 910 r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, 911 MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname); 912 if (r != 0) 913 goto err; 914 nvqs++; 915 rxq->rxq_vq->vq_intrhand = vioif_rx_intr; 916 rxq->rxq_vq->vq_intrhand_arg = (void *)rxq; 917 rxq->rxq_stopping = true; 918 vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq); 919 920 txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 921 922 txq->txq_deferred_transmit = softint_establish(softint_flags, 923 vioif_deferred_transmit, txq); 924 if (txq->txq_deferred_transmit == NULL) { 925 aprint_error_dev(self, "cannot establish tx softint\n"); 926 goto err; 927 } 928 txq->txq_handle_si = softint_establish(softint_flags, 929 vioif_tx_handle, txq); 930 if (txq->txq_handle_si == NULL) { 931 aprint_error_dev(self, "cannot establish tx softint\n"); 932 goto err; 933 } 934 935 snprintf(qname, sizeof(qname), "tx%d", i); 936 r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, 937 sizeof(struct virtio_net_hdr) 938 + (ETHER_MAX_LEN - ETHER_HDR_LEN), 939 VIRTIO_NET_TX_MAXNSEGS + 1, qname); 940 if (r != 0) 941 goto err; 942 nvqs++; 943 txq->txq_vq->vq_intrhand = vioif_tx_intr; 944 txq->txq_vq->vq_intrhand_arg = (void *)txq; 945 txq->txq_link_active = sc->sc_link_active; 946 txq->txq_stopping = false; 947 txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP); 948 vioif_work_set(&txq->txq_work, vioif_tx_handle, txq); 949 } 950 951 if (sc->sc_has_ctrl) { 952 /* 953 * Allocating a virtqueue for control channel 954 */ 955 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, 956 NBPG, 1, "control"); 957 if (r != 0) { 958 aprint_error_dev(self, "failed to allocate " 959 "a virtqueue for control channel, error code %d\n", 960 r); 961 962 sc->sc_has_ctrl = false; 963 cv_destroy(&ctrlq->ctrlq_wait); 964 mutex_destroy(&ctrlq->ctrlq_wait_lock); 965 } else { 966 nvqs++; 967 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; 968 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; 969 } 970 } 971 972 sc->sc_ctl_softint = softint_establish(softint_flags, 973 vioif_ctl_softint, sc); 974 if (sc->sc_ctl_softint == NULL) { 975 aprint_error_dev(self, "cannot establish ctl softint\n"); 976 goto err; 977 } 978 979 if (vioif_alloc_mems(sc) < 0) 980 goto err; 981 982 if (virtio_child_attach_finish(vsc) != 0) 983 goto err; 984 985 if (vioif_setup_sysctl(sc) != 0) { 986 aprint_error_dev(self, "unable to create sysctl node\n"); 987 /* continue */ 988 } 989 990 vioif_setup_stats(sc); 991 992 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 993 ifp->if_softc = sc; 994 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 995#ifdef VIOIF_MPSAFE 996 ifp->if_extflags = IFEF_MPSAFE; 997#endif 998 ifp->if_start = vioif_start; 999 if (sc->sc_req_nvq_pairs > 1) 1000 ifp->if_transmit = vioif_transmit; 1001 ifp->if_ioctl = vioif_ioctl; 1002 ifp->if_init = vioif_init; 1003 ifp->if_stop = vioif_stop; 1004 ifp->if_capabilities = 0; 1005 ifp->if_watchdog = vioif_watchdog; 1006 txq = &sc->sc_txq[0]; 1007 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); 1008 IFQ_SET_READY(&ifp->if_snd); 1009 1010 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 1011 1012 if_attach(ifp); 1013 if_deferred_start_init(ifp, NULL); 1014 ether_ifattach(ifp, sc->sc_mac); 1015 1016 return; 1017 1018err: 1019 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 1020 rxq = &sc->sc_rxq[i]; 1021 txq = &sc->sc_txq[i]; 1022 1023 if (rxq->rxq_lock) { 1024 mutex_obj_free(rxq->rxq_lock); 1025 rxq->rxq_lock = NULL; 1026 } 1027 1028 if (rxq->rxq_handle_si) { 1029 softint_disestablish(rxq->rxq_handle_si); 1030 rxq->rxq_handle_si = NULL; 1031 } 1032 1033 if (txq->txq_lock) { 1034 mutex_obj_free(txq->txq_lock); 1035 txq->txq_lock = NULL; 1036 } 1037 1038 if (txq->txq_handle_si) { 1039 softint_disestablish(txq->txq_handle_si); 1040 txq->txq_handle_si = NULL; 1041 } 1042 1043 if (txq->txq_deferred_transmit) { 1044 softint_disestablish(txq->txq_deferred_transmit); 1045 txq->txq_deferred_transmit = NULL; 1046 } 1047 1048 if (txq->txq_intrq) { 1049 pcq_destroy(txq->txq_intrq); 1050 txq->txq_intrq = NULL; 1051 } 1052 } 1053 1054 if (sc->sc_has_ctrl) { 1055 cv_destroy(&ctrlq->ctrlq_wait); 1056 mutex_destroy(&ctrlq->ctrlq_wait_lock); 1057 } 1058 1059 while (nvqs > 0) 1060 virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); 1061 1062 vioif_free_queues(sc); 1063 mutex_destroy(&sc->sc_lock); 1064 virtio_child_attach_failed(vsc); 1065 config_finalize_register(self, vioif_finalize_teardown); 1066 1067 return; 1068} 1069 1070static int 1071vioif_finalize_teardown(device_t self) 1072{ 1073 struct vioif_softc *sc = device_private(self); 1074 1075 if (sc->sc_txrx_workqueue != NULL) { 1076 vioif_workq_destroy(sc->sc_txrx_workqueue); 1077 sc->sc_txrx_workqueue = NULL; 1078 } 1079 1080 return 0; 1081} 1082 1083/* we need interrupts to make promiscuous mode off */ 1084static void 1085vioif_deferred_init(device_t self) 1086{ 1087 struct vioif_softc *sc = device_private(self); 1088 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1089 int r; 1090 1091 if (ifp->if_flags & IFF_PROMISC) 1092 return; 1093 1094 r = vioif_set_promisc(sc, false); 1095 if (r != 0) 1096 aprint_error_dev(self, "resetting promisc mode failed, " 1097 "error code %d\n", r); 1098} 1099 1100static void 1101vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) 1102{ 1103 struct virtio_softc *vsc = sc->sc_virtio; 1104 struct vioif_txqueue *txq; 1105 struct vioif_rxqueue *rxq; 1106 int i; 1107 1108 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1109 txq = &sc->sc_txq[i]; 1110 rxq = &sc->sc_rxq[i]; 1111 1112 virtio_start_vq_intr(vsc, txq->txq_vq); 1113 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1114 } 1115} 1116 1117static void 1118vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) 1119{ 1120 struct virtio_softc *vsc = sc->sc_virtio; 1121 struct vioif_txqueue *txq; 1122 struct vioif_rxqueue *rxq; 1123 int i; 1124 1125 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1126 rxq = &sc->sc_rxq[i]; 1127 txq = &sc->sc_txq[i]; 1128 1129 virtio_stop_vq_intr(vsc, rxq->rxq_vq); 1130 virtio_stop_vq_intr(vsc, txq->txq_vq); 1131 } 1132} 1133 1134/* 1135 * Interface functions for ifnet 1136 */ 1137static int 1138vioif_init(struct ifnet *ifp) 1139{ 1140 struct vioif_softc *sc = ifp->if_softc; 1141 struct virtio_softc *vsc = sc->sc_virtio; 1142 struct vioif_rxqueue *rxq; 1143 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1144 int r, i; 1145 1146 vioif_stop(ifp, 0); 1147 1148 virtio_reinit_start(vsc); 1149 virtio_negotiate_features(vsc, virtio_features(vsc)); 1150 1151 for (i = 0; i < sc->sc_req_nvq_pairs; i++) { 1152 rxq = &sc->sc_rxq[i]; 1153 1154 /* Have to set false before vioif_populate_rx_mbufs */ 1155 mutex_enter(rxq->rxq_lock); 1156 rxq->rxq_stopping = false; 1157 vioif_populate_rx_mbufs_locked(rxq); 1158 mutex_exit(rxq->rxq_lock); 1159 1160 } 1161 1162 virtio_reinit_end(vsc); 1163 1164 if (sc->sc_has_ctrl) 1165 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 1166 1167 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); 1168 if (r == 0) 1169 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; 1170 else 1171 sc->sc_act_nvq_pairs = 1; 1172 1173 for (i = 0; i < sc->sc_act_nvq_pairs; i++) 1174 sc->sc_txq[i].txq_stopping = false; 1175 1176 vioif_enable_interrupt_vqpairs(sc); 1177 1178 if (!sc->sc_deferred_init_done) { 1179 sc->sc_deferred_init_done = 1; 1180 if (sc->sc_has_ctrl) 1181 vioif_deferred_init(sc->sc_dev); 1182 } 1183 1184 vioif_update_link_status(sc); 1185 ifp->if_flags |= IFF_RUNNING; 1186 ifp->if_flags &= ~IFF_OACTIVE; 1187 vioif_rx_filter(sc); 1188 1189 return 0; 1190} 1191 1192static void 1193vioif_stop(struct ifnet *ifp, int disable) 1194{ 1195 struct vioif_softc *sc = ifp->if_softc; 1196 struct virtio_softc *vsc = sc->sc_virtio; 1197 struct vioif_txqueue *txq; 1198 struct vioif_rxqueue *rxq; 1199 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1200 int i; 1201 1202 /* Take the locks to ensure that ongoing TX/RX finish */ 1203 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1204 txq = &sc->sc_txq[i]; 1205 rxq = &sc->sc_rxq[i]; 1206 1207 mutex_enter(rxq->rxq_lock); 1208 rxq->rxq_stopping = true; 1209 mutex_exit(rxq->rxq_lock); 1210 1211 mutex_enter(txq->txq_lock); 1212 txq->txq_stopping = true; 1213 mutex_exit(txq->txq_lock); 1214 } 1215 1216 /* disable interrupts */ 1217 vioif_disable_interrupt_vqpairs(sc); 1218 1219 if (sc->sc_has_ctrl) 1220 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); 1221 1222 /* only way to stop I/O and DMA is resetting... */ 1223 virtio_reset(vsc); 1224 1225 /* rendezvous for finish of handlers */ 1226 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1227 txq = &sc->sc_txq[i]; 1228 rxq = &sc->sc_rxq[i]; 1229 1230 mutex_enter(rxq->rxq_lock); 1231 mutex_exit(rxq->rxq_lock); 1232 vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work); 1233 1234 mutex_enter(txq->txq_lock); 1235 mutex_exit(txq->txq_lock); 1236 vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work); 1237 } 1238 1239 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1240 vioif_rx_queue_clear(&sc->sc_rxq[i]); 1241 vioif_tx_queue_clear(&sc->sc_txq[i]); 1242 } 1243 1244 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1245 sc->sc_link_active = false; 1246 1247 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1248 txq = &sc->sc_txq[i]; 1249 rxq = &sc->sc_rxq[i]; 1250 1251 txq->txq_link_active = false; 1252 1253 if (disable) 1254 vioif_rx_drain(rxq); 1255 1256 vioif_tx_drain(txq); 1257 } 1258} 1259 1260static void 1261vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, 1262 bool is_transmit) 1263{ 1264 struct vioif_softc *sc = ifp->if_softc; 1265 struct virtio_softc *vsc = sc->sc_virtio; 1266 struct virtqueue *vq = txq->txq_vq; 1267 struct mbuf *m; 1268 int queued = 0; 1269 1270 KASSERT(mutex_owned(txq->txq_lock)); 1271 1272 if ((ifp->if_flags & IFF_RUNNING) == 0) 1273 return; 1274 1275 if (!txq->txq_link_active || txq->txq_stopping) 1276 return; 1277 1278 if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) 1279 return; 1280 1281 for (;;) { 1282 int slot, r; 1283 1284 if (is_transmit) 1285 m = pcq_get(txq->txq_intrq); 1286 else 1287 IFQ_DEQUEUE(&ifp->if_snd, m); 1288 1289 if (m == NULL) 1290 break; 1291 1292 r = virtio_enqueue_prep(vsc, vq, &slot); 1293 if (r == EAGAIN) { 1294 ifp->if_flags |= IFF_OACTIVE; 1295 m_freem(m); 1296 break; 1297 } 1298 if (r != 0) 1299 panic("enqueue_prep for a tx buffer"); 1300 1301 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1302 txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1303 if (r != 0) { 1304 /* maybe just too fragmented */ 1305 struct mbuf *newm; 1306 1307 newm = m_defrag(m, M_NOWAIT); 1308 if (newm == NULL) { 1309 txq->txq_defrag_failed.ev_count++; 1310 goto skip; 1311 } 1312 1313 m = newm; 1314 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1315 txq->txq_dmamaps[slot], m, 1316 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1317 if (r != 0) { 1318 txq->txq_mbuf_load_failed.ev_count++; 1319skip: 1320 m_freem(m); 1321 virtio_enqueue_abort(vsc, vq, slot); 1322 continue; 1323 } 1324 } 1325 1326 /* This should actually never fail */ 1327 r = virtio_enqueue_reserve(vsc, vq, slot, 1328 txq->txq_dmamaps[slot]->dm_nsegs + 1); 1329 if (r != 0) { 1330 txq->txq_enqueue_reserve_failed.ev_count++; 1331 bus_dmamap_unload(virtio_dmat(vsc), 1332 txq->txq_dmamaps[slot]); 1333 /* slot already freed by virtio_enqueue_reserve */ 1334 m_freem(m); 1335 continue; 1336 } 1337 1338 txq->txq_mbufs[slot] = m; 1339 1340 memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 1341 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1342 0, txq->txq_dmamaps[slot]->dm_mapsize, 1343 BUS_DMASYNC_PREWRITE); 1344 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1345 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, 1346 BUS_DMASYNC_PREWRITE); 1347 virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); 1348 virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); 1349 virtio_enqueue_commit(vsc, vq, slot, false); 1350 1351 queued++; 1352 bpf_mtap(ifp, m, BPF_D_OUT); 1353 } 1354 1355 if (queued > 0) { 1356 virtio_enqueue_commit(vsc, vq, -1, true); 1357 ifp->if_timer = 5; 1358 } 1359} 1360 1361static void 1362vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1363{ 1364 1365 /* 1366 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. 1367 */ 1368 vioif_send_common_locked(ifp, txq, false); 1369 1370} 1371 1372static void 1373vioif_start(struct ifnet *ifp) 1374{ 1375 struct vioif_softc *sc = ifp->if_softc; 1376 struct vioif_txqueue *txq = &sc->sc_txq[0]; 1377 1378#ifdef VIOIF_MPSAFE 1379 KASSERT(if_is_mpsafe(ifp)); 1380#endif 1381 1382 mutex_enter(txq->txq_lock); 1383 vioif_start_locked(ifp, txq); 1384 mutex_exit(txq->txq_lock); 1385} 1386 1387static inline int 1388vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) 1389{ 1390 struct vioif_softc *sc = ifp->if_softc; 1391 u_int cpuid = cpu_index(curcpu()); 1392 1393 return cpuid % sc->sc_act_nvq_pairs; 1394} 1395 1396static void 1397vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1398{ 1399 1400 vioif_send_common_locked(ifp, txq, true); 1401} 1402 1403static int 1404vioif_transmit(struct ifnet *ifp, struct mbuf *m) 1405{ 1406 struct vioif_softc *sc = ifp->if_softc; 1407 struct vioif_txqueue *txq; 1408 int qid; 1409 1410 qid = vioif_select_txqueue(ifp, m); 1411 txq = &sc->sc_txq[qid]; 1412 1413 if (__predict_false(!pcq_put(txq->txq_intrq, m))) { 1414 m_freem(m); 1415 return ENOBUFS; 1416 } 1417 1418 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1419 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1420 if (m->m_flags & M_MCAST) 1421 if_statinc_ref(nsr, if_omcasts); 1422 IF_STAT_PUTREF(ifp); 1423 1424 if (mutex_tryenter(txq->txq_lock)) { 1425 vioif_transmit_locked(ifp, txq); 1426 mutex_exit(txq->txq_lock); 1427 } 1428 1429 return 0; 1430} 1431 1432static void 1433vioif_deferred_transmit(void *arg) 1434{ 1435 struct vioif_txqueue *txq = arg; 1436 struct virtio_softc *vsc = txq->txq_vq->vq_owner; 1437 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1438 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1439 1440 mutex_enter(txq->txq_lock); 1441 vioif_send_common_locked(ifp, txq, true); 1442 mutex_exit(txq->txq_lock); 1443} 1444 1445static int 1446vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1447{ 1448 int s, r; 1449 1450 s = splnet(); 1451 1452 r = ether_ioctl(ifp, cmd, data); 1453 if ((r == 0 && cmd == SIOCSIFFLAGS) || 1454 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 1455 if (ifp->if_flags & IFF_RUNNING) 1456 r = vioif_rx_filter(ifp->if_softc); 1457 else 1458 r = 0; 1459 } 1460 1461 splx(s); 1462 1463 return r; 1464} 1465 1466void 1467vioif_watchdog(struct ifnet *ifp) 1468{ 1469 struct vioif_softc *sc = ifp->if_softc; 1470 int i; 1471 1472 if (ifp->if_flags & IFF_RUNNING) { 1473 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1474 vioif_tx_queue_clear(&sc->sc_txq[i]); 1475 } 1476 } 1477} 1478 1479/* 1480 * Receive implementation 1481 */ 1482/* allocate and initialize a mbuf for receive */ 1483static int 1484vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1485{ 1486 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1487 struct mbuf *m; 1488 int r; 1489 1490 MGETHDR(m, M_DONTWAIT, MT_DATA); 1491 if (m == NULL) 1492 return ENOBUFS; 1493 MCLGET(m, M_DONTWAIT); 1494 if ((m->m_flags & M_EXT) == 0) { 1495 m_freem(m); 1496 return ENOBUFS; 1497 } 1498 rxq->rxq_mbufs[i] = m; 1499 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 1500 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1501 rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT); 1502 if (r) { 1503 m_freem(m); 1504 rxq->rxq_mbufs[i] = NULL; 1505 return r; 1506 } 1507 1508 return 0; 1509} 1510 1511/* free a mbuf for receive */ 1512static void 1513vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1514{ 1515 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1516 1517 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); 1518 m_freem(rxq->rxq_mbufs[i]); 1519 rxq->rxq_mbufs[i] = NULL; 1520} 1521 1522/* add mbufs for all the empty receive slots */ 1523static void 1524vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq) 1525{ 1526 struct virtqueue *vq = rxq->rxq_vq; 1527 struct virtio_softc *vsc = vq->vq_owner; 1528 int i, r, ndone = 0; 1529 1530 KASSERT(mutex_owned(rxq->rxq_lock)); 1531 1532 if (rxq->rxq_stopping) 1533 return; 1534 1535 for (i = 0; i < vq->vq_num; i++) { 1536 int slot; 1537 r = virtio_enqueue_prep(vsc, vq, &slot); 1538 if (r == EAGAIN) 1539 break; 1540 if (r != 0) 1541 panic("enqueue_prep for rx buffers"); 1542 if (rxq->rxq_mbufs[slot] == NULL) { 1543 r = vioif_add_rx_mbuf(rxq, slot); 1544 if (r != 0) { 1545 rxq->rxq_mbuf_add_failed.ev_count++; 1546 break; 1547 } 1548 } 1549 r = virtio_enqueue_reserve(vsc, vq, slot, 1550 rxq->rxq_dmamaps[slot]->dm_nsegs + 1); 1551 if (r != 0) { 1552 vioif_free_rx_mbuf(rxq, slot); 1553 break; 1554 } 1555 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1556 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 1557 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1558 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1559 virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], 1560 false); 1561 virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); 1562 virtio_enqueue_commit(vsc, vq, slot, false); 1563 ndone++; 1564 } 1565 if (ndone > 0) 1566 virtio_enqueue_commit(vsc, vq, -1, true); 1567} 1568 1569static void 1570vioif_rx_queue_clear(struct vioif_rxqueue *rxq) 1571{ 1572 struct virtqueue *vq = rxq->rxq_vq; 1573 struct virtio_softc *vsc = vq->vq_owner; 1574 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1575 u_int limit = UINT_MAX; 1576 bool more; 1577 1578 KASSERT(rxq->rxq_stopping); 1579 1580 mutex_enter(rxq->rxq_lock); 1581 for (;;) { 1582 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1583 if (more == false) 1584 break; 1585 } 1586 mutex_exit(rxq->rxq_lock); 1587} 1588 1589/* dequeue received packets */ 1590static bool 1591vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1592 struct vioif_rxqueue *rxq, u_int limit) 1593{ 1594 struct virtqueue *vq = rxq->rxq_vq; 1595 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1596 struct mbuf *m; 1597 int slot, len; 1598 bool more = false, dequeued = false; 1599 1600 KASSERT(mutex_owned(rxq->rxq_lock)); 1601 1602 if (virtio_vq_is_enqueued(vsc, vq) == false) 1603 return false; 1604 1605 for (;;) { 1606 if (limit-- == 0) { 1607 more = true; 1608 break; 1609 } 1610 1611 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1612 break; 1613 1614 dequeued = true; 1615 1616 len -= sizeof(struct virtio_net_hdr); 1617 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1618 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD); 1619 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1620 0, MCLBYTES, BUS_DMASYNC_POSTREAD); 1621 m = rxq->rxq_mbufs[slot]; 1622 KASSERT(m != NULL); 1623 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); 1624 rxq->rxq_mbufs[slot] = NULL; 1625 virtio_dequeue_commit(vsc, vq, slot); 1626 m_set_rcvif(m, ifp); 1627 m->m_len = m->m_pkthdr.len = len; 1628 1629 mutex_exit(rxq->rxq_lock); 1630 if_percpuq_enqueue(ifp->if_percpuq, m); 1631 mutex_enter(rxq->rxq_lock); 1632 1633 if (rxq->rxq_stopping) 1634 break; 1635 } 1636 1637 if (dequeued) 1638 vioif_populate_rx_mbufs_locked(rxq); 1639 1640 return more; 1641} 1642 1643/* rx interrupt; call _dequeue above and schedule a softint */ 1644static int 1645vioif_rx_intr(void *arg) 1646{ 1647 struct vioif_rxqueue *rxq = arg; 1648 struct virtqueue *vq = rxq->rxq_vq; 1649 struct virtio_softc *vsc = vq->vq_owner; 1650 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1651 u_int limit; 1652 bool more; 1653 1654 limit = sc->sc_rx_intr_process_limit; 1655 1656 if (atomic_load_relaxed(&rxq->rxq_active) == true) 1657 return 1; 1658 1659 mutex_enter(rxq->rxq_lock); 1660 1661 if (!rxq->rxq_stopping) { 1662 rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl; 1663 1664 virtio_stop_vq_intr(vsc, vq); 1665 atomic_store_relaxed(&rxq->rxq_active, true); 1666 1667 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1668 if (more) { 1669 vioif_rx_sched_handle(sc, rxq); 1670 } else { 1671 atomic_store_relaxed(&rxq->rxq_active, false); 1672 virtio_start_vq_intr(vsc, vq); 1673 } 1674 } 1675 1676 mutex_exit(rxq->rxq_lock); 1677 return 1; 1678} 1679 1680static void 1681vioif_rx_handle(void *xrxq) 1682{ 1683 struct vioif_rxqueue *rxq = xrxq; 1684 struct virtqueue *vq = rxq->rxq_vq; 1685 struct virtio_softc *vsc = vq->vq_owner; 1686 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1687 u_int limit; 1688 bool more; 1689 1690 limit = sc->sc_rx_process_limit; 1691 1692 mutex_enter(rxq->rxq_lock); 1693 1694 if (!rxq->rxq_stopping) { 1695 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1696 if (more) { 1697 vioif_rx_sched_handle(sc, rxq); 1698 } else { 1699 atomic_store_relaxed(&rxq->rxq_active, false); 1700 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1701 } 1702 } 1703 1704 mutex_exit(rxq->rxq_lock); 1705} 1706 1707static void 1708vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq) 1709{ 1710 1711 if (rxq->rxq_workqueue) 1712 vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work); 1713 else 1714 softint_schedule(rxq->rxq_handle_si); 1715} 1716 1717/* free all the mbufs; called from if_stop(disable) */ 1718static void 1719vioif_rx_drain(struct vioif_rxqueue *rxq) 1720{ 1721 struct virtqueue *vq = rxq->rxq_vq; 1722 int i; 1723 1724 for (i = 0; i < vq->vq_num; i++) { 1725 if (rxq->rxq_mbufs[i] == NULL) 1726 continue; 1727 vioif_free_rx_mbuf(rxq, i); 1728 } 1729} 1730 1731/* 1732 * Transmition implementation 1733 */ 1734/* actual transmission is done in if_start */ 1735/* tx interrupt; dequeue and free mbufs */ 1736/* 1737 * tx interrupt is actually disabled; this should be called upon 1738 * tx vq full and watchdog 1739 */ 1740 1741static int 1742vioif_tx_intr(void *arg) 1743{ 1744 struct vioif_txqueue *txq = arg; 1745 struct virtqueue *vq = txq->txq_vq; 1746 struct virtio_softc *vsc = vq->vq_owner; 1747 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1748 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1749 bool more; 1750 u_int limit; 1751 1752 limit = sc->sc_tx_intr_process_limit; 1753 1754 if (atomic_load_relaxed(&txq->txq_active) == true) 1755 return 1; 1756 1757 mutex_enter(txq->txq_lock); 1758 1759 if (!txq->txq_stopping) { 1760 txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl; 1761 1762 virtio_stop_vq_intr(vsc, vq); 1763 atomic_store_relaxed(&txq->txq_active, true); 1764 1765 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1766 if (more) { 1767 vioif_tx_sched_handle(sc, txq); 1768 } else { 1769 atomic_store_relaxed(&txq->txq_active, false); 1770 1771 /* for ALTQ */ 1772 if (txq == &sc->sc_txq[0]) { 1773 if_schedule_deferred_start(ifp); 1774 ifp->if_flags &= ~IFF_OACTIVE; 1775 } 1776 softint_schedule(txq->txq_deferred_transmit); 1777 1778 virtio_start_vq_intr(vsc, vq); 1779 } 1780 } 1781 1782 mutex_exit(txq->txq_lock); 1783 1784 return 1; 1785} 1786 1787static void 1788vioif_tx_handle(void *xtxq) 1789{ 1790 struct vioif_txqueue *txq = xtxq; 1791 struct virtqueue *vq = txq->txq_vq; 1792 struct virtio_softc *vsc = vq->vq_owner; 1793 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1794 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1795 u_int limit; 1796 bool more; 1797 1798 limit = sc->sc_tx_process_limit; 1799 1800 mutex_enter(txq->txq_lock); 1801 1802 if (!txq->txq_stopping) { 1803 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1804 if (more) { 1805 vioif_tx_sched_handle(sc, txq); 1806 } else { 1807 atomic_store_relaxed(&txq->txq_active, false); 1808 1809 /* for ALTQ */ 1810 if (txq == &sc->sc_txq[0]) { 1811 if_schedule_deferred_start(ifp); 1812 ifp->if_flags &= ~IFF_OACTIVE; 1813 } 1814 softint_schedule(txq->txq_deferred_transmit); 1815 1816 virtio_start_vq_intr(vsc, txq->txq_vq); 1817 } 1818 } 1819 1820 mutex_exit(txq->txq_lock); 1821} 1822 1823static void 1824vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq) 1825{ 1826 1827 if (txq->txq_workqueue) 1828 vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work); 1829 else 1830 softint_schedule(txq->txq_handle_si); 1831} 1832 1833static void 1834vioif_tx_queue_clear(struct vioif_txqueue *txq) 1835{ 1836 struct virtqueue *vq = txq->txq_vq; 1837 struct virtio_softc *vsc = vq->vq_owner; 1838 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1839 u_int limit = UINT_MAX; 1840 bool more; 1841 1842 mutex_enter(txq->txq_lock); 1843 for (;;) { 1844 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1845 if (more == false) 1846 break; 1847 } 1848 mutex_exit(txq->txq_lock); 1849} 1850 1851static bool 1852vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1853 struct vioif_txqueue *txq, u_int limit) 1854{ 1855 struct virtqueue *vq = txq->txq_vq; 1856 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1857 struct mbuf *m; 1858 int slot, len; 1859 bool more = false; 1860 1861 KASSERT(mutex_owned(txq->txq_lock)); 1862 1863 if (virtio_vq_is_enqueued(vsc, vq) == false) 1864 return false; 1865 1866 for (;;) { 1867 if (limit-- == 0) { 1868 more = true; 1869 break; 1870 } 1871 1872 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1873 break; 1874 1875 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1876 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE); 1877 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1878 0, txq->txq_dmamaps[slot]->dm_mapsize, 1879 BUS_DMASYNC_POSTWRITE); 1880 m = txq->txq_mbufs[slot]; 1881 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); 1882 txq->txq_mbufs[slot] = NULL; 1883 virtio_dequeue_commit(vsc, vq, slot); 1884 if_statinc(ifp, if_opackets); 1885 m_freem(m); 1886 } 1887 1888 return more; 1889} 1890 1891/* free all the mbufs already put on vq; called from if_stop(disable) */ 1892static void 1893vioif_tx_drain(struct vioif_txqueue *txq) 1894{ 1895 struct virtqueue *vq = txq->txq_vq; 1896 struct virtio_softc *vsc = vq->vq_owner; 1897 int i; 1898 1899 KASSERT(txq->txq_stopping); 1900 1901 for (i = 0; i < vq->vq_num; i++) { 1902 if (txq->txq_mbufs[i] == NULL) 1903 continue; 1904 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); 1905 m_freem(txq->txq_mbufs[i]); 1906 txq->txq_mbufs[i] = NULL; 1907 } 1908} 1909 1910/* 1911 * Control vq 1912 */ 1913/* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1914static void 1915vioif_ctrl_acquire(struct vioif_softc *sc) 1916{ 1917 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1918 1919 mutex_enter(&ctrlq->ctrlq_wait_lock); 1920 while (ctrlq->ctrlq_inuse != FREE) 1921 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 1922 ctrlq->ctrlq_inuse = INUSE; 1923 ctrlq->ctrlq_owner = curlwp; 1924 mutex_exit(&ctrlq->ctrlq_wait_lock); 1925} 1926 1927static void 1928vioif_ctrl_release(struct vioif_softc *sc) 1929{ 1930 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1931 1932 KASSERT(ctrlq->ctrlq_inuse != FREE); 1933 KASSERT(ctrlq->ctrlq_owner == curlwp); 1934 1935 mutex_enter(&ctrlq->ctrlq_wait_lock); 1936 ctrlq->ctrlq_inuse = FREE; 1937 ctrlq->ctrlq_owner = NULL; 1938 cv_signal(&ctrlq->ctrlq_wait); 1939 mutex_exit(&ctrlq->ctrlq_wait_lock); 1940} 1941 1942static int 1943vioif_ctrl_load_cmdspec(struct vioif_softc *sc, 1944 struct vioif_ctrl_cmdspec *specs, int nspecs) 1945{ 1946 struct virtio_softc *vsc = sc->sc_virtio; 1947 int i, r, loaded; 1948 1949 loaded = 0; 1950 for (i = 0; i < nspecs; i++) { 1951 r = bus_dmamap_load(virtio_dmat(vsc), 1952 specs[i].dmamap, specs[i].buf, specs[i].bufsize, 1953 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1954 if (r) { 1955 sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++; 1956 goto err; 1957 } 1958 loaded++; 1959 1960 } 1961 1962 return r; 1963 1964err: 1965 for (i = 0; i < loaded; i++) { 1966 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1967 } 1968 1969 return r; 1970} 1971 1972static void 1973vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, 1974 struct vioif_ctrl_cmdspec *specs, int nspecs) 1975{ 1976 struct virtio_softc *vsc = sc->sc_virtio; 1977 int i; 1978 1979 for (i = 0; i < nspecs; i++) { 1980 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1981 } 1982} 1983 1984static int 1985vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, 1986 struct vioif_ctrl_cmdspec *specs, int nspecs) 1987{ 1988 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1989 struct virtqueue *vq = ctrlq->ctrlq_vq; 1990 struct virtio_softc *vsc = sc->sc_virtio; 1991 int i, r, slot; 1992 1993 ctrlq->ctrlq_cmd->class = class; 1994 ctrlq->ctrlq_cmd->command = cmd; 1995 1996 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 1997 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); 1998 for (i = 0; i < nspecs; i++) { 1999 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 2000 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); 2001 } 2002 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 2003 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); 2004 2005 r = virtio_enqueue_prep(vsc, vq, &slot); 2006 if (r != 0) 2007 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2008 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); 2009 if (r != 0) 2010 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2011 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); 2012 for (i = 0; i < nspecs; i++) { 2013 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); 2014 } 2015 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); 2016 virtio_enqueue_commit(vsc, vq, slot, true); 2017 2018 /* wait for done */ 2019 mutex_enter(&ctrlq->ctrlq_wait_lock); 2020 while (ctrlq->ctrlq_inuse != DONE) 2021 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2022 mutex_exit(&ctrlq->ctrlq_wait_lock); 2023 /* already dequeueued */ 2024 2025 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, 2026 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 2027 for (i = 0; i < nspecs; i++) { 2028 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, 2029 specs[i].bufsize, BUS_DMASYNC_POSTWRITE); 2030 } 2031 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, 2032 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); 2033 2034 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) 2035 r = 0; 2036 else { 2037 device_printf(sc->sc_dev, "failed setting rx mode\n"); 2038 sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++; 2039 r = EIO; 2040 } 2041 2042 return r; 2043} 2044 2045static int 2046vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 2047{ 2048 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; 2049 struct vioif_ctrl_cmdspec specs[1]; 2050 int r; 2051 2052 if (!sc->sc_has_ctrl) 2053 return ENOTSUP; 2054 2055 vioif_ctrl_acquire(sc); 2056 2057 rx->onoff = onoff; 2058 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; 2059 specs[0].buf = rx; 2060 specs[0].bufsize = sizeof(*rx); 2061 2062 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, 2063 specs, __arraycount(specs)); 2064 2065 vioif_ctrl_release(sc); 2066 return r; 2067} 2068 2069static int 2070vioif_set_promisc(struct vioif_softc *sc, bool onoff) 2071{ 2072 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 2073} 2074 2075static int 2076vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 2077{ 2078 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 2079} 2080 2081/* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 2082static int 2083vioif_set_rx_filter(struct vioif_softc *sc) 2084{ 2085 /* filter already set in ctrlq->ctrlq_mac_tbl */ 2086 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; 2087 struct vioif_ctrl_cmdspec specs[2]; 2088 int nspecs = __arraycount(specs); 2089 int r; 2090 2091 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; 2092 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; 2093 2094 if (!sc->sc_has_ctrl) 2095 return ENOTSUP; 2096 2097 vioif_ctrl_acquire(sc); 2098 2099 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; 2100 specs[0].buf = mac_tbl_uc; 2101 specs[0].bufsize = sizeof(*mac_tbl_uc) 2102 + (ETHER_ADDR_LEN * mac_tbl_uc->nentries); 2103 2104 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; 2105 specs[1].buf = mac_tbl_mc; 2106 specs[1].bufsize = sizeof(*mac_tbl_mc) 2107 + (ETHER_ADDR_LEN * mac_tbl_mc->nentries); 2108 2109 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); 2110 if (r != 0) 2111 goto out; 2112 2113 r = vioif_ctrl_send_command(sc, 2114 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, 2115 specs, nspecs); 2116 2117 vioif_ctrl_unload_cmdspec(sc, specs, nspecs); 2118 2119out: 2120 vioif_ctrl_release(sc); 2121 2122 return r; 2123} 2124 2125static int 2126vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) 2127{ 2128 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; 2129 struct vioif_ctrl_cmdspec specs[1]; 2130 int r; 2131 2132 if (!sc->sc_has_ctrl) 2133 return ENOTSUP; 2134 2135 if (nvq_pairs <= 1) 2136 return EINVAL; 2137 2138 vioif_ctrl_acquire(sc); 2139 2140 mq->virtqueue_pairs = nvq_pairs; 2141 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; 2142 specs[0].buf = mq; 2143 specs[0].bufsize = sizeof(*mq); 2144 2145 r = vioif_ctrl_send_command(sc, 2146 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 2147 specs, __arraycount(specs)); 2148 2149 vioif_ctrl_release(sc); 2150 2151 return r; 2152} 2153 2154/* ctrl vq interrupt; wake up the command issuer */ 2155static int 2156vioif_ctrl_intr(void *arg) 2157{ 2158 struct vioif_ctrlqueue *ctrlq = arg; 2159 struct virtqueue *vq = ctrlq->ctrlq_vq; 2160 struct virtio_softc *vsc = vq->vq_owner; 2161 int r, slot; 2162 2163 if (virtio_vq_is_enqueued(vsc, vq) == false) 2164 return 0; 2165 2166 r = virtio_dequeue(vsc, vq, &slot, NULL); 2167 if (r == ENOENT) 2168 return 0; 2169 virtio_dequeue_commit(vsc, vq, slot); 2170 2171 mutex_enter(&ctrlq->ctrlq_wait_lock); 2172 ctrlq->ctrlq_inuse = DONE; 2173 cv_signal(&ctrlq->ctrlq_wait); 2174 mutex_exit(&ctrlq->ctrlq_wait_lock); 2175 2176 return 1; 2177} 2178 2179/* 2180 * If IFF_PROMISC requested, set promiscuous 2181 * If multicast filter small enough (<=MAXENTRIES) set rx filter 2182 * If large multicast filter exist use ALLMULTI 2183 */ 2184/* 2185 * If setting rx filter fails fall back to ALLMULTI 2186 * If ALLMULTI fails fall back to PROMISC 2187 */ 2188static int 2189vioif_rx_filter(struct vioif_softc *sc) 2190{ 2191 struct ethercom *ec = &sc->sc_ethercom; 2192 struct ifnet *ifp = &ec->ec_if; 2193 struct ether_multi *enm; 2194 struct ether_multistep step; 2195 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2196 int nentries; 2197 int promisc = 0, allmulti = 0, rxfilter = 0; 2198 int r; 2199 2200 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 2201 ifp->if_flags |= IFF_PROMISC; 2202 return 0; 2203 } 2204 2205 if (ifp->if_flags & IFF_PROMISC) { 2206 promisc = 1; 2207 goto set; 2208 } 2209 2210 nentries = -1; 2211 ETHER_LOCK(ec); 2212 ETHER_FIRST_MULTI(step, ec, enm); 2213 while (nentries++, enm != NULL) { 2214 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 2215 allmulti = 1; 2216 goto set_unlock; 2217 } 2218 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 2219 allmulti = 1; 2220 goto set_unlock; 2221 } 2222 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], 2223 enm->enm_addrlo, ETHER_ADDR_LEN); 2224 ETHER_NEXT_MULTI(step, enm); 2225 } 2226 rxfilter = 1; 2227 2228set_unlock: 2229 ETHER_UNLOCK(ec); 2230 2231set: 2232 if (rxfilter) { 2233 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2234 ctrlq->ctrlq_mac_tbl_mc->nentries = nentries; 2235 r = vioif_set_rx_filter(sc); 2236 if (r != 0) { 2237 rxfilter = 0; 2238 allmulti = 1; /* fallback */ 2239 } 2240 } else { 2241 /* remove rx filter */ 2242 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2243 ctrlq->ctrlq_mac_tbl_mc->nentries = 0; 2244 r = vioif_set_rx_filter(sc); 2245 /* what to do on failure? */ 2246 } 2247 if (allmulti) { 2248 r = vioif_set_allmulti(sc, true); 2249 if (r != 0) { 2250 allmulti = 0; 2251 promisc = 1; /* fallback */ 2252 } 2253 } else { 2254 r = vioif_set_allmulti(sc, false); 2255 /* what to do on failure? */ 2256 } 2257 if (promisc) { 2258 r = vioif_set_promisc(sc, true); 2259 } else { 2260 r = vioif_set_promisc(sc, false); 2261 } 2262 2263 return r; 2264} 2265 2266static bool 2267vioif_is_link_up(struct vioif_softc *sc) 2268{ 2269 struct virtio_softc *vsc = sc->sc_virtio; 2270 uint16_t status; 2271 2272 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 2273 status = virtio_read_device_config_2(vsc, 2274 VIRTIO_NET_CONFIG_STATUS); 2275 else 2276 status = VIRTIO_NET_S_LINK_UP; 2277 2278 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 2279} 2280 2281/* change link status */ 2282static void 2283vioif_update_link_status(struct vioif_softc *sc) 2284{ 2285 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2286 struct vioif_txqueue *txq; 2287 bool active, changed; 2288 int link, i; 2289 2290 mutex_enter(&sc->sc_lock); 2291 2292 active = vioif_is_link_up(sc); 2293 changed = false; 2294 2295 if (active) { 2296 if (!sc->sc_link_active) 2297 changed = true; 2298 2299 link = LINK_STATE_UP; 2300 sc->sc_link_active = true; 2301 } else { 2302 if (sc->sc_link_active) 2303 changed = true; 2304 2305 link = LINK_STATE_DOWN; 2306 sc->sc_link_active = false; 2307 } 2308 2309 if (changed) { 2310 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 2311 txq = &sc->sc_txq[i]; 2312 2313 mutex_enter(txq->txq_lock); 2314 txq->txq_link_active = sc->sc_link_active; 2315 mutex_exit(txq->txq_lock); 2316 } 2317 2318 if_link_state_change(ifp, link); 2319 } 2320 2321 mutex_exit(&sc->sc_lock); 2322} 2323 2324static int 2325vioif_config_change(struct virtio_softc *vsc) 2326{ 2327 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2328 2329 softint_schedule(sc->sc_ctl_softint); 2330 return 0; 2331} 2332 2333static void 2334vioif_ctl_softint(void *arg) 2335{ 2336 struct vioif_softc *sc = arg; 2337 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2338 2339 vioif_update_link_status(sc); 2340 vioif_start(ifp); 2341} 2342 2343static struct workqueue * 2344vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) 2345{ 2346 struct workqueue *wq; 2347 int error; 2348 2349 error = workqueue_create(&wq, name, vioif_workq_work, NULL, 2350 prio, ipl, flags); 2351 2352 if (error) 2353 return NULL; 2354 2355 return wq; 2356} 2357 2358static void 2359vioif_workq_destroy(struct workqueue *wq) 2360{ 2361 2362 workqueue_destroy(wq); 2363} 2364 2365static void 2366vioif_workq_work(struct work *wk, void *context) 2367{ 2368 struct vioif_work *work; 2369 2370 work = container_of(wk, struct vioif_work, cookie); 2371 2372 atomic_store_relaxed(&work->added, 0); 2373 work->func(work->arg); 2374} 2375 2376static void 2377vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) 2378{ 2379 2380 memset(work, 0, sizeof(*work)); 2381 work->func = func; 2382 work->arg = arg; 2383} 2384 2385static void 2386vioif_work_add(struct workqueue *wq, struct vioif_work *work) 2387{ 2388 2389 if (atomic_load_relaxed(&work->added) != 0) 2390 return; 2391 2392 atomic_store_relaxed(&work->added, 1); 2393 kpreempt_disable(); 2394 workqueue_enqueue(wq, &work->cookie, NULL); 2395 kpreempt_enable(); 2396} 2397 2398static void 2399vioif_work_wait(struct workqueue *wq, struct vioif_work *work) 2400{ 2401 2402 workqueue_wait(wq, &work->cookie); 2403} 2404 2405static int 2406vioif_setup_sysctl(struct vioif_softc *sc) 2407{ 2408 const char *devname; 2409 struct sysctllog **log; 2410 const struct sysctlnode *rnode, *rxnode, *txnode; 2411 int error; 2412 2413 log = &sc->sc_sysctllog; 2414 devname = device_xname(sc->sc_dev); 2415 2416 error = sysctl_createv(log, 0, NULL, &rnode, 2417 0, CTLTYPE_NODE, devname, 2418 SYSCTL_DESCR("virtio-net information and settings"), 2419 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 2420 if (error) 2421 goto out; 2422 2423 error = sysctl_createv(log, 0, &rnode, NULL, 2424 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2425 SYSCTL_DESCR("Use workqueue for packet processing"), 2426 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); 2427 if (error) 2428 goto out; 2429 2430 error = sysctl_createv(log, 0, &rnode, &rxnode, 2431 0, CTLTYPE_NODE, "rx", 2432 SYSCTL_DESCR("virtio-net information and settings for Rx"), 2433 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2434 if (error) 2435 goto out; 2436 2437 error = sysctl_createv(log, 0, &rxnode, NULL, 2438 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2439 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 2440 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2441 if (error) 2442 goto out; 2443 2444 error = sysctl_createv(log, 0, &rxnode, NULL, 2445 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2446 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 2447 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 2448 if (error) 2449 goto out; 2450 2451 error = sysctl_createv(log, 0, &rnode, &txnode, 2452 0, CTLTYPE_NODE, "tx", 2453 SYSCTL_DESCR("virtio-net information and settings for Tx"), 2454 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2455 if (error) 2456 goto out; 2457 2458 error = sysctl_createv(log, 0, &txnode, NULL, 2459 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2460 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 2461 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2462 if (error) 2463 goto out; 2464 2465 error = sysctl_createv(log, 0, &txnode, NULL, 2466 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2467 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 2468 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 2469 2470out: 2471 if (error) 2472 sysctl_teardown(log); 2473 2474 return error; 2475} 2476 2477static void 2478vioif_setup_stats(struct vioif_softc *sc) 2479{ 2480 struct vioif_rxqueue *rxq; 2481 struct vioif_txqueue *txq; 2482 int i; 2483 2484 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 2485 rxq = &sc->sc_rxq[i]; 2486 txq = &sc->sc_txq[i]; 2487 2488 snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d", 2489 device_xname(sc->sc_dev), i); 2490 evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC, 2491 NULL, txq->txq_evgroup, "tx m_defrag() failed"); 2492 evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC, 2493 NULL, txq->txq_evgroup, "tx dmamap load failed"); 2494 evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC, 2495 NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed"); 2496 2497 snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d", 2498 device_xname(sc->sc_dev), i); 2499 evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC, 2500 NULL, rxq->rxq_evgroup, "rx mbuf allocation failed"); 2501 } 2502 2503 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, 2504 NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); 2505 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, 2506 NULL, device_xname(sc->sc_dev), "control command failed"); 2507} 2508 2509MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 2510 2511#ifdef _MODULE 2512#include "ioconf.c" 2513#endif 2514 2515static int 2516if_vioif_modcmd(modcmd_t cmd, void *opaque) 2517{ 2518 int error = 0; 2519 2520#ifdef _MODULE 2521 switch (cmd) { 2522 case MODULE_CMD_INIT: 2523 error = config_init_component(cfdriver_ioconf_if_vioif, 2524 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2525 break; 2526 case MODULE_CMD_FINI: 2527 error = config_fini_component(cfdriver_ioconf_if_vioif, 2528 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2529 break; 2530 default: 2531 error = ENOTTY; 2532 break; 2533 } 2534#endif 2535 2536 return error; 2537} 2538