if_vioif.c revision 1.57
1/* $NetBSD: if_vioif.c,v 1.57 2020/05/25 08:25:28 yamaguchi Exp $ */ 2 3/* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.57 2020/05/25 08:25:28 yamaguchi Exp $"); 30 31#ifdef _KERNEL_OPT 32#include "opt_net_mpsafe.h" 33#endif 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/atomic.h> 39#include <sys/bus.h> 40#include <sys/condvar.h> 41#include <sys/device.h> 42#include <sys/intr.h> 43#include <sys/kmem.h> 44#include <sys/mbuf.h> 45#include <sys/mutex.h> 46#include <sys/sockio.h> 47#include <sys/cpu.h> 48#include <sys/module.h> 49#include <sys/pcq.h> 50#include <sys/workqueue.h> 51 52#include <dev/pci/virtioreg.h> 53#include <dev/pci/virtiovar.h> 54 55#include <net/if.h> 56#include <net/if_media.h> 57#include <net/if_ether.h> 58 59#include <net/bpf.h> 60 61#include "ioconf.h" 62 63#ifdef NET_MPSAFE 64#define VIOIF_MPSAFE 1 65#define VIOIF_MULTIQ 1 66#endif 67 68/* 69 * if_vioifreg.h: 70 */ 71/* Configuration registers */ 72#define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 73#define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 74#define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ 75 76/* Feature bits */ 77#define VIRTIO_NET_F_CSUM __BIT(0) 78#define VIRTIO_NET_F_GUEST_CSUM __BIT(1) 79#define VIRTIO_NET_F_MAC __BIT(5) 80#define VIRTIO_NET_F_GSO __BIT(6) 81#define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) 82#define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) 83#define VIRTIO_NET_F_GUEST_ECN __BIT(9) 84#define VIRTIO_NET_F_GUEST_UFO __BIT(10) 85#define VIRTIO_NET_F_HOST_TSO4 __BIT(11) 86#define VIRTIO_NET_F_HOST_TSO6 __BIT(12) 87#define VIRTIO_NET_F_HOST_ECN __BIT(13) 88#define VIRTIO_NET_F_HOST_UFO __BIT(14) 89#define VIRTIO_NET_F_MRG_RXBUF __BIT(15) 90#define VIRTIO_NET_F_STATUS __BIT(16) 91#define VIRTIO_NET_F_CTRL_VQ __BIT(17) 92#define VIRTIO_NET_F_CTRL_RX __BIT(18) 93#define VIRTIO_NET_F_CTRL_VLAN __BIT(19) 94#define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) 95#define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) 96#define VIRTIO_NET_F_MQ __BIT(22) 97 98#define VIRTIO_NET_FLAG_BITS \ 99 VIRTIO_COMMON_FLAG_BITS \ 100 "\x17""MQ" \ 101 "\x16""GUEST_ANNOUNCE" \ 102 "\x15""CTRL_RX_EXTRA" \ 103 "\x14""CTRL_VLAN" \ 104 "\x13""CTRL_RX" \ 105 "\x12""CTRL_VQ" \ 106 "\x11""STATUS" \ 107 "\x10""MRG_RXBUF" \ 108 "\x0f""HOST_UFO" \ 109 "\x0e""HOST_ECN" \ 110 "\x0d""HOST_TSO6" \ 111 "\x0c""HOST_TSO4" \ 112 "\x0b""GUEST_UFO" \ 113 "\x0a""GUEST_ECN" \ 114 "\x09""GUEST_TSO6" \ 115 "\x08""GUEST_TSO4" \ 116 "\x07""GSO" \ 117 "\x06""MAC" \ 118 "\x02""GUEST_CSUM" \ 119 "\x01""CSUM" 120 121/* Status */ 122#define VIRTIO_NET_S_LINK_UP 1 123 124/* Packet header structure */ 125struct virtio_net_hdr { 126 uint8_t flags; 127 uint8_t gso_type; 128 uint16_t hdr_len; 129 uint16_t gso_size; 130 uint16_t csum_start; 131 uint16_t csum_offset; 132#if 0 133 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 134#endif 135} __packed; 136 137#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 138#define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 139#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 140#define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 141#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 142#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 143 144#define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 145 146/* Control virtqueue */ 147struct virtio_net_ctrl_cmd { 148 uint8_t class; 149 uint8_t command; 150} __packed; 151#define VIRTIO_NET_CTRL_RX 0 152# define VIRTIO_NET_CTRL_RX_PROMISC 0 153# define VIRTIO_NET_CTRL_RX_ALLMULTI 1 154 155#define VIRTIO_NET_CTRL_MAC 1 156# define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 157 158#define VIRTIO_NET_CTRL_VLAN 2 159# define VIRTIO_NET_CTRL_VLAN_ADD 0 160# define VIRTIO_NET_CTRL_VLAN_DEL 1 161 162#define VIRTIO_NET_CTRL_MQ 4 163# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 164# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 165# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 166 167struct virtio_net_ctrl_status { 168 uint8_t ack; 169} __packed; 170#define VIRTIO_NET_OK 0 171#define VIRTIO_NET_ERR 1 172 173struct virtio_net_ctrl_rx { 174 uint8_t onoff; 175} __packed; 176 177struct virtio_net_ctrl_mac_tbl { 178 uint32_t nentries; 179 uint8_t macs[][ETHER_ADDR_LEN]; 180} __packed; 181 182struct virtio_net_ctrl_vlan { 183 uint16_t id; 184} __packed; 185 186struct virtio_net_ctrl_mq { 187 uint16_t virtqueue_pairs; 188} __packed; 189 190struct vioif_ctrl_cmdspec { 191 bus_dmamap_t dmamap; 192 void *buf; 193 bus_size_t bufsize; 194}; 195 196/* 197 * if_vioifvar.h: 198 */ 199 200/* 201 * Locking notes: 202 * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and 203 * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex). 204 * - more than one lock cannot be held at onece 205 * + ctrlq_inuse is protected by ctrlq_wait_lock. 206 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse 207 * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock 208 */ 209 210struct vioif_work { 211 struct work cookie; 212 void (*func)(void *); 213 void *arg; 214 unsigned int added; 215}; 216 217struct vioif_txqueue { 218 kmutex_t *txq_lock; /* lock for tx operations */ 219 220 struct virtqueue *txq_vq; 221 bool txq_stopping; 222 bool txq_link_active; 223 pcq_t *txq_intrq; 224 225 struct virtio_net_hdr *txq_hdrs; 226 bus_dmamap_t *txq_hdr_dmamaps; 227 228 struct mbuf **txq_mbufs; 229 bus_dmamap_t *txq_dmamaps; 230 231 void *txq_deferred_transmit; 232 void *txq_handle_si; 233 struct vioif_work txq_work; 234 bool txq_workqueue; 235 bool txq_active; 236}; 237 238struct vioif_rxqueue { 239 kmutex_t *rxq_lock; /* lock for rx operations */ 240 241 struct virtqueue *rxq_vq; 242 bool rxq_stopping; 243 244 struct virtio_net_hdr *rxq_hdrs; 245 bus_dmamap_t *rxq_hdr_dmamaps; 246 247 struct mbuf **rxq_mbufs; 248 bus_dmamap_t *rxq_dmamaps; 249 250 void *rxq_softint; 251 void *rxq_handle_si; 252 struct vioif_work rxq_work; 253 bool rxq_workqueue; 254 bool rxq_active; 255}; 256 257struct vioif_ctrlqueue { 258 struct virtqueue *ctrlq_vq; 259 enum { 260 FREE, INUSE, DONE 261 } ctrlq_inuse; 262 kcondvar_t ctrlq_wait; 263 kmutex_t ctrlq_wait_lock; 264 struct lwp *ctrlq_owner; 265 266 struct virtio_net_ctrl_cmd *ctrlq_cmd; 267 struct virtio_net_ctrl_status *ctrlq_status; 268 struct virtio_net_ctrl_rx *ctrlq_rx; 269 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; 270 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; 271 struct virtio_net_ctrl_mq *ctrlq_mq; 272 273 bus_dmamap_t ctrlq_cmd_dmamap; 274 bus_dmamap_t ctrlq_status_dmamap; 275 bus_dmamap_t ctrlq_rx_dmamap; 276 bus_dmamap_t ctrlq_tbl_uc_dmamap; 277 bus_dmamap_t ctrlq_tbl_mc_dmamap; 278 bus_dmamap_t ctrlq_mq_dmamap; 279}; 280 281struct vioif_softc { 282 device_t sc_dev; 283 struct sysctllog *sc_sysctllog; 284 285 struct virtio_softc *sc_virtio; 286 struct virtqueue *sc_vqs; 287 288 int sc_max_nvq_pairs; 289 int sc_req_nvq_pairs; 290 int sc_act_nvq_pairs; 291 292 uint8_t sc_mac[ETHER_ADDR_LEN]; 293 struct ethercom sc_ethercom; 294 short sc_deferred_init_done; 295 bool sc_link_active; 296 297 struct vioif_txqueue *sc_txq; 298 struct vioif_rxqueue *sc_rxq; 299 300 bool sc_has_ctrl; 301 struct vioif_ctrlqueue sc_ctrlq; 302 303 bus_dma_segment_t sc_hdr_segs[1]; 304 void *sc_dmamem; 305 void *sc_kmem; 306 307 void *sc_ctl_softint; 308 309 struct workqueue *sc_txrx_workqueue; 310 bool sc_txrx_workqueue_sysctl; 311 u_int sc_tx_intr_process_limit; 312 u_int sc_tx_process_limit; 313 u_int sc_rx_intr_process_limit; 314 u_int sc_rx_process_limit; 315}; 316#define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 317#define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 318 319#define VIOIF_TX_INTR_PROCESS_LIMIT 256 320#define VIOIF_TX_PROCESS_LIMIT 256 321#define VIOIF_RX_INTR_PROCESS_LIMIT 0U 322#define VIOIF_RX_PROCESS_LIMIT 256 323 324#define VIOIF_WORKQUEUE_PRI PRI_SOFTNET 325 326/* cfattach interface functions */ 327static int vioif_match(device_t, cfdata_t, void *); 328static void vioif_attach(device_t, device_t, void *); 329static void vioif_deferred_init(device_t); 330static int vioif_finalize_teardown(device_t); 331 332/* ifnet interface functions */ 333static int vioif_init(struct ifnet *); 334static void vioif_stop(struct ifnet *, int); 335static void vioif_start(struct ifnet *); 336static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); 337static int vioif_transmit(struct ifnet *, struct mbuf *); 338static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); 339static int vioif_ioctl(struct ifnet *, u_long, void *); 340static void vioif_watchdog(struct ifnet *); 341 342/* rx */ 343static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); 344static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); 345static void vioif_populate_rx_mbufs(struct vioif_rxqueue *); 346static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *); 347static void vioif_rx_queue_clear(struct vioif_rxqueue *); 348static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, 349 struct vioif_rxqueue *, u_int); 350static int vioif_rx_intr(void *); 351static void vioif_rx_handle(void *); 352static void vioif_rx_sched_handle(struct vioif_softc *, 353 struct vioif_rxqueue *); 354static void vioif_rx_softint(void *); 355static void vioif_rx_drain(struct vioif_rxqueue *); 356 357/* tx */ 358static int vioif_tx_intr(void *); 359static void vioif_tx_handle(void *); 360static void vioif_tx_sched_handle(struct vioif_softc *, 361 struct vioif_txqueue *); 362static void vioif_tx_queue_clear(struct vioif_txqueue *); 363static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, 364 struct vioif_txqueue *, u_int); 365static void vioif_tx_drain(struct vioif_txqueue *); 366static void vioif_deferred_transmit(void *); 367 368/* workqueue */ 369static struct workqueue* 370 vioif_workq_create(const char *, pri_t, int, int); 371static void vioif_workq_destroy(struct workqueue *); 372static void vioif_workq_work(struct work *, void *); 373static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); 374static void vioif_work_add(struct workqueue *, struct vioif_work *); 375static void vioif_work_wait(struct workqueue *, struct vioif_work *); 376 377/* other control */ 378static bool vioif_is_link_up(struct vioif_softc *); 379static void vioif_update_link_status(struct vioif_softc *); 380static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 381static int vioif_set_promisc(struct vioif_softc *, bool); 382static int vioif_set_allmulti(struct vioif_softc *, bool); 383static int vioif_set_rx_filter(struct vioif_softc *); 384static int vioif_rx_filter(struct vioif_softc *); 385static int vioif_ctrl_intr(void *); 386static int vioif_config_change(struct virtio_softc *); 387static void vioif_ctl_softint(void *); 388static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); 389static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); 390static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); 391static int vioif_setup_sysctl(struct vioif_softc *); 392 393CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 394 vioif_match, vioif_attach, NULL, NULL); 395 396static int 397vioif_match(device_t parent, cfdata_t match, void *aux) 398{ 399 struct virtio_attach_args *va = aux; 400 401 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 402 return 1; 403 404 return 0; 405} 406 407static void 408vioif_alloc_queues(struct vioif_softc *sc) 409{ 410 int nvq_pairs = sc->sc_max_nvq_pairs; 411 int nvqs = nvq_pairs * 2; 412 int i; 413 414 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 415 416 sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, 417 KM_SLEEP); 418 sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, 419 KM_SLEEP); 420 421 if (sc->sc_has_ctrl) 422 nvqs++; 423 424 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); 425 nvqs = 0; 426 for (i = 0; i < nvq_pairs; i++) { 427 sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; 428 sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; 429 } 430 431 if (sc->sc_has_ctrl) 432 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; 433} 434 435static void 436vioif_free_queues(struct vioif_softc *sc) 437{ 438 int nvq_pairs = sc->sc_max_nvq_pairs; 439 int nvqs = nvq_pairs * 2; 440 441 if (sc->sc_ctrlq.ctrlq_vq) 442 nvqs++; 443 444 if (sc->sc_txq) { 445 kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); 446 sc->sc_txq = NULL; 447 } 448 449 if (sc->sc_rxq) { 450 kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); 451 sc->sc_rxq = NULL; 452 } 453 454 if (sc->sc_vqs) { 455 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); 456 sc->sc_vqs = NULL; 457 } 458} 459 460/* allocate memory */ 461/* 462 * dma memory is used for: 463 * rxq_hdrs[slot]: metadata array for received frames (READ) 464 * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) 465 * ctrlq_cmd: command to be sent via ctrl vq (WRITE) 466 * ctrlq_status: return value for a command via ctrl vq (READ) 467 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command 468 * (WRITE) 469 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 470 * class command (WRITE) 471 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 472 * class command (WRITE) 473 * ctrlq_* structures are allocated only one each; they are protected by 474 * ctrlq_inuse variable and ctrlq_wait condvar. 475 */ 476/* 477 * dynamically allocated memory is used for: 478 * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 479 * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 480 * rxq_dmamaps[slot]: bus_dmamap_t array for received payload 481 * txq_dmamaps[slot]: bus_dmamap_t array for sent payload 482 * rxq_mbufs[slot]: mbuf pointer array for received frames 483 * txq_mbufs[slot]: mbuf pointer array for sent frames 484 */ 485static int 486vioif_alloc_mems(struct vioif_softc *sc) 487{ 488 struct virtio_softc *vsc = sc->sc_virtio; 489 struct vioif_txqueue *txq; 490 struct vioif_rxqueue *rxq; 491 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 492 int allocsize, allocsize2, r, rsegs, i, qid; 493 void *vaddr; 494 intptr_t p; 495 496 allocsize = 0; 497 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 498 rxq = &sc->sc_rxq[qid]; 499 txq = &sc->sc_txq[qid]; 500 501 allocsize += 502 sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num; 503 allocsize += 504 sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num; 505 } 506 if (sc->sc_has_ctrl) { 507 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 508 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 509 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 510 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 511 + sizeof(struct virtio_net_ctrl_mac_tbl) 512 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 513 allocsize += sizeof(struct virtio_net_ctrl_mq) * 1; 514 } 515 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 516 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 517 if (r != 0) { 518 aprint_error_dev(sc->sc_dev, 519 "DMA memory allocation failed, size %d, " 520 "error code %d\n", allocsize, r); 521 goto err_none; 522 } 523 r = bus_dmamem_map(virtio_dmat(vsc), 524 &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT); 525 if (r != 0) { 526 aprint_error_dev(sc->sc_dev, 527 "DMA memory map failed, error code %d\n", r); 528 goto err_dmamem_alloc; 529 } 530 531#define P(p, p0, p0size) do { p0 = (void *) p; \ 532 p += p0size; } while (0) 533 memset(vaddr, 0, allocsize); 534 sc->sc_dmamem = vaddr; 535 p = (intptr_t) vaddr; 536 537 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 538 rxq = &sc->sc_rxq[qid]; 539 txq = &sc->sc_txq[qid]; 540 541 P(p, rxq->rxq_hdrs, 542 sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num); 543 P(p, txq->txq_hdrs, 544 sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num); 545 } 546 if (sc->sc_has_ctrl) { 547 P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd)); 548 P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status)); 549 P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx)); 550 P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc)); 551 P(p, ctrlq->ctrlq_mac_tbl_mc, sizeof(*ctrlq->ctrlq_mac_tbl_mc) 552 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); 553 P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq)); 554 } 555 556 allocsize2 = 0; 557 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 558 int rxqsize, txqsize; 559 560 rxq = &sc->sc_rxq[qid]; 561 txq = &sc->sc_txq[qid]; 562 rxqsize = rxq->rxq_vq->vq_num; 563 txqsize = txq->txq_vq->vq_num; 564 565 allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; 566 allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; 567 allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; 568 569 allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; 570 allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; 571 allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; 572 } 573 vaddr = kmem_zalloc(allocsize2, KM_SLEEP); 574 sc->sc_kmem = vaddr; 575 p = (intptr_t) vaddr; 576 577 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 578 int rxqsize, txqsize; 579 rxq = &sc->sc_rxq[qid]; 580 txq = &sc->sc_txq[qid]; 581 rxqsize = rxq->rxq_vq->vq_num; 582 txqsize = txq->txq_vq->vq_num; 583 584 P(p, rxq->rxq_hdr_dmamaps, 585 sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); 586 P(p, txq->txq_hdr_dmamaps, 587 sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); 588 P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize); 589 P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize); 590 P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize); 591 P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize); 592 } 593#undef P 594 595#define C(map, size, nsegs, usage) \ 596 do { \ 597 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ 598 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, \ 599 &map); \ 600 if (r != 0) { \ 601 aprint_error_dev(sc->sc_dev, \ 602 usage " dmamap creation failed, " \ 603 "error code %d\n", r); \ 604 goto err_reqs; \ 605 } \ 606 } while (0) 607#define C_L(map, buf, size, nsegs, rw, usage) \ 608 C(map, size, nsegs, usage); \ 609 do { \ 610 r = bus_dmamap_load(virtio_dmat(vsc), map, \ 611 buf, size, NULL, \ 612 rw | BUS_DMA_NOWAIT); \ 613 if (r != 0) { \ 614 aprint_error_dev(sc->sc_dev, \ 615 usage " dmamap load failed, " \ 616 "error code %d\n", r); \ 617 goto err_reqs; \ 618 } \ 619 } while (0) 620 621 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 622 rxq = &sc->sc_rxq[qid]; 623 txq = &sc->sc_txq[qid]; 624 625 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 626 C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i], 627 sizeof(rxq->rxq_hdrs[0]), 1, 628 BUS_DMA_READ, "rx header"); 629 C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload"); 630 } 631 632 for (i = 0; i < txq->txq_vq->vq_num; i++) { 633 C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i], 634 sizeof(txq->txq_hdrs[0]), 1, 635 BUS_DMA_READ, "tx header"); 636 C(txq->txq_dmamaps[i], ETHER_MAX_LEN, 637 VIRTIO_NET_TX_MAXNSEGS, "tx payload"); 638 } 639 } 640 641 if (sc->sc_has_ctrl) { 642 /* control vq class & command */ 643 C_L(ctrlq->ctrlq_cmd_dmamap, 644 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, 645 BUS_DMA_WRITE, "control command"); 646 C_L(ctrlq->ctrlq_status_dmamap, 647 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, 648 BUS_DMA_READ, "control status"); 649 650 /* control vq rx mode command parameter */ 651 C_L(ctrlq->ctrlq_rx_dmamap, 652 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, 653 BUS_DMA_WRITE, "rx mode control command"); 654 655 /* multiqueue set command */ 656 C_L(ctrlq->ctrlq_mq_dmamap, 657 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, 658 BUS_DMA_WRITE, "multiqueue set command"); 659 660 /* control vq MAC filter table for unicast */ 661 /* do not load now since its length is variable */ 662 C(ctrlq->ctrlq_tbl_uc_dmamap, 663 sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1, 664 "unicast MAC address filter command"); 665 666 /* control vq MAC filter table for multicast */ 667 C(ctrlq->ctrlq_tbl_mc_dmamap, 668 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 669 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 670 "multicast MAC address filter command"); 671 } 672#undef C_L 673#undef C 674 675 return 0; 676 677err_reqs: 678#define D(map) \ 679 do { \ 680 if (map) { \ 681 bus_dmamap_destroy(virtio_dmat(vsc), map); \ 682 map = NULL; \ 683 } \ 684 } while (0) 685 D(ctrlq->ctrlq_tbl_mc_dmamap); 686 D(ctrlq->ctrlq_tbl_uc_dmamap); 687 D(ctrlq->ctrlq_rx_dmamap); 688 D(ctrlq->ctrlq_status_dmamap); 689 D(ctrlq->ctrlq_cmd_dmamap); 690 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 691 rxq = &sc->sc_rxq[qid]; 692 txq = &sc->sc_txq[qid]; 693 694 for (i = 0; i < txq->txq_vq->vq_num; i++) { 695 D(txq->txq_dmamaps[i]); 696 D(txq->txq_hdr_dmamaps[i]); 697 } 698 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 699 D(rxq->rxq_dmamaps[i]); 700 D(rxq->rxq_hdr_dmamaps[i]); 701 } 702 } 703#undef D 704 if (sc->sc_kmem) { 705 kmem_free(sc->sc_kmem, allocsize2); 706 sc->sc_kmem = NULL; 707 } 708 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); 709err_dmamem_alloc: 710 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 711err_none: 712 return -1; 713} 714 715static void 716vioif_attach(device_t parent, device_t self, void *aux) 717{ 718 struct vioif_softc *sc = device_private(self); 719 struct virtio_softc *vsc = device_private(parent); 720 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 721 struct vioif_txqueue *txq; 722 struct vioif_rxqueue *rxq; 723 uint32_t features, req_features; 724 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 725 u_int softint_flags; 726 int r, i, nvqs=0, req_flags; 727 char xnamebuf[MAXCOMLEN]; 728 729 if (virtio_child(vsc) != NULL) { 730 aprint_normal(": child already attached for %s; " 731 "something wrong...\n", device_xname(parent)); 732 return; 733 } 734 735 sc->sc_dev = self; 736 sc->sc_virtio = vsc; 737 sc->sc_link_active = false; 738 739 sc->sc_max_nvq_pairs = 1; 740 sc->sc_req_nvq_pairs = 1; 741 sc->sc_act_nvq_pairs = 1; 742 sc->sc_txrx_workqueue_sysctl = true; 743 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; 744 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; 745 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; 746 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; 747 748 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); 749 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, 750 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 751 if (sc->sc_txrx_workqueue == NULL) 752 goto err; 753 754 req_flags = 0; 755 756#ifdef VIOIF_MPSAFE 757 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 758#endif 759 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 760 761 req_features = 762 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 763 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; 764#ifdef VIOIF_MULTIQ 765 req_features |= VIRTIO_NET_F_MQ; 766#endif 767 virtio_child_attach_start(vsc, self, IPL_NET, NULL, 768 vioif_config_change, virtio_vq_intrhand, req_flags, 769 req_features, VIRTIO_NET_FLAG_BITS); 770 771 features = virtio_features(vsc); 772 773 if (features & VIRTIO_NET_F_MAC) { 774 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 775 sc->sc_mac[i] = virtio_read_device_config_1(vsc, 776 VIRTIO_NET_CONFIG_MAC + i); 777 } 778 } else { 779 /* code stolen from sys/net/if_tap.c */ 780 struct timeval tv; 781 uint32_t ui; 782 getmicrouptime(&tv); 783 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 784 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 785 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 786 virtio_write_device_config_1(vsc, 787 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 788 } 789 } 790 791 aprint_normal_dev(self, "Ethernet address %s\n", 792 ether_sprintf(sc->sc_mac)); 793 794 if ((features & VIRTIO_NET_F_CTRL_VQ) && 795 (features & VIRTIO_NET_F_CTRL_RX)) { 796 sc->sc_has_ctrl = true; 797 798 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); 799 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); 800 ctrlq->ctrlq_inuse = FREE; 801 } else { 802 sc->sc_has_ctrl = false; 803 } 804 805 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { 806 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, 807 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); 808 809 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 810 goto err; 811 812 /* Limit the number of queue pairs to use */ 813 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); 814 } 815 816 vioif_alloc_queues(sc); 817 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); 818 819#ifdef VIOIF_MPSAFE 820 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; 821#else 822 softint_flags = SOFTINT_NET; 823#endif 824 825 /* 826 * Allocating virtqueues 827 */ 828 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 829 rxq = &sc->sc_rxq[i]; 830 txq = &sc->sc_txq[i]; 831 char qname[32]; 832 833 rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 834 835 rxq->rxq_softint = softint_establish(softint_flags, 836 vioif_rx_softint, rxq); 837 if (rxq->rxq_softint == NULL) { 838 aprint_error_dev(self, "cannot establish rx softint\n"); 839 goto err; 840 } 841 rxq->rxq_handle_si = softint_establish(softint_flags, 842 vioif_rx_handle, rxq); 843 if (rxq->rxq_handle_si == NULL) { 844 aprint_error_dev(self, "cannot establish rx softint\n"); 845 goto err; 846 } 847 848 snprintf(qname, sizeof(qname), "rx%d", i); 849 r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, 850 MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname); 851 if (r != 0) 852 goto err; 853 nvqs++; 854 rxq->rxq_vq->vq_intrhand = vioif_rx_intr; 855 rxq->rxq_vq->vq_intrhand_arg = (void *)rxq; 856 rxq->rxq_stopping = true; 857 vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq); 858 859 txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 860 861 txq->txq_deferred_transmit = softint_establish(softint_flags, 862 vioif_deferred_transmit, txq); 863 if (txq->txq_deferred_transmit == NULL) { 864 aprint_error_dev(self, "cannot establish tx softint\n"); 865 goto err; 866 } 867 txq->txq_handle_si = softint_establish(softint_flags, 868 vioif_tx_handle, txq); 869 if (txq->txq_handle_si == NULL) { 870 aprint_error_dev(self, "cannot establish tx softint\n"); 871 goto err; 872 } 873 874 snprintf(qname, sizeof(qname), "tx%d", i); 875 r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, 876 sizeof(struct virtio_net_hdr) 877 + (ETHER_MAX_LEN - ETHER_HDR_LEN), 878 VIRTIO_NET_TX_MAXNSEGS + 1, qname); 879 if (r != 0) 880 goto err; 881 nvqs++; 882 txq->txq_vq->vq_intrhand = vioif_tx_intr; 883 txq->txq_vq->vq_intrhand_arg = (void *)txq; 884 txq->txq_link_active = sc->sc_link_active; 885 txq->txq_stopping = false; 886 txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP); 887 vioif_work_set(&txq->txq_work, vioif_tx_handle, txq); 888 } 889 890 if (sc->sc_has_ctrl) { 891 /* 892 * Allocating a virtqueue for control channel 893 */ 894 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, 895 NBPG, 1, "control"); 896 if (r != 0) { 897 aprint_error_dev(self, "failed to allocate " 898 "a virtqueue for control channel, error code %d\n", 899 r); 900 901 sc->sc_has_ctrl = false; 902 cv_destroy(&ctrlq->ctrlq_wait); 903 mutex_destroy(&ctrlq->ctrlq_wait_lock); 904 } else { 905 nvqs++; 906 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; 907 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; 908 } 909 } 910 911 sc->sc_ctl_softint = softint_establish(softint_flags, 912 vioif_ctl_softint, sc); 913 if (sc->sc_ctl_softint == NULL) { 914 aprint_error_dev(self, "cannot establish ctl softint\n"); 915 goto err; 916 } 917 918 if (vioif_alloc_mems(sc) < 0) 919 goto err; 920 921 if (virtio_child_attach_finish(vsc) != 0) 922 goto err; 923 924 if (vioif_setup_sysctl(sc) != 0) { 925 aprint_error_dev(self, "unable to create sysctl node\n"); 926 /* continue */ 927 } 928 929 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 930 ifp->if_softc = sc; 931 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 932#ifdef VIOIF_MPSAFE 933 ifp->if_extflags = IFEF_MPSAFE; 934#endif 935 ifp->if_start = vioif_start; 936 if (sc->sc_req_nvq_pairs > 1) 937 ifp->if_transmit = vioif_transmit; 938 ifp->if_ioctl = vioif_ioctl; 939 ifp->if_init = vioif_init; 940 ifp->if_stop = vioif_stop; 941 ifp->if_capabilities = 0; 942 ifp->if_watchdog = vioif_watchdog; 943 txq = &sc->sc_txq[0]; 944 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); 945 IFQ_SET_READY(&ifp->if_snd); 946 947 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 948 949 if_attach(ifp); 950 if_deferred_start_init(ifp, NULL); 951 ether_ifattach(ifp, sc->sc_mac); 952 953 return; 954 955err: 956 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 957 rxq = &sc->sc_rxq[i]; 958 txq = &sc->sc_txq[i]; 959 960 if (rxq->rxq_lock) { 961 mutex_obj_free(rxq->rxq_lock); 962 rxq->rxq_lock = NULL; 963 } 964 965 if (rxq->rxq_softint) { 966 softint_disestablish(rxq->rxq_softint); 967 rxq->rxq_softint = NULL; 968 } 969 970 if (rxq->rxq_handle_si) { 971 softint_disestablish(rxq->rxq_handle_si); 972 rxq->rxq_handle_si = NULL; 973 } 974 975 if (txq->txq_lock) { 976 mutex_obj_free(txq->txq_lock); 977 txq->txq_lock = NULL; 978 } 979 980 if (txq->txq_handle_si) { 981 softint_disestablish(txq->txq_handle_si); 982 txq->txq_handle_si = NULL; 983 } 984 985 if (txq->txq_deferred_transmit) { 986 softint_disestablish(txq->txq_deferred_transmit); 987 txq->txq_deferred_transmit = NULL; 988 } 989 990 if (txq->txq_intrq) { 991 pcq_destroy(txq->txq_intrq); 992 txq->txq_intrq = NULL; 993 } 994 } 995 996 if (sc->sc_has_ctrl) { 997 cv_destroy(&ctrlq->ctrlq_wait); 998 mutex_destroy(&ctrlq->ctrlq_wait_lock); 999 } 1000 1001 while (nvqs > 0) 1002 virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); 1003 1004 vioif_free_queues(sc); 1005 virtio_child_attach_failed(vsc); 1006 config_finalize_register(self, vioif_finalize_teardown); 1007 1008 return; 1009} 1010 1011static int 1012vioif_finalize_teardown(device_t self) 1013{ 1014 struct vioif_softc *sc = device_private(self); 1015 1016 if (sc->sc_txrx_workqueue != NULL) { 1017 vioif_workq_destroy(sc->sc_txrx_workqueue); 1018 sc->sc_txrx_workqueue = NULL; 1019 } 1020 1021 return 0; 1022} 1023 1024/* we need interrupts to make promiscuous mode off */ 1025static void 1026vioif_deferred_init(device_t self) 1027{ 1028 struct vioif_softc *sc = device_private(self); 1029 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1030 int r; 1031 1032 if (ifp->if_flags & IFF_PROMISC) 1033 return; 1034 1035 r = vioif_set_promisc(sc, false); 1036 if (r != 0) 1037 aprint_error_dev(self, "resetting promisc mode failed, " 1038 "error code %d\n", r); 1039} 1040 1041static void 1042vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) 1043{ 1044 struct virtio_softc *vsc = sc->sc_virtio; 1045 struct vioif_txqueue *txq; 1046 struct vioif_rxqueue *rxq; 1047 int i; 1048 1049 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1050 txq = &sc->sc_txq[i]; 1051 rxq = &sc->sc_rxq[i]; 1052 1053 virtio_start_vq_intr(vsc, txq->txq_vq); 1054 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1055 } 1056} 1057 1058static void 1059vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) 1060{ 1061 struct virtio_softc *vsc = sc->sc_virtio; 1062 struct vioif_txqueue *txq; 1063 struct vioif_rxqueue *rxq; 1064 int i; 1065 1066 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1067 txq = &sc->sc_txq[i]; 1068 rxq = &sc->sc_rxq[i]; 1069 1070 virtio_stop_vq_intr(vsc, txq->txq_vq); 1071 virtio_stop_vq_intr(vsc, rxq->rxq_vq); 1072 } 1073} 1074 1075/* 1076 * Interface functions for ifnet 1077 */ 1078static int 1079vioif_init(struct ifnet *ifp) 1080{ 1081 struct vioif_softc *sc = ifp->if_softc; 1082 struct virtio_softc *vsc = sc->sc_virtio; 1083 struct vioif_rxqueue *rxq; 1084 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1085 int r, i; 1086 1087 vioif_stop(ifp, 0); 1088 1089 virtio_reinit_start(vsc); 1090 virtio_negotiate_features(vsc, virtio_features(vsc)); 1091 1092 for (i = 0; i < sc->sc_req_nvq_pairs; i++) { 1093 rxq = &sc->sc_rxq[i]; 1094 1095 /* Have to set false before vioif_populate_rx_mbufs */ 1096 rxq->rxq_stopping = false; 1097 vioif_populate_rx_mbufs(rxq); 1098 } 1099 1100 virtio_reinit_end(vsc); 1101 1102 if (sc->sc_has_ctrl) 1103 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 1104 1105 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); 1106 if (r == 0) 1107 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; 1108 else 1109 sc->sc_act_nvq_pairs = 1; 1110 1111 for (i = 0; i < sc->sc_act_nvq_pairs; i++) 1112 sc->sc_txq[i].txq_stopping = false; 1113 1114 vioif_enable_interrupt_vqpairs(sc); 1115 1116 if (!sc->sc_deferred_init_done) { 1117 sc->sc_deferred_init_done = 1; 1118 if (sc->sc_has_ctrl) 1119 vioif_deferred_init(sc->sc_dev); 1120 } 1121 1122 vioif_update_link_status(sc); 1123 ifp->if_flags |= IFF_RUNNING; 1124 ifp->if_flags &= ~IFF_OACTIVE; 1125 vioif_rx_filter(sc); 1126 1127 return 0; 1128} 1129 1130static void 1131vioif_stop(struct ifnet *ifp, int disable) 1132{ 1133 struct vioif_softc *sc = ifp->if_softc; 1134 struct virtio_softc *vsc = sc->sc_virtio; 1135 struct vioif_txqueue *txq; 1136 struct vioif_rxqueue *rxq; 1137 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1138 int i; 1139 1140 /* Take the locks to ensure that ongoing TX/RX finish */ 1141 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1142 txq = &sc->sc_txq[i]; 1143 rxq = &sc->sc_rxq[i]; 1144 1145 mutex_enter(txq->txq_lock); 1146 txq->txq_stopping = true; 1147 mutex_exit(txq->txq_lock); 1148 1149 mutex_enter(rxq->rxq_lock); 1150 rxq->rxq_stopping = true; 1151 mutex_exit(rxq->rxq_lock); 1152 } 1153 1154 /* disable interrupts */ 1155 vioif_disable_interrupt_vqpairs(sc); 1156 1157 if (sc->sc_has_ctrl) 1158 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); 1159 1160 /* only way to stop I/O and DMA is resetting... */ 1161 virtio_reset(vsc); 1162 1163 /* rendezvous for finish of handlers */ 1164 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1165 txq = &sc->sc_txq[i]; 1166 rxq = &sc->sc_rxq[i]; 1167 1168 mutex_enter(txq->txq_lock); 1169 mutex_exit(txq->txq_lock); 1170 1171 mutex_enter(rxq->rxq_lock); 1172 mutex_exit(rxq->rxq_lock); 1173 1174 vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work); 1175 vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work); 1176 } 1177 1178 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1179 vioif_rx_queue_clear(&sc->sc_rxq[i]); 1180 vioif_tx_queue_clear(&sc->sc_txq[i]); 1181 } 1182 1183 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1184 sc->sc_link_active = false; 1185 1186 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1187 txq = &sc->sc_txq[i]; 1188 rxq = &sc->sc_rxq[i]; 1189 1190 txq->txq_link_active = false; 1191 1192 if (disable) 1193 vioif_rx_drain(rxq); 1194 1195 vioif_tx_drain(txq); 1196 } 1197} 1198 1199static void 1200vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, 1201 bool is_transmit) 1202{ 1203 struct vioif_softc *sc = ifp->if_softc; 1204 struct virtio_softc *vsc = sc->sc_virtio; 1205 struct virtqueue *vq = txq->txq_vq; 1206 struct mbuf *m; 1207 int queued = 0; 1208 1209 KASSERT(mutex_owned(txq->txq_lock)); 1210 1211 if ((ifp->if_flags & IFF_RUNNING) == 0) 1212 return; 1213 1214 if (!txq->txq_link_active || txq->txq_stopping) 1215 return; 1216 1217 if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) 1218 return; 1219 1220 for (;;) { 1221 int slot, r; 1222 1223 if (is_transmit) 1224 m = pcq_get(txq->txq_intrq); 1225 else 1226 IFQ_DEQUEUE(&ifp->if_snd, m); 1227 1228 if (m == NULL) 1229 break; 1230 1231 r = virtio_enqueue_prep(vsc, vq, &slot); 1232 if (r == EAGAIN) { 1233 ifp->if_flags |= IFF_OACTIVE; 1234 m_freem(m); 1235 break; 1236 } 1237 if (r != 0) 1238 panic("enqueue_prep for a tx buffer"); 1239 1240 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1241 txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1242 if (r != 0) { 1243 /* maybe just too fragmented */ 1244 struct mbuf *newm; 1245 1246 newm = m_defrag(m, M_NOWAIT); 1247 if (newm == NULL) { 1248 aprint_error_dev(sc->sc_dev, 1249 "m_defrag() failed\n"); 1250 goto skip; 1251 } 1252 1253 m = newm; 1254 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1255 txq->txq_dmamaps[slot], m, 1256 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1257 if (r != 0) { 1258 aprint_error_dev(sc->sc_dev, 1259 "tx dmamap load failed, error code %d\n", 1260 r); 1261skip: 1262 m_freem(m); 1263 virtio_enqueue_abort(vsc, vq, slot); 1264 continue; 1265 } 1266 } 1267 1268 /* This should actually never fail */ 1269 r = virtio_enqueue_reserve(vsc, vq, slot, 1270 txq->txq_dmamaps[slot]->dm_nsegs + 1); 1271 if (r != 0) { 1272 aprint_error_dev(sc->sc_dev, 1273 "virtio_enqueue_reserve failed, error code %d\n", 1274 r); 1275 bus_dmamap_unload(virtio_dmat(vsc), 1276 txq->txq_dmamaps[slot]); 1277 /* slot already freed by virtio_enqueue_reserve */ 1278 m_freem(m); 1279 continue; 1280 } 1281 1282 txq->txq_mbufs[slot] = m; 1283 1284 memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 1285 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1286 0, txq->txq_dmamaps[slot]->dm_mapsize, 1287 BUS_DMASYNC_PREWRITE); 1288 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1289 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, 1290 BUS_DMASYNC_PREWRITE); 1291 virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); 1292 virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); 1293 virtio_enqueue_commit(vsc, vq, slot, false); 1294 1295 queued++; 1296 bpf_mtap(ifp, m, BPF_D_OUT); 1297 } 1298 1299 if (queued > 0) { 1300 virtio_enqueue_commit(vsc, vq, -1, true); 1301 ifp->if_timer = 5; 1302 } 1303} 1304 1305static void 1306vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1307{ 1308 1309 /* 1310 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. 1311 */ 1312 vioif_send_common_locked(ifp, txq, false); 1313 1314} 1315 1316static void 1317vioif_start(struct ifnet *ifp) 1318{ 1319 struct vioif_softc *sc = ifp->if_softc; 1320 struct vioif_txqueue *txq = &sc->sc_txq[0]; 1321 1322#ifdef VIOIF_MPSAFE 1323 KASSERT(if_is_mpsafe(ifp)); 1324#endif 1325 1326 mutex_enter(txq->txq_lock); 1327 if (!txq->txq_stopping) 1328 vioif_start_locked(ifp, txq); 1329 mutex_exit(txq->txq_lock); 1330} 1331 1332static inline int 1333vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) 1334{ 1335 struct vioif_softc *sc = ifp->if_softc; 1336 u_int cpuid = cpu_index(curcpu()); 1337 1338 return cpuid % sc->sc_act_nvq_pairs; 1339} 1340 1341static void 1342vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1343{ 1344 1345 vioif_send_common_locked(ifp, txq, true); 1346} 1347 1348static int 1349vioif_transmit(struct ifnet *ifp, struct mbuf *m) 1350{ 1351 struct vioif_softc *sc = ifp->if_softc; 1352 struct vioif_txqueue *txq; 1353 int qid; 1354 1355 qid = vioif_select_txqueue(ifp, m); 1356 txq = &sc->sc_txq[qid]; 1357 1358 if (__predict_false(!pcq_put(txq->txq_intrq, m))) { 1359 m_freem(m); 1360 return ENOBUFS; 1361 } 1362 1363 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1364 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1365 if (m->m_flags & M_MCAST) 1366 if_statinc_ref(nsr, if_omcasts); 1367 IF_STAT_PUTREF(ifp); 1368 1369 if (mutex_tryenter(txq->txq_lock)) { 1370 if (!txq->txq_stopping) 1371 vioif_transmit_locked(ifp, txq); 1372 mutex_exit(txq->txq_lock); 1373 } 1374 1375 return 0; 1376} 1377 1378static void 1379vioif_deferred_transmit(void *arg) 1380{ 1381 struct vioif_txqueue *txq = arg; 1382 struct virtio_softc *vsc = txq->txq_vq->vq_owner; 1383 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1384 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1385 1386 if (mutex_tryenter(txq->txq_lock)) { 1387 vioif_send_common_locked(ifp, txq, true); 1388 mutex_exit(txq->txq_lock); 1389 } 1390} 1391 1392static int 1393vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1394{ 1395 int s, r; 1396 1397 s = splnet(); 1398 1399 r = ether_ioctl(ifp, cmd, data); 1400 if ((r == 0 && cmd == SIOCSIFFLAGS) || 1401 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 1402 if (ifp->if_flags & IFF_RUNNING) 1403 r = vioif_rx_filter(ifp->if_softc); 1404 else 1405 r = 0; 1406 } 1407 1408 splx(s); 1409 1410 return r; 1411} 1412 1413void 1414vioif_watchdog(struct ifnet *ifp) 1415{ 1416 struct vioif_softc *sc = ifp->if_softc; 1417 int i; 1418 1419 if (ifp->if_flags & IFF_RUNNING) { 1420 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1421 vioif_tx_queue_clear(&sc->sc_txq[i]); 1422 } 1423 } 1424} 1425 1426/* 1427 * Receive implementation 1428 */ 1429/* allocate and initialize a mbuf for receive */ 1430static int 1431vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1432{ 1433 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1434 struct mbuf *m; 1435 int r; 1436 1437 MGETHDR(m, M_DONTWAIT, MT_DATA); 1438 if (m == NULL) 1439 return ENOBUFS; 1440 MCLGET(m, M_DONTWAIT); 1441 if ((m->m_flags & M_EXT) == 0) { 1442 m_freem(m); 1443 return ENOBUFS; 1444 } 1445 rxq->rxq_mbufs[i] = m; 1446 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 1447 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1448 rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT); 1449 if (r) { 1450 m_freem(m); 1451 rxq->rxq_mbufs[i] = NULL; 1452 return r; 1453 } 1454 1455 return 0; 1456} 1457 1458/* free a mbuf for receive */ 1459static void 1460vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1461{ 1462 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1463 1464 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); 1465 m_freem(rxq->rxq_mbufs[i]); 1466 rxq->rxq_mbufs[i] = NULL; 1467} 1468 1469/* add mbufs for all the empty receive slots */ 1470static void 1471vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq) 1472{ 1473 1474 mutex_enter(rxq->rxq_lock); 1475 vioif_populate_rx_mbufs_locked(rxq); 1476 mutex_exit(rxq->rxq_lock); 1477} 1478 1479static void 1480vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq) 1481{ 1482 struct virtqueue *vq = rxq->rxq_vq; 1483 struct virtio_softc *vsc = vq->vq_owner; 1484 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1485 int i, r, ndone = 0; 1486 1487 KASSERT(mutex_owned(rxq->rxq_lock)); 1488 1489 if (rxq->rxq_stopping) 1490 return; 1491 1492 for (i = 0; i < vq->vq_num; i++) { 1493 int slot; 1494 r = virtio_enqueue_prep(vsc, vq, &slot); 1495 if (r == EAGAIN) 1496 break; 1497 if (r != 0) 1498 panic("enqueue_prep for rx buffers"); 1499 if (rxq->rxq_mbufs[slot] == NULL) { 1500 r = vioif_add_rx_mbuf(rxq, slot); 1501 if (r != 0) { 1502 aprint_error_dev(sc->sc_dev, 1503 "rx mbuf allocation failed, " 1504 "error code %d\n", r); 1505 break; 1506 } 1507 } 1508 r = virtio_enqueue_reserve(vsc, vq, slot, 1509 rxq->rxq_dmamaps[slot]->dm_nsegs + 1); 1510 if (r != 0) { 1511 vioif_free_rx_mbuf(rxq, slot); 1512 break; 1513 } 1514 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1515 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 1516 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1517 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1518 virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], 1519 false); 1520 virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); 1521 virtio_enqueue_commit(vsc, vq, slot, false); 1522 ndone++; 1523 } 1524 if (ndone > 0) 1525 virtio_enqueue_commit(vsc, vq, -1, true); 1526} 1527 1528static void 1529vioif_rx_queue_clear(struct vioif_rxqueue *rxq) 1530{ 1531 struct virtqueue *vq = rxq->rxq_vq; 1532 struct virtio_softc *vsc = vq->vq_owner; 1533 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1534 u_int limit = UINT_MAX; 1535 bool more; 1536 1537 KASSERT(rxq->rxq_stopping); 1538 1539 mutex_enter(rxq->rxq_lock); 1540 for (;;) { 1541 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1542 if (more == false) 1543 break; 1544 } 1545 mutex_exit(rxq->rxq_lock); 1546} 1547 1548/* dequeue received packets */ 1549static bool 1550vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1551 struct vioif_rxqueue *rxq, u_int limit) 1552{ 1553 struct virtqueue *vq = rxq->rxq_vq; 1554 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1555 struct mbuf *m; 1556 int slot, len; 1557 bool more = false, dequeued = false; 1558 1559 KASSERT(mutex_owned(rxq->rxq_lock)); 1560 1561 if (virtio_vq_is_enqueued(vsc, vq) == false) 1562 return false; 1563 1564 for (;;) { 1565 if (limit-- == 0) { 1566 more = true; 1567 break; 1568 } 1569 1570 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1571 break; 1572 1573 dequeued = true; 1574 1575 len -= sizeof(struct virtio_net_hdr); 1576 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1577 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD); 1578 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1579 0, MCLBYTES, BUS_DMASYNC_POSTREAD); 1580 m = rxq->rxq_mbufs[slot]; 1581 KASSERT(m != NULL); 1582 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); 1583 rxq->rxq_mbufs[slot] = NULL; 1584 virtio_dequeue_commit(vsc, vq, slot); 1585 m_set_rcvif(m, ifp); 1586 m->m_len = m->m_pkthdr.len = len; 1587 1588 mutex_exit(rxq->rxq_lock); 1589 if_percpuq_enqueue(ifp->if_percpuq, m); 1590 mutex_enter(rxq->rxq_lock); 1591 1592 if (rxq->rxq_stopping) 1593 break; 1594 } 1595 1596 if (dequeued) 1597 softint_schedule(rxq->rxq_softint); 1598 1599 return more; 1600} 1601 1602/* rx interrupt; call _dequeue above and schedule a softint */ 1603static int 1604vioif_rx_intr(void *arg) 1605{ 1606 struct vioif_rxqueue *rxq = arg; 1607 struct virtqueue *vq = rxq->rxq_vq; 1608 struct virtio_softc *vsc = vq->vq_owner; 1609 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1610 u_int limit; 1611 bool more; 1612 1613 limit = sc->sc_rx_intr_process_limit; 1614 1615 if (atomic_load_relaxed(&rxq->rxq_active) == true) 1616 return 1; 1617 1618 mutex_enter(rxq->rxq_lock); 1619 1620 if (!rxq->rxq_stopping) { 1621 rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl; 1622 1623 virtio_stop_vq_intr(vsc, vq); 1624 atomic_store_relaxed(&rxq->rxq_active, true); 1625 1626 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1627 if (more) { 1628 vioif_rx_sched_handle(sc, rxq); 1629 } else { 1630 atomic_store_relaxed(&rxq->rxq_active, false); 1631 virtio_start_vq_intr(vsc, vq); 1632 } 1633 } 1634 1635 mutex_exit(rxq->rxq_lock); 1636 return 1; 1637} 1638 1639static void 1640vioif_rx_handle(void *xrxq) 1641{ 1642 struct vioif_rxqueue *rxq = xrxq; 1643 struct virtqueue *vq = rxq->rxq_vq; 1644 struct virtio_softc *vsc = vq->vq_owner; 1645 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1646 u_int limit; 1647 bool more; 1648 1649 limit = sc->sc_rx_process_limit; 1650 1651 mutex_enter(rxq->rxq_lock); 1652 1653 if (!rxq->rxq_stopping) { 1654 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1655 if (more) { 1656 vioif_rx_sched_handle(sc, rxq); 1657 } else { 1658 atomic_store_relaxed(&rxq->rxq_active, false); 1659 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1660 } 1661 } 1662 1663 mutex_exit(rxq->rxq_lock); 1664} 1665 1666static void 1667vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq) 1668{ 1669 1670 if (rxq->rxq_workqueue) 1671 vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work); 1672 else 1673 softint_schedule(rxq->rxq_handle_si); 1674} 1675 1676/* softint: enqueue receive requests for new incoming packets */ 1677static void 1678vioif_rx_softint(void *arg) 1679{ 1680 struct vioif_rxqueue *rxq = arg; 1681 1682 vioif_populate_rx_mbufs(rxq); 1683} 1684 1685/* free all the mbufs; called from if_stop(disable) */ 1686static void 1687vioif_rx_drain(struct vioif_rxqueue *rxq) 1688{ 1689 struct virtqueue *vq = rxq->rxq_vq; 1690 int i; 1691 1692 for (i = 0; i < vq->vq_num; i++) { 1693 if (rxq->rxq_mbufs[i] == NULL) 1694 continue; 1695 vioif_free_rx_mbuf(rxq, i); 1696 } 1697} 1698 1699/* 1700 * Transmition implementation 1701 */ 1702/* actual transmission is done in if_start */ 1703/* tx interrupt; dequeue and free mbufs */ 1704/* 1705 * tx interrupt is actually disabled; this should be called upon 1706 * tx vq full and watchdog 1707 */ 1708 1709static int 1710vioif_tx_intr(void *arg) 1711{ 1712 struct vioif_txqueue *txq = arg; 1713 struct virtqueue *vq = txq->txq_vq; 1714 struct virtio_softc *vsc = vq->vq_owner; 1715 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1716 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1717 bool more; 1718 u_int limit; 1719 1720 limit = sc->sc_tx_intr_process_limit; 1721 1722 if (atomic_load_relaxed(&txq->txq_active) == true) 1723 return 1; 1724 1725 mutex_enter(txq->txq_lock); 1726 1727 if (!txq->txq_stopping) { 1728 txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl; 1729 1730 virtio_stop_vq_intr(vsc, vq); 1731 atomic_store_relaxed(&txq->txq_active, true); 1732 1733 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1734 if (more) { 1735 vioif_tx_sched_handle(sc, txq); 1736 } else { 1737 atomic_store_relaxed(&txq->txq_active, false); 1738 1739 /* for ALTQ */ 1740 if (txq == &sc->sc_txq[0]) { 1741 if_schedule_deferred_start(ifp); 1742 ifp->if_flags &= ~IFF_OACTIVE; 1743 } 1744 softint_schedule(txq->txq_deferred_transmit); 1745 1746 virtio_start_vq_intr(vsc, vq); 1747 } 1748 } 1749 1750 mutex_exit(txq->txq_lock); 1751 1752 return 1; 1753} 1754 1755static void 1756vioif_tx_handle(void *xtxq) 1757{ 1758 struct vioif_txqueue *txq = xtxq; 1759 struct virtqueue *vq = txq->txq_vq; 1760 struct virtio_softc *vsc = vq->vq_owner; 1761 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1762 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1763 u_int limit; 1764 bool more; 1765 1766 limit = sc->sc_tx_process_limit; 1767 1768 mutex_enter(txq->txq_lock); 1769 1770 if (!txq->txq_stopping) { 1771 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1772 if (more) { 1773 vioif_tx_sched_handle(sc, txq); 1774 } else { 1775 atomic_store_relaxed(&txq->txq_active, false); 1776 1777 /* for ALTQ */ 1778 if (txq == &sc->sc_txq[0]) { 1779 if_schedule_deferred_start(ifp); 1780 ifp->if_flags &= ~IFF_OACTIVE; 1781 } 1782 softint_schedule(txq->txq_deferred_transmit); 1783 1784 virtio_start_vq_intr(vsc, txq->txq_vq); 1785 } 1786 } 1787 1788 mutex_exit(txq->txq_lock); 1789} 1790 1791static void 1792vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq) 1793{ 1794 1795 if (txq->txq_workqueue) 1796 vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work); 1797 else 1798 softint_schedule(txq->txq_handle_si); 1799} 1800 1801static void 1802vioif_tx_queue_clear(struct vioif_txqueue *txq) 1803{ 1804 struct virtqueue *vq = txq->txq_vq; 1805 struct virtio_softc *vsc = vq->vq_owner; 1806 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1807 u_int limit = UINT_MAX; 1808 bool more; 1809 1810 mutex_enter(txq->txq_lock); 1811 for (;;) { 1812 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1813 if (more == false) 1814 break; 1815 } 1816 mutex_exit(txq->txq_lock); 1817} 1818 1819static bool 1820vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1821 struct vioif_txqueue *txq, u_int limit) 1822{ 1823 struct virtqueue *vq = txq->txq_vq; 1824 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1825 struct mbuf *m; 1826 int slot, len; 1827 bool more = false; 1828 1829 KASSERT(mutex_owned(txq->txq_lock)); 1830 1831 if (virtio_vq_is_enqueued(vsc, vq) == false) 1832 return false; 1833 1834 for (;;) { 1835 if (limit-- == 0) { 1836 more = true; 1837 break; 1838 } 1839 1840 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1841 break; 1842 1843 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1844 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE); 1845 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1846 0, txq->txq_dmamaps[slot]->dm_mapsize, 1847 BUS_DMASYNC_POSTWRITE); 1848 m = txq->txq_mbufs[slot]; 1849 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); 1850 txq->txq_mbufs[slot] = NULL; 1851 virtio_dequeue_commit(vsc, vq, slot); 1852 if_statinc(ifp, if_opackets); 1853 m_freem(m); 1854 } 1855 1856 return more; 1857} 1858 1859/* free all the mbufs already put on vq; called from if_stop(disable) */ 1860static void 1861vioif_tx_drain(struct vioif_txqueue *txq) 1862{ 1863 struct virtqueue *vq = txq->txq_vq; 1864 struct virtio_softc *vsc = vq->vq_owner; 1865 int i; 1866 1867 KASSERT(txq->txq_stopping); 1868 1869 for (i = 0; i < vq->vq_num; i++) { 1870 if (txq->txq_mbufs[i] == NULL) 1871 continue; 1872 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); 1873 m_freem(txq->txq_mbufs[i]); 1874 txq->txq_mbufs[i] = NULL; 1875 } 1876} 1877 1878/* 1879 * Control vq 1880 */ 1881/* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1882static void 1883vioif_ctrl_acquire(struct vioif_softc *sc) 1884{ 1885 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1886 1887 mutex_enter(&ctrlq->ctrlq_wait_lock); 1888 while (ctrlq->ctrlq_inuse != FREE) 1889 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 1890 ctrlq->ctrlq_inuse = INUSE; 1891 ctrlq->ctrlq_owner = curlwp; 1892 mutex_exit(&ctrlq->ctrlq_wait_lock); 1893} 1894 1895static void 1896vioif_ctrl_release(struct vioif_softc *sc) 1897{ 1898 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1899 1900 KASSERT(ctrlq->ctrlq_inuse != FREE); 1901 KASSERT(ctrlq->ctrlq_owner == curlwp); 1902 1903 mutex_enter(&ctrlq->ctrlq_wait_lock); 1904 ctrlq->ctrlq_inuse = FREE; 1905 ctrlq->ctrlq_owner = NULL; 1906 cv_signal(&ctrlq->ctrlq_wait); 1907 mutex_exit(&ctrlq->ctrlq_wait_lock); 1908} 1909 1910static int 1911vioif_ctrl_load_cmdspec(struct vioif_softc *sc, 1912 struct vioif_ctrl_cmdspec *specs, int nspecs) 1913{ 1914 struct virtio_softc *vsc = sc->sc_virtio; 1915 int i, r, loaded; 1916 1917 loaded = 0; 1918 for (i = 0; i < nspecs; i++) { 1919 r = bus_dmamap_load(virtio_dmat(vsc), 1920 specs[i].dmamap, specs[i].buf, specs[i].bufsize, 1921 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1922 if (r) { 1923 aprint_error_dev(sc->sc_dev, "control command dmamap" 1924 " load failed, error code %d\n", r); 1925 goto err; 1926 } 1927 loaded++; 1928 1929 } 1930 1931 return r; 1932 1933err: 1934 for (i = 0; i < loaded; i++) { 1935 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1936 } 1937 1938 return r; 1939} 1940 1941static void 1942vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, 1943 struct vioif_ctrl_cmdspec *specs, int nspecs) 1944{ 1945 struct virtio_softc *vsc = sc->sc_virtio; 1946 int i; 1947 1948 for (i = 0; i < nspecs; i++) { 1949 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1950 } 1951} 1952 1953static int 1954vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, 1955 struct vioif_ctrl_cmdspec *specs, int nspecs) 1956{ 1957 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1958 struct virtqueue *vq = ctrlq->ctrlq_vq; 1959 struct virtio_softc *vsc = sc->sc_virtio; 1960 int i, r, slot; 1961 1962 ctrlq->ctrlq_cmd->class = class; 1963 ctrlq->ctrlq_cmd->command = cmd; 1964 1965 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 1966 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); 1967 for (i = 0; i < nspecs; i++) { 1968 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 1969 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); 1970 } 1971 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 1972 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); 1973 1974 r = virtio_enqueue_prep(vsc, vq, &slot); 1975 if (r != 0) 1976 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1977 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); 1978 if (r != 0) 1979 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1980 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); 1981 for (i = 0; i < nspecs; i++) { 1982 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); 1983 } 1984 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); 1985 virtio_enqueue_commit(vsc, vq, slot, true); 1986 1987 /* wait for done */ 1988 mutex_enter(&ctrlq->ctrlq_wait_lock); 1989 while (ctrlq->ctrlq_inuse != DONE) 1990 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 1991 mutex_exit(&ctrlq->ctrlq_wait_lock); 1992 /* already dequeueued */ 1993 1994 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, 1995 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 1996 for (i = 0; i < nspecs; i++) { 1997 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, 1998 specs[i].bufsize, BUS_DMASYNC_POSTWRITE); 1999 } 2000 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, 2001 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); 2002 2003 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) 2004 r = 0; 2005 else { 2006 aprint_error_dev(sc->sc_dev, "failed setting rx mode\n"); 2007 r = EIO; 2008 } 2009 2010 return r; 2011} 2012 2013static int 2014vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 2015{ 2016 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; 2017 struct vioif_ctrl_cmdspec specs[1]; 2018 int r; 2019 2020 if (!sc->sc_has_ctrl) 2021 return ENOTSUP; 2022 2023 vioif_ctrl_acquire(sc); 2024 2025 rx->onoff = onoff; 2026 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; 2027 specs[0].buf = rx; 2028 specs[0].bufsize = sizeof(*rx); 2029 2030 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, 2031 specs, __arraycount(specs)); 2032 2033 vioif_ctrl_release(sc); 2034 return r; 2035} 2036 2037static int 2038vioif_set_promisc(struct vioif_softc *sc, bool onoff) 2039{ 2040 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 2041} 2042 2043static int 2044vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 2045{ 2046 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 2047} 2048 2049/* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 2050static int 2051vioif_set_rx_filter(struct vioif_softc *sc) 2052{ 2053 /* filter already set in ctrlq->ctrlq_mac_tbl */ 2054 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; 2055 struct vioif_ctrl_cmdspec specs[2]; 2056 int nspecs = __arraycount(specs); 2057 int r; 2058 2059 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; 2060 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; 2061 2062 if (!sc->sc_has_ctrl) 2063 return ENOTSUP; 2064 2065 vioif_ctrl_acquire(sc); 2066 2067 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; 2068 specs[0].buf = mac_tbl_uc; 2069 specs[0].bufsize = sizeof(*mac_tbl_uc) 2070 + (ETHER_ADDR_LEN * mac_tbl_uc->nentries); 2071 2072 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; 2073 specs[1].buf = mac_tbl_mc; 2074 specs[1].bufsize = sizeof(*mac_tbl_mc) 2075 + (ETHER_ADDR_LEN * mac_tbl_mc->nentries); 2076 2077 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); 2078 if (r != 0) 2079 goto out; 2080 2081 r = vioif_ctrl_send_command(sc, 2082 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, 2083 specs, nspecs); 2084 2085 vioif_ctrl_unload_cmdspec(sc, specs, nspecs); 2086 2087out: 2088 vioif_ctrl_release(sc); 2089 2090 return r; 2091} 2092 2093static int 2094vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) 2095{ 2096 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; 2097 struct vioif_ctrl_cmdspec specs[1]; 2098 int r; 2099 2100 if (!sc->sc_has_ctrl) 2101 return ENOTSUP; 2102 2103 if (nvq_pairs <= 1) 2104 return EINVAL; 2105 2106 vioif_ctrl_acquire(sc); 2107 2108 mq->virtqueue_pairs = nvq_pairs; 2109 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; 2110 specs[0].buf = mq; 2111 specs[0].bufsize = sizeof(*mq); 2112 2113 r = vioif_ctrl_send_command(sc, 2114 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 2115 specs, __arraycount(specs)); 2116 2117 vioif_ctrl_release(sc); 2118 2119 return r; 2120} 2121 2122/* ctrl vq interrupt; wake up the command issuer */ 2123static int 2124vioif_ctrl_intr(void *arg) 2125{ 2126 struct vioif_ctrlqueue *ctrlq = arg; 2127 struct virtqueue *vq = ctrlq->ctrlq_vq; 2128 struct virtio_softc *vsc = vq->vq_owner; 2129 int r, slot; 2130 2131 if (virtio_vq_is_enqueued(vsc, vq) == false) 2132 return 0; 2133 2134 r = virtio_dequeue(vsc, vq, &slot, NULL); 2135 if (r == ENOENT) 2136 return 0; 2137 virtio_dequeue_commit(vsc, vq, slot); 2138 2139 mutex_enter(&ctrlq->ctrlq_wait_lock); 2140 ctrlq->ctrlq_inuse = DONE; 2141 cv_signal(&ctrlq->ctrlq_wait); 2142 mutex_exit(&ctrlq->ctrlq_wait_lock); 2143 2144 return 1; 2145} 2146 2147/* 2148 * If IFF_PROMISC requested, set promiscuous 2149 * If multicast filter small enough (<=MAXENTRIES) set rx filter 2150 * If large multicast filter exist use ALLMULTI 2151 */ 2152/* 2153 * If setting rx filter fails fall back to ALLMULTI 2154 * If ALLMULTI fails fall back to PROMISC 2155 */ 2156static int 2157vioif_rx_filter(struct vioif_softc *sc) 2158{ 2159 struct ethercom *ec = &sc->sc_ethercom; 2160 struct ifnet *ifp = &ec->ec_if; 2161 struct ether_multi *enm; 2162 struct ether_multistep step; 2163 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2164 int nentries; 2165 int promisc = 0, allmulti = 0, rxfilter = 0; 2166 int r; 2167 2168 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 2169 ifp->if_flags |= IFF_PROMISC; 2170 return 0; 2171 } 2172 2173 if (ifp->if_flags & IFF_PROMISC) { 2174 promisc = 1; 2175 goto set; 2176 } 2177 2178 nentries = -1; 2179 ETHER_LOCK(ec); 2180 ETHER_FIRST_MULTI(step, ec, enm); 2181 while (nentries++, enm != NULL) { 2182 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 2183 allmulti = 1; 2184 goto set_unlock; 2185 } 2186 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 2187 allmulti = 1; 2188 goto set_unlock; 2189 } 2190 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], 2191 enm->enm_addrlo, ETHER_ADDR_LEN); 2192 ETHER_NEXT_MULTI(step, enm); 2193 } 2194 rxfilter = 1; 2195 2196set_unlock: 2197 ETHER_UNLOCK(ec); 2198 2199set: 2200 if (rxfilter) { 2201 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2202 ctrlq->ctrlq_mac_tbl_mc->nentries = nentries; 2203 r = vioif_set_rx_filter(sc); 2204 if (r != 0) { 2205 rxfilter = 0; 2206 allmulti = 1; /* fallback */ 2207 } 2208 } else { 2209 /* remove rx filter */ 2210 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2211 ctrlq->ctrlq_mac_tbl_mc->nentries = 0; 2212 r = vioif_set_rx_filter(sc); 2213 /* what to do on failure? */ 2214 } 2215 if (allmulti) { 2216 r = vioif_set_allmulti(sc, true); 2217 if (r != 0) { 2218 allmulti = 0; 2219 promisc = 1; /* fallback */ 2220 } 2221 } else { 2222 r = vioif_set_allmulti(sc, false); 2223 /* what to do on failure? */ 2224 } 2225 if (promisc) { 2226 r = vioif_set_promisc(sc, true); 2227 } else { 2228 r = vioif_set_promisc(sc, false); 2229 } 2230 2231 return r; 2232} 2233 2234static bool 2235vioif_is_link_up(struct vioif_softc *sc) 2236{ 2237 struct virtio_softc *vsc = sc->sc_virtio; 2238 uint16_t status; 2239 2240 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 2241 status = virtio_read_device_config_2(vsc, 2242 VIRTIO_NET_CONFIG_STATUS); 2243 else 2244 status = VIRTIO_NET_S_LINK_UP; 2245 2246 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 2247} 2248 2249/* change link status */ 2250static void 2251vioif_update_link_status(struct vioif_softc *sc) 2252{ 2253 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2254 struct vioif_txqueue *txq; 2255 bool active, changed; 2256 int link, i; 2257 2258 active = vioif_is_link_up(sc); 2259 changed = false; 2260 2261 if (active) { 2262 if (!sc->sc_link_active) 2263 changed = true; 2264 2265 link = LINK_STATE_UP; 2266 sc->sc_link_active = true; 2267 } else { 2268 if (sc->sc_link_active) 2269 changed = true; 2270 2271 link = LINK_STATE_DOWN; 2272 sc->sc_link_active = false; 2273 } 2274 2275 if (changed) { 2276 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 2277 txq = &sc->sc_txq[i]; 2278 2279 mutex_enter(txq->txq_lock); 2280 txq->txq_link_active = sc->sc_link_active; 2281 mutex_exit(txq->txq_lock); 2282 } 2283 2284 if_link_state_change(ifp, link); 2285 } 2286} 2287 2288static int 2289vioif_config_change(struct virtio_softc *vsc) 2290{ 2291 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2292 2293 softint_schedule(sc->sc_ctl_softint); 2294 return 0; 2295} 2296 2297static void 2298vioif_ctl_softint(void *arg) 2299{ 2300 struct vioif_softc *sc = arg; 2301 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2302 2303 vioif_update_link_status(sc); 2304 vioif_start(ifp); 2305} 2306 2307static struct workqueue * 2308vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) 2309{ 2310 struct workqueue *wq; 2311 int error; 2312 2313 error = workqueue_create(&wq, name, vioif_workq_work, NULL, 2314 prio, ipl, flags); 2315 2316 if (error) 2317 return NULL; 2318 2319 return wq; 2320} 2321 2322static void 2323vioif_workq_destroy(struct workqueue *wq) 2324{ 2325 2326 workqueue_destroy(wq); 2327} 2328 2329static void 2330vioif_workq_work(struct work *wk, void *context) 2331{ 2332 struct vioif_work *work; 2333 2334 work = container_of(wk, struct vioif_work, cookie); 2335 2336 atomic_store_relaxed(&work->added, 0); 2337 work->func(work->arg); 2338} 2339 2340static void 2341vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) 2342{ 2343 2344 memset(work, 0, sizeof(*work)); 2345 work->func = func; 2346 work->arg = arg; 2347} 2348 2349static void 2350vioif_work_add(struct workqueue *wq, struct vioif_work *work) 2351{ 2352 2353 if (atomic_load_relaxed(&work->added) != 0) 2354 return; 2355 2356 atomic_store_relaxed(&work->added, 1); 2357 kpreempt_disable(); 2358 workqueue_enqueue(wq, &work->cookie, NULL); 2359 kpreempt_enable(); 2360} 2361 2362static void 2363vioif_work_wait(struct workqueue *wq, struct vioif_work *work) 2364{ 2365 2366 workqueue_wait(wq, &work->cookie); 2367} 2368 2369static int 2370vioif_setup_sysctl(struct vioif_softc *sc) 2371{ 2372 const char *devname; 2373 struct sysctllog **log; 2374 const struct sysctlnode *rnode, *rxnode, *txnode; 2375 int error; 2376 2377 log = &sc->sc_sysctllog; 2378 devname = device_xname(sc->sc_dev); 2379 2380 error = sysctl_createv(log, 0, NULL, &rnode, 2381 0, CTLTYPE_NODE, devname, 2382 SYSCTL_DESCR("virtio-net information and settings"), 2383 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 2384 if (error) 2385 goto out; 2386 2387 error = sysctl_createv(log, 0, &rnode, NULL, 2388 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2389 SYSCTL_DESCR("Use workqueue for packet processing"), 2390 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); 2391 if (error) 2392 goto out; 2393 2394 error = sysctl_createv(log, 0, &rnode, &rxnode, 2395 0, CTLTYPE_NODE, "rx", 2396 SYSCTL_DESCR("virtio-net information and settings for Rx"), 2397 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2398 if (error) 2399 goto out; 2400 2401 error = sysctl_createv(log, 0, &rxnode, NULL, 2402 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2403 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 2404 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2405 if (error) 2406 goto out; 2407 2408 error = sysctl_createv(log, 0, &rxnode, NULL, 2409 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2410 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 2411 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 2412 if (error) 2413 goto out; 2414 2415 error = sysctl_createv(log, 0, &rnode, &txnode, 2416 0, CTLTYPE_NODE, "tx", 2417 SYSCTL_DESCR("virtio-net information and settings for Tx"), 2418 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2419 if (error) 2420 goto out; 2421 2422 error = sysctl_createv(log, 0, &txnode, NULL, 2423 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2424 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 2425 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2426 if (error) 2427 goto out; 2428 2429 error = sysctl_createv(log, 0, &txnode, NULL, 2430 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2431 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 2432 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 2433 2434out: 2435 if (error) 2436 sysctl_teardown(log); 2437 2438 return error; 2439} 2440 2441MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 2442 2443#ifdef _MODULE 2444#include "ioconf.c" 2445#endif 2446 2447static int 2448if_vioif_modcmd(modcmd_t cmd, void *opaque) 2449{ 2450 int error = 0; 2451 2452#ifdef _MODULE 2453 switch (cmd) { 2454 case MODULE_CMD_INIT: 2455 error = config_init_component(cfdriver_ioconf_if_vioif, 2456 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2457 break; 2458 case MODULE_CMD_FINI: 2459 error = config_fini_component(cfdriver_ioconf_if_vioif, 2460 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2461 break; 2462 default: 2463 error = ENOTTY; 2464 break; 2465 } 2466#endif 2467 2468 return error; 2469} 2470