/* $NetBSD: if_vioif.c,v 1.76 2022/03/29 01:57:51 yamaguchi Exp $ */ /* * Copyright (c) 2020 The NetBSD Foundation, Inc. * Copyright (c) 2010 Minoura Makoto. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.76 2022/03/29 01:57:51 yamaguchi Exp $"); #ifdef _KERNEL_OPT #include "opt_net_mpsafe.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ioconf.h" #ifdef NET_MPSAFE #define VIOIF_MPSAFE 1 #define VIOIF_MULTIQ 1 #endif /* * if_vioifreg.h: */ /* Configuration registers */ #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */ /* Feature bits */ #define VIRTIO_NET_F_CSUM __BIT(0) #define VIRTIO_NET_F_GUEST_CSUM __BIT(1) #define VIRTIO_NET_F_MAC __BIT(5) #define VIRTIO_NET_F_GSO __BIT(6) #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) #define VIRTIO_NET_F_GUEST_ECN __BIT(9) #define VIRTIO_NET_F_GUEST_UFO __BIT(10) #define VIRTIO_NET_F_HOST_TSO4 __BIT(11) #define VIRTIO_NET_F_HOST_TSO6 __BIT(12) #define VIRTIO_NET_F_HOST_ECN __BIT(13) #define VIRTIO_NET_F_HOST_UFO __BIT(14) #define VIRTIO_NET_F_MRG_RXBUF __BIT(15) #define VIRTIO_NET_F_STATUS __BIT(16) #define VIRTIO_NET_F_CTRL_VQ __BIT(17) #define VIRTIO_NET_F_CTRL_RX __BIT(18) #define VIRTIO_NET_F_CTRL_VLAN __BIT(19) #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) #define VIRTIO_NET_F_MQ __BIT(22) #define VIRTIO_NET_FLAG_BITS \ VIRTIO_COMMON_FLAG_BITS \ "\x17""MQ" \ "\x16""GUEST_ANNOUNCE" \ "\x15""CTRL_RX_EXTRA" \ "\x14""CTRL_VLAN" \ "\x13""CTRL_RX" \ "\x12""CTRL_VQ" \ "\x11""STATUS" \ "\x10""MRG_RXBUF" \ "\x0f""HOST_UFO" \ "\x0e""HOST_ECN" \ "\x0d""HOST_TSO6" \ "\x0c""HOST_TSO4" \ "\x0b""GUEST_UFO" \ "\x0a""GUEST_ECN" \ "\x09""GUEST_TSO6" \ "\x08""GUEST_TSO4" \ "\x07""GSO" \ "\x06""MAC" \ "\x02""GUEST_CSUM" \ "\x01""CSUM" /* Status */ #define VIRTIO_NET_S_LINK_UP 1 /* Packet header structure */ struct virtio_net_hdr { uint8_t flags; uint8_t gso_type; uint16_t hdr_len; uint16_t gso_size; uint16_t csum_start; uint16_t csum_offset; uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */ } __packed; #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) /* Control virtqueue */ struct virtio_net_ctrl_cmd { uint8_t class; uint8_t command; } __packed; #define VIRTIO_NET_CTRL_RX 0 # define VIRTIO_NET_CTRL_RX_PROMISC 0 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 #define VIRTIO_NET_CTRL_MAC 1 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 #define VIRTIO_NET_CTRL_VLAN 2 # define VIRTIO_NET_CTRL_VLAN_ADD 0 # define VIRTIO_NET_CTRL_VLAN_DEL 1 #define VIRTIO_NET_CTRL_MQ 4 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 struct virtio_net_ctrl_status { uint8_t ack; } __packed; #define VIRTIO_NET_OK 0 #define VIRTIO_NET_ERR 1 struct virtio_net_ctrl_rx { uint8_t onoff; } __packed; struct virtio_net_ctrl_mac_tbl { uint32_t nentries; uint8_t macs[][ETHER_ADDR_LEN]; } __packed; struct virtio_net_ctrl_mac_addr { uint8_t mac[ETHER_ADDR_LEN]; } __packed; struct virtio_net_ctrl_vlan { uint16_t id; } __packed; struct virtio_net_ctrl_mq { uint16_t virtqueue_pairs; } __packed; /* * if_vioifvar.h: */ /* * Locking notes: * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex). * - more than one lock cannot be held at onece * + ctrlq_inuse is protected by ctrlq_wait_lock. * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock * + fields in vioif_softc except queues are protected by * sc->sc_lock(an adaptive mutex) * - the lock is held before acquisition of other locks */ struct vioif_ctrl_cmdspec { bus_dmamap_t dmamap; void *buf; bus_size_t bufsize; }; struct vioif_work { struct work cookie; void (*func)(void *); void *arg; unsigned int added; }; struct vioif_txqueue { kmutex_t *txq_lock; /* lock for tx operations */ struct virtqueue *txq_vq; bool txq_stopping; bool txq_link_active; pcq_t *txq_intrq; struct virtio_net_hdr *txq_hdrs; bus_dmamap_t *txq_hdr_dmamaps; struct mbuf **txq_mbufs; bus_dmamap_t *txq_dmamaps; void *txq_deferred_transmit; void *txq_handle_si; struct vioif_work txq_work; bool txq_workqueue; bool txq_active; char txq_evgroup[16]; struct evcnt txq_defrag_failed; struct evcnt txq_mbuf_load_failed; struct evcnt txq_enqueue_reserve_failed; }; struct vioif_rxqueue { kmutex_t *rxq_lock; /* lock for rx operations */ struct virtqueue *rxq_vq; bool rxq_stopping; struct virtio_net_hdr *rxq_hdrs; bus_dmamap_t *rxq_hdr_dmamaps; struct mbuf **rxq_mbufs; bus_dmamap_t *rxq_dmamaps; void *rxq_handle_si; struct vioif_work rxq_work; bool rxq_workqueue; bool rxq_active; char rxq_evgroup[16]; struct evcnt rxq_mbuf_add_failed; }; struct vioif_ctrlqueue { struct virtqueue *ctrlq_vq; enum { FREE, INUSE, DONE } ctrlq_inuse; kcondvar_t ctrlq_wait; kmutex_t ctrlq_wait_lock; struct lwp *ctrlq_owner; struct virtio_net_ctrl_cmd *ctrlq_cmd; struct virtio_net_ctrl_status *ctrlq_status; struct virtio_net_ctrl_rx *ctrlq_rx; struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr; struct virtio_net_ctrl_mq *ctrlq_mq; bus_dmamap_t ctrlq_cmd_dmamap; bus_dmamap_t ctrlq_status_dmamap; bus_dmamap_t ctrlq_rx_dmamap; bus_dmamap_t ctrlq_tbl_uc_dmamap; bus_dmamap_t ctrlq_tbl_mc_dmamap; bus_dmamap_t ctrlq_mac_addr_dmamap; bus_dmamap_t ctrlq_mq_dmamap; struct evcnt ctrlq_cmd_load_failed; struct evcnt ctrlq_cmd_failed; }; struct vioif_softc { device_t sc_dev; kmutex_t sc_lock; struct sysctllog *sc_sysctllog; struct virtio_softc *sc_virtio; struct virtqueue *sc_vqs; u_int sc_hdr_size; int sc_max_nvq_pairs; int sc_req_nvq_pairs; int sc_act_nvq_pairs; uint8_t sc_mac[ETHER_ADDR_LEN]; struct ethercom sc_ethercom; bool sc_link_active; struct vioif_txqueue *sc_txq; struct vioif_rxqueue *sc_rxq; bool sc_has_ctrl; struct vioif_ctrlqueue sc_ctrlq; bus_dma_segment_t sc_hdr_segs[1]; void *sc_dmamem; void *sc_kmem; void *sc_ctl_softint; struct workqueue *sc_txrx_workqueue; bool sc_txrx_workqueue_sysctl; u_int sc_tx_intr_process_limit; u_int sc_tx_process_limit; u_int sc_rx_intr_process_limit; u_int sc_rx_process_limit; }; #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ #define VIOIF_TX_INTR_PROCESS_LIMIT 256 #define VIOIF_TX_PROCESS_LIMIT 256 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U #define VIOIF_RX_PROCESS_LIMIT 256 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET /* cfattach interface functions */ static int vioif_match(device_t, cfdata_t, void *); static void vioif_attach(device_t, device_t, void *); static int vioif_finalize_teardown(device_t); /* ifnet interface functions */ static int vioif_init(struct ifnet *); static void vioif_stop(struct ifnet *, int); static void vioif_start(struct ifnet *); static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); static int vioif_transmit(struct ifnet *, struct mbuf *); static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); static int vioif_ioctl(struct ifnet *, u_long, void *); static void vioif_watchdog(struct ifnet *); static int vioif_ifflags_cb(struct ethercom *); /* rx */ static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); static void vioif_populate_rx_mbufs_locked(struct vioif_softc *, struct vioif_rxqueue *); static void vioif_rx_queue_clear(struct vioif_rxqueue *); static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, struct vioif_rxqueue *, u_int); static int vioif_rx_intr(void *); static void vioif_rx_handle(void *); static void vioif_rx_sched_handle(struct vioif_softc *, struct vioif_rxqueue *); static void vioif_rx_drain(struct vioif_rxqueue *); /* tx */ static int vioif_tx_intr(void *); static void vioif_tx_handle(void *); static void vioif_tx_sched_handle(struct vioif_softc *, struct vioif_txqueue *); static void vioif_tx_queue_clear(struct vioif_txqueue *); static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, struct vioif_txqueue *, u_int); static void vioif_tx_drain(struct vioif_txqueue *); static void vioif_deferred_transmit(void *); /* workqueue */ static struct workqueue* vioif_workq_create(const char *, pri_t, int, int); static void vioif_workq_destroy(struct workqueue *); static void vioif_workq_work(struct work *, void *); static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); static void vioif_work_add(struct workqueue *, struct vioif_work *); static void vioif_work_wait(struct workqueue *, struct vioif_work *); /* other control */ static bool vioif_is_link_up(struct vioif_softc *); static void vioif_update_link_status(struct vioif_softc *); static int vioif_ctrl_rx(struct vioif_softc *, int, bool); static int vioif_set_promisc(struct vioif_softc *, bool); static int vioif_set_allmulti(struct vioif_softc *, bool); static int vioif_set_rx_filter(struct vioif_softc *); static int vioif_rx_filter(struct vioif_softc *); static int vioif_set_mac_addr(struct vioif_softc *); static int vioif_ctrl_intr(void *); static int vioif_config_change(struct virtio_softc *); static void vioif_ctl_softint(void *); static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); static int vioif_setup_sysctl(struct vioif_softc *); static void vioif_setup_stats(struct vioif_softc *); static int vioif_ifflags(struct vioif_softc *); CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), vioif_match, vioif_attach, NULL, NULL); static int vioif_match(device_t parent, cfdata_t match, void *aux) { struct virtio_attach_args *va = aux; if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK) return 1; return 0; } static int vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, bus_size_t size, int nsegs, const char *usage) { int r; r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); if (r != 0) { aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " "error code %d\n", usage, r); } return r; } static void vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) { if (*map) { bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); *map = NULL; } } static int vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, void *buf, bus_size_t size, int nsegs, int rw, const char *usage) { int r; r = vioif_dmamap_create(sc, map, size, nsegs, usage); if (r != 0) return 1; r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, size, NULL, rw | BUS_DMA_NOWAIT); if (r != 0) { vioif_dmamap_destroy(sc, map); aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " "error code %d\n", usage, r); } return r; } static void * vioif_assign_mem(intptr_t *p, size_t size) { intptr_t rv; rv = *p; *p += size; return (void *)rv; } static void vioif_alloc_queues(struct vioif_softc *sc) { int nvq_pairs = sc->sc_max_nvq_pairs; int nvqs = nvq_pairs * 2; int i; KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, KM_SLEEP); sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, KM_SLEEP); if (sc->sc_has_ctrl) nvqs++; sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); nvqs = 0; for (i = 0; i < nvq_pairs; i++) { sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; } if (sc->sc_has_ctrl) sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; } static void vioif_free_queues(struct vioif_softc *sc) { int nvq_pairs = sc->sc_max_nvq_pairs; int nvqs = nvq_pairs * 2; if (sc->sc_ctrlq.ctrlq_vq) nvqs++; if (sc->sc_txq) { kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); sc->sc_txq = NULL; } if (sc->sc_rxq) { kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); sc->sc_rxq = NULL; } if (sc->sc_vqs) { kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); sc->sc_vqs = NULL; } } /* allocate memory */ /* * dma memory is used for: * rxq_hdrs[slot]: metadata array for received frames (READ) * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) * ctrlq_cmd: command to be sent via ctrl vq (WRITE) * ctrlq_status: return value for a command via ctrl vq (READ) * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command * (WRITE) * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC * class command (WRITE) * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC * class command (WRITE) * ctrlq_* structures are allocated only one each; they are protected by * ctrlq_inuse variable and ctrlq_wait condvar. */ /* * dynamically allocated memory is used for: * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] * rxq_dmamaps[slot]: bus_dmamap_t array for received payload * txq_dmamaps[slot]: bus_dmamap_t array for sent payload * rxq_mbufs[slot]: mbuf pointer array for received frames * txq_mbufs[slot]: mbuf pointer array for sent frames */ static int vioif_alloc_mems(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; int allocsize, allocsize2, r, rsegs, i, qid; void *vaddr; intptr_t p; allocsize = 0; for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; allocsize += sizeof(struct virtio_net_hdr) * (rxq->rxq_vq->vq_num + txq->txq_vq->vq_num); } if (sc->sc_has_ctrl) { allocsize += sizeof(struct virtio_net_ctrl_cmd); allocsize += sizeof(struct virtio_net_ctrl_status); allocsize += sizeof(struct virtio_net_ctrl_rx); allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) + ETHER_ADDR_LEN; allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; allocsize += sizeof(struct virtio_net_ctrl_mac_addr); allocsize += sizeof(struct virtio_net_ctrl_mq); } r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); if (r != 0) { aprint_error_dev(sc->sc_dev, "DMA memory allocation failed, size %d, " "error code %d\n", allocsize, r); goto err_none; } r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT); if (r != 0) { aprint_error_dev(sc->sc_dev, "DMA memory map failed, error code %d\n", r); goto err_dmamem_alloc; } memset(vaddr, 0, allocsize); sc->sc_dmamem = vaddr; p = (intptr_t) vaddr; for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; rxq->rxq_hdrs = vioif_assign_mem(&p, sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num); txq->txq_hdrs = vioif_assign_mem(&p, sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num); } if (sc->sc_has_ctrl) { ctrlq->ctrlq_cmd = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_cmd)); ctrlq->ctrlq_status = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_status)); ctrlq->ctrlq_rx = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_rx)); ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mac_tbl_uc) + ETHER_ADDR_LEN); ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mac_tbl_mc) + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mac_addr)); ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); } allocsize2 = 0; for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { int rxqsize, txqsize; rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; rxqsize = rxq->rxq_vq->vq_num; txqsize = txq->txq_vq->vq_num; allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; } vaddr = kmem_zalloc(allocsize2, KM_SLEEP); sc->sc_kmem = vaddr; p = (intptr_t) vaddr; for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { int rxqsize, txqsize; rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; rxqsize = rxq->rxq_vq->vq_num; txqsize = txq->txq_vq->vq_num; rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p, sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); txq->txq_hdr_dmamaps = vioif_assign_mem(&p, sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); rxq->rxq_dmamaps = vioif_assign_mem(&p, sizeof(rxq->rxq_dmamaps[0]) * rxqsize); txq->txq_dmamaps = vioif_assign_mem(&p, sizeof(txq->txq_dmamaps[0]) * txqsize); rxq->rxq_mbufs = vioif_assign_mem(&p, sizeof(rxq->rxq_mbufs[0]) * rxqsize); txq->txq_mbufs = vioif_assign_mem(&p, sizeof(txq->txq_mbufs[0]) * txqsize); } for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; for (i = 0; i < rxq->rxq_vq->vq_num; i++) { r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i], sc->sc_hdr_size, 1, BUS_DMA_READ, "rx header"); if (r != 0) goto err_reqs; r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload"); if (r != 0) goto err_reqs; } for (i = 0; i < txq->txq_vq->vq_num; i++) { r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i], sc->sc_hdr_size, 1, BUS_DMA_READ, "tx header"); if (r != 0) goto err_reqs; r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN, VIRTIO_NET_TX_MAXNSEGS, "tx payload"); if (r != 0) goto err_reqs; } } if (sc->sc_has_ctrl) { /* control vq class & command */ r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, BUS_DMA_WRITE, "control command"); if (r != 0) goto err_reqs; r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, BUS_DMA_READ, "control status"); if (r != 0) goto err_reqs; /* control vq rx mode command parameter */ r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, BUS_DMA_WRITE, "rx mode control command"); if (r != 0) goto err_reqs; /* multiqueue set command */ r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, BUS_DMA_WRITE, "multiqueue set command"); if (r != 0) goto err_reqs; /* control vq MAC filter table for unicast */ /* do not load now since its length is variable */ r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, sizeof(*ctrlq->ctrlq_mac_tbl_uc) + ETHER_ADDR_LEN, 1, "unicast MAC address filter command"); if (r != 0) goto err_reqs; /* control vq MAC filter table for multicast */ r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, sizeof(*ctrlq->ctrlq_mac_tbl_mc) + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, "multicast MAC address filter command"); if (r != 0) goto err_reqs; /* control vq MAC address set command */ r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mac_addr_dmamap, ctrlq->ctrlq_mac_addr, sizeof(*ctrlq->ctrlq_mac_addr), 1, BUS_DMA_WRITE, "mac addr set command"); if (r != 0) goto err_reqs; } return 0; err_reqs: vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { rxq = &sc->sc_rxq[qid]; txq = &sc->sc_txq[qid]; for (i = 0; i < txq->txq_vq->vq_num; i++) { vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]); vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]); } for (i = 0; i < rxq->rxq_vq->vq_num; i++) { vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]); vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]); } } if (sc->sc_kmem) { kmem_free(sc->sc_kmem, allocsize2); sc->sc_kmem = NULL; } bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); err_dmamem_alloc: bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); err_none: return -1; } static void vioif_attach(device_t parent, device_t self, void *aux) { struct vioif_softc *sc = device_private(self); struct virtio_softc *vsc = device_private(parent); struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; uint64_t features, req_features; struct ifnet *ifp = &sc->sc_ethercom.ec_if; u_int softint_flags; int r, i, nvqs=0, req_flags; char xnamebuf[MAXCOMLEN]; if (virtio_child(vsc) != NULL) { aprint_normal(": child already attached for %s; " "something wrong...\n", device_xname(parent)); return; } sc->sc_dev = self; sc->sc_virtio = vsc; sc->sc_link_active = false; sc->sc_max_nvq_pairs = 1; sc->sc_req_nvq_pairs = 1; sc->sc_act_nvq_pairs = 1; sc->sc_txrx_workqueue_sysctl = true; sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); if (sc->sc_txrx_workqueue == NULL) goto err; req_flags = 0; #ifdef VIOIF_MPSAFE req_flags |= VIRTIO_F_INTR_MPSAFE; #endif req_flags |= VIRTIO_F_INTR_MSIX; req_features = VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; req_features |= VIRTIO_F_RING_EVENT_IDX; #ifdef VIOIF_MULTIQ req_features |= VIRTIO_NET_F_MQ; #endif virtio_child_attach_start(vsc, self, IPL_NET, NULL, vioif_config_change, virtio_vq_intrhand, req_flags, req_features, VIRTIO_NET_FLAG_BITS); features = virtio_features(vsc); if (features == 0) goto err; if (features & VIRTIO_NET_F_MAC) { for (i = 0; i < __arraycount(sc->sc_mac); i++) { sc->sc_mac[i] = virtio_read_device_config_1(vsc, VIRTIO_NET_CONFIG_MAC + i); } } else { /* code stolen from sys/net/if_tap.c */ struct timeval tv; uint32_t ui; getmicrouptime(&tv); ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); for (i = 0; i < __arraycount(sc->sc_mac); i++) { virtio_write_device_config_1(vsc, VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); } } /* 'Ethernet' with capital follows other ethernet driver attachment */ aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { sc->sc_hdr_size = sizeof(struct virtio_net_hdr); } else { sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); } if ((features & VIRTIO_NET_F_CTRL_VQ) && (features & VIRTIO_NET_F_CTRL_RX)) { sc->sc_has_ctrl = true; cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); ctrlq->ctrlq_inuse = FREE; } else { sc->sc_has_ctrl = false; } if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) goto err; /* Limit the number of queue pairs to use */ sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); } vioif_alloc_queues(sc); virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); #ifdef VIOIF_MPSAFE softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; #else softint_flags = SOFTINT_NET; #endif /* * Allocating virtqueues */ for (i = 0; i < sc->sc_max_nvq_pairs; i++) { rxq = &sc->sc_rxq[i]; txq = &sc->sc_txq[i]; char qname[32]; rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); rxq->rxq_handle_si = softint_establish(softint_flags, vioif_rx_handle, rxq); if (rxq->rxq_handle_si == NULL) { aprint_error_dev(self, "cannot establish rx softint\n"); goto err; } snprintf(qname, sizeof(qname), "rx%d", i); r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, MCLBYTES + sc->sc_hdr_size, 2, qname); if (r != 0) goto err; nvqs++; rxq->rxq_vq->vq_intrhand = vioif_rx_intr; rxq->rxq_vq->vq_intrhand_arg = (void *)rxq; rxq->rxq_stopping = true; vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq); txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); txq->txq_deferred_transmit = softint_establish(softint_flags, vioif_deferred_transmit, txq); if (txq->txq_deferred_transmit == NULL) { aprint_error_dev(self, "cannot establish tx softint\n"); goto err; } txq->txq_handle_si = softint_establish(softint_flags, vioif_tx_handle, txq); if (txq->txq_handle_si == NULL) { aprint_error_dev(self, "cannot establish tx softint\n"); goto err; } snprintf(qname, sizeof(qname), "tx%d", i); r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, sc->sc_hdr_size + (ETHER_MAX_LEN - ETHER_HDR_LEN), VIRTIO_NET_TX_MAXNSEGS + 1, qname); if (r != 0) goto err; nvqs++; txq->txq_vq->vq_intrhand = vioif_tx_intr; txq->txq_vq->vq_intrhand_arg = (void *)txq; txq->txq_link_active = sc->sc_link_active; txq->txq_stopping = false; txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP); vioif_work_set(&txq->txq_work, vioif_tx_handle, txq); } if (sc->sc_has_ctrl) { /* * Allocating a virtqueue for control channel */ r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, NBPG, 1, "control"); if (r != 0) { aprint_error_dev(self, "failed to allocate " "a virtqueue for control channel, error code %d\n", r); sc->sc_has_ctrl = false; cv_destroy(&ctrlq->ctrlq_wait); mutex_destroy(&ctrlq->ctrlq_wait_lock); } else { nvqs++; ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; } } sc->sc_ctl_softint = softint_establish(softint_flags, vioif_ctl_softint, sc); if (sc->sc_ctl_softint == NULL) { aprint_error_dev(self, "cannot establish ctl softint\n"); goto err; } if (vioif_alloc_mems(sc) < 0) goto err; if (virtio_child_attach_finish(vsc) != 0) goto err; if (vioif_setup_sysctl(sc) != 0) { aprint_error_dev(self, "unable to create sysctl node\n"); /* continue */ } vioif_setup_stats(sc); strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; #ifdef VIOIF_MPSAFE ifp->if_extflags = IFEF_MPSAFE; #endif ifp->if_start = vioif_start; if (sc->sc_req_nvq_pairs > 1) ifp->if_transmit = vioif_transmit; ifp->if_ioctl = vioif_ioctl; ifp->if_init = vioif_init; ifp->if_stop = vioif_stop; ifp->if_capabilities = 0; ifp->if_watchdog = vioif_watchdog; txq = &sc->sc_txq[0]; IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); IFQ_SET_READY(&ifp->if_snd); sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; if_attach(ifp); if_deferred_start_init(ifp, NULL); ether_ifattach(ifp, sc->sc_mac); ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); return; err: for (i = 0; i < sc->sc_max_nvq_pairs; i++) { rxq = &sc->sc_rxq[i]; txq = &sc->sc_txq[i]; if (rxq->rxq_lock) { mutex_obj_free(rxq->rxq_lock); rxq->rxq_lock = NULL; } if (rxq->rxq_handle_si) { softint_disestablish(rxq->rxq_handle_si); rxq->rxq_handle_si = NULL; } if (txq->txq_lock) { mutex_obj_free(txq->txq_lock); txq->txq_lock = NULL; } if (txq->txq_handle_si) { softint_disestablish(txq->txq_handle_si); txq->txq_handle_si = NULL; } if (txq->txq_deferred_transmit) { softint_disestablish(txq->txq_deferred_transmit); txq->txq_deferred_transmit = NULL; } if (txq->txq_intrq) { pcq_destroy(txq->txq_intrq); txq->txq_intrq = NULL; } } if (sc->sc_has_ctrl) { cv_destroy(&ctrlq->ctrlq_wait); mutex_destroy(&ctrlq->ctrlq_wait_lock); } while (nvqs > 0) virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); vioif_free_queues(sc); mutex_destroy(&sc->sc_lock); virtio_child_attach_failed(vsc); config_finalize_register(self, vioif_finalize_teardown); return; } static int vioif_finalize_teardown(device_t self) { struct vioif_softc *sc = device_private(self); if (sc->sc_txrx_workqueue != NULL) { vioif_workq_destroy(sc->sc_txrx_workqueue); sc->sc_txrx_workqueue = NULL; } return 0; } static void vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; int i; for (i = 0; i < sc->sc_act_nvq_pairs; i++) { txq = &sc->sc_txq[i]; rxq = &sc->sc_rxq[i]; virtio_start_vq_intr(vsc, txq->txq_vq); virtio_start_vq_intr(vsc, rxq->rxq_vq); } } static void vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; int i; for (i = 0; i < sc->sc_act_nvq_pairs; i++) { rxq = &sc->sc_rxq[i]; txq = &sc->sc_txq[i]; virtio_stop_vq_intr(vsc, rxq->rxq_vq); virtio_stop_vq_intr(vsc, txq->txq_vq); } } /* * Interface functions for ifnet */ static int vioif_init(struct ifnet *ifp) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; struct vioif_rxqueue *rxq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; int r, i; vioif_stop(ifp, 0); r = virtio_reinit_start(vsc); if (r != 0) { log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); return EIO; } virtio_negotiate_features(vsc, virtio_features(vsc)); for (i = 0; i < sc->sc_req_nvq_pairs; i++) { rxq = &sc->sc_rxq[i]; /* Have to set false before vioif_populate_rx_mbufs */ mutex_enter(rxq->rxq_lock); rxq->rxq_stopping = false; vioif_populate_rx_mbufs_locked(sc, rxq); mutex_exit(rxq->rxq_lock); } virtio_reinit_end(vsc); if (sc->sc_has_ctrl) virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); if (r == 0) sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; else sc->sc_act_nvq_pairs = 1; for (i = 0; i < sc->sc_act_nvq_pairs; i++) sc->sc_txq[i].txq_stopping = false; vioif_enable_interrupt_vqpairs(sc); vioif_update_link_status(sc); ifp->if_flags |= IFF_RUNNING; ifp->if_flags &= ~IFF_OACTIVE; r = vioif_rx_filter(sc); return r; } static void vioif_stop_rendezvous(struct vioif_softc *sc) { struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; int i; /* * stop all packet processing: * 1. acquire a lock for queue to wait * for finish of interrupt handler * 2. stop workqueue for packet processing */ for (i =0; i < sc->sc_act_nvq_pairs; i++) { txq = &sc->sc_txq[i]; rxq = &sc->sc_rxq[i]; mutex_enter(rxq->rxq_lock); mutex_exit(rxq->rxq_lock); vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work); mutex_enter(txq->txq_lock); mutex_exit(txq->txq_lock); vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work); } } static void vioif_stop(struct ifnet *ifp, int disable) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; struct vioif_txqueue *txq; struct vioif_rxqueue *rxq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; int i; /* Take the locks to ensure that ongoing TX/RX finish */ for (i = 0; i < sc->sc_act_nvq_pairs; i++) { txq = &sc->sc_txq[i]; rxq = &sc->sc_rxq[i]; mutex_enter(rxq->rxq_lock); rxq->rxq_stopping = true; mutex_exit(rxq->rxq_lock); mutex_enter(txq->txq_lock); txq->txq_stopping = true; mutex_exit(txq->txq_lock); } /* disable interrupts */ vioif_disable_interrupt_vqpairs(sc); if (sc->sc_has_ctrl) virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); /* only way to stop I/O and DMA is resetting... */ virtio_reset(vsc); vioif_stop_rendezvous(sc); for (i = 0; i < sc->sc_act_nvq_pairs; i++) { vioif_rx_queue_clear(&sc->sc_rxq[i]); vioif_tx_queue_clear(&sc->sc_txq[i]); } ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); sc->sc_link_active = false; for (i = 0; i < sc->sc_act_nvq_pairs; i++) { txq = &sc->sc_txq[i]; rxq = &sc->sc_rxq[i]; txq->txq_link_active = false; if (disable) vioif_rx_drain(rxq); vioif_tx_drain(txq); } } static void vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, bool is_transmit) { struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; struct virtqueue *vq = txq->txq_vq; struct virtio_net_hdr *hdr; struct mbuf *m; int queued = 0; KASSERT(mutex_owned(txq->txq_lock)); if ((ifp->if_flags & IFF_RUNNING) == 0) return; if (!txq->txq_link_active || txq->txq_stopping) return; if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) return; for (;;) { int slot, r; if (is_transmit) m = pcq_get(txq->txq_intrq); else IFQ_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; r = virtio_enqueue_prep(vsc, vq, &slot); if (r == EAGAIN) { ifp->if_flags |= IFF_OACTIVE; m_freem(m); break; } if (r != 0) panic("enqueue_prep for a tx buffer"); r = bus_dmamap_load_mbuf(virtio_dmat(vsc), txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); if (r != 0) { /* maybe just too fragmented */ struct mbuf *newm; newm = m_defrag(m, M_NOWAIT); if (newm == NULL) { txq->txq_defrag_failed.ev_count++; goto skip; } m = newm; r = bus_dmamap_load_mbuf(virtio_dmat(vsc), txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); if (r != 0) { txq->txq_mbuf_load_failed.ev_count++; skip: m_freem(m); virtio_enqueue_abort(vsc, vq, slot); continue; } } /* This should actually never fail */ r = virtio_enqueue_reserve(vsc, vq, slot, txq->txq_dmamaps[slot]->dm_nsegs + 1); if (r != 0) { txq->txq_enqueue_reserve_failed.ev_count++; bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); /* slot already freed by virtio_enqueue_reserve */ m_freem(m); continue; } txq->txq_mbufs[slot] = m; hdr = &txq->txq_hdrs[slot]; memset(hdr, 0, sc->sc_hdr_size); bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 0, txq->txq_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_PREWRITE); virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); virtio_enqueue_commit(vsc, vq, slot, false); queued++; bpf_mtap(ifp, m, BPF_D_OUT); } if (queued > 0) { virtio_enqueue_commit(vsc, vq, -1, true); ifp->if_timer = 5; } } static void vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) { /* * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. */ vioif_send_common_locked(ifp, txq, false); } static void vioif_start(struct ifnet *ifp) { struct vioif_softc *sc = ifp->if_softc; struct vioif_txqueue *txq = &sc->sc_txq[0]; #ifdef VIOIF_MPSAFE KASSERT(if_is_mpsafe(ifp)); #endif mutex_enter(txq->txq_lock); vioif_start_locked(ifp, txq); mutex_exit(txq->txq_lock); } static inline int vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) { struct vioif_softc *sc = ifp->if_softc; u_int cpuid = cpu_index(curcpu()); return cpuid % sc->sc_act_nvq_pairs; } static void vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) { vioif_send_common_locked(ifp, txq, true); } static int vioif_transmit(struct ifnet *ifp, struct mbuf *m) { struct vioif_softc *sc = ifp->if_softc; struct vioif_txqueue *txq; int qid; qid = vioif_select_txqueue(ifp, m); txq = &sc->sc_txq[qid]; if (__predict_false(!pcq_put(txq->txq_intrq, m))) { m_freem(m); return ENOBUFS; } net_stat_ref_t nsr = IF_STAT_GETREF(ifp); if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); if (m->m_flags & M_MCAST) if_statinc_ref(nsr, if_omcasts); IF_STAT_PUTREF(ifp); if (mutex_tryenter(txq->txq_lock)) { vioif_transmit_locked(ifp, txq); mutex_exit(txq->txq_lock); } return 0; } static void vioif_deferred_transmit(void *arg) { struct vioif_txqueue *txq = arg; struct virtio_softc *vsc = txq->txq_vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); struct ifnet *ifp = &sc->sc_ethercom.ec_if; mutex_enter(txq->txq_lock); vioif_send_common_locked(ifp, txq, true); mutex_exit(txq->txq_lock); } static int vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) { int s, r; s = splnet(); r = ether_ioctl(ifp, cmd, data); if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { if (ifp->if_flags & IFF_RUNNING) { r = vioif_rx_filter(ifp->if_softc); } else { r = 0; } } splx(s); return r; } void vioif_watchdog(struct ifnet *ifp) { struct vioif_softc *sc = ifp->if_softc; int i; if (ifp->if_flags & IFF_RUNNING) { for (i = 0; i < sc->sc_act_nvq_pairs; i++) { vioif_tx_queue_clear(&sc->sc_txq[i]); } } } /* * Receive implementation */ /* allocate and initialize a mbuf for receive */ static int vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) { struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; struct mbuf *m; int r; MGETHDR(m, M_DONTWAIT, MT_DATA); if (m == NULL) return ENOBUFS; MCLGET(m, M_DONTWAIT); if ((m->m_flags & M_EXT) == 0) { m_freem(m); return ENOBUFS; } rxq->rxq_mbufs[i] = m; m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; r = bus_dmamap_load_mbuf(virtio_dmat(vsc), rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT); if (r) { m_freem(m); rxq->rxq_mbufs[i] = NULL; return r; } return 0; } /* free a mbuf for receive */ static void vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) { struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); m_freem(rxq->rxq_mbufs[i]); rxq->rxq_mbufs[i] = NULL; } /* add mbufs for all the empty receive slots */ static void vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_rxqueue *rxq) { struct virtqueue *vq = rxq->rxq_vq; struct virtio_softc *vsc = vq->vq_owner; int i, r, ndone = 0; KASSERT(mutex_owned(rxq->rxq_lock)); if (rxq->rxq_stopping) return; for (i = 0; i < vq->vq_num; i++) { int slot; r = virtio_enqueue_prep(vsc, vq, &slot); if (r == EAGAIN) break; if (r != 0) panic("enqueue_prep for rx buffers"); if (rxq->rxq_mbufs[slot] == NULL) { r = vioif_add_rx_mbuf(rxq, slot); if (r != 0) { rxq->rxq_mbuf_add_failed.ev_count++; break; } } r = virtio_enqueue_reserve(vsc, vq, slot, rxq->rxq_dmamaps[slot]->dm_nsegs + 1); if (r != 0) { vioif_free_rx_mbuf(rxq, slot); break; } bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 0, sc->sc_hdr_size, BUS_DMASYNC_PREREAD); bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 0, MCLBYTES, BUS_DMASYNC_PREREAD); virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], false); virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); virtio_enqueue_commit(vsc, vq, slot, false); ndone++; } if (ndone > 0) virtio_enqueue_commit(vsc, vq, -1, true); } static void vioif_rx_queue_clear(struct vioif_rxqueue *rxq) { struct virtqueue *vq = rxq->rxq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit = UINT_MAX; bool more; KASSERT(rxq->rxq_stopping); mutex_enter(rxq->rxq_lock); for (;;) { more = vioif_rx_deq_locked(sc, vsc, rxq, limit); if (more == false) break; } mutex_exit(rxq->rxq_lock); } /* dequeue received packets */ static bool vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, struct vioif_rxqueue *rxq, u_int limit) { struct virtqueue *vq = rxq->rxq_vq; struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct mbuf *m; int slot, len; bool more = false, dequeued = false; KASSERT(mutex_owned(rxq->rxq_lock)); if (virtio_vq_is_enqueued(vsc, vq) == false) return false; for (;;) { if (limit-- == 0) { more = true; break; } if (virtio_dequeue(vsc, vq, &slot, &len) != 0) break; dequeued = true; len -= sc->sc_hdr_size; bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 0, sc->sc_hdr_size, BUS_DMASYNC_POSTREAD); bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 0, MCLBYTES, BUS_DMASYNC_POSTREAD); m = rxq->rxq_mbufs[slot]; KASSERT(m != NULL); bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); rxq->rxq_mbufs[slot] = NULL; virtio_dequeue_commit(vsc, vq, slot); m_set_rcvif(m, ifp); m->m_len = m->m_pkthdr.len = len; mutex_exit(rxq->rxq_lock); if_percpuq_enqueue(ifp->if_percpuq, m); mutex_enter(rxq->rxq_lock); if (rxq->rxq_stopping) break; } if (dequeued) vioif_populate_rx_mbufs_locked(sc, rxq); return more; } /* rx interrupt; call _dequeue above and schedule a softint */ static void vioif_rx_handle_locked(void *xrxq, u_int limit) { struct vioif_rxqueue *rxq = xrxq; struct virtqueue *vq = rxq->rxq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); bool more; KASSERT(!rxq->rxq_stopping); more = vioif_rx_deq_locked(sc, vsc, rxq, limit); if (more) { vioif_rx_sched_handle(sc, rxq); return; } more = virtio_start_vq_intr(vsc, rxq->rxq_vq); if (more) { vioif_rx_sched_handle(sc, rxq); return; } atomic_store_relaxed(&rxq->rxq_active, false); } static int vioif_rx_intr(void *arg) { struct vioif_rxqueue *rxq = arg; struct virtqueue *vq = rxq->rxq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; limit = sc->sc_rx_intr_process_limit; if (atomic_load_relaxed(&rxq->rxq_active) == true) return 1; mutex_enter(rxq->rxq_lock); if (!rxq->rxq_stopping) { rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl; virtio_stop_vq_intr(vsc, vq); atomic_store_relaxed(&rxq->rxq_active, true); vioif_rx_handle_locked(rxq, limit); } mutex_exit(rxq->rxq_lock); return 1; } static void vioif_rx_handle(void *xrxq) { struct vioif_rxqueue *rxq = xrxq; struct virtqueue *vq = rxq->rxq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; limit = sc->sc_rx_process_limit; mutex_enter(rxq->rxq_lock); if (!rxq->rxq_stopping) vioif_rx_handle_locked(rxq, limit); mutex_exit(rxq->rxq_lock); } static void vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq) { KASSERT(mutex_owned(rxq->rxq_lock)); if (rxq->rxq_stopping) return; if (rxq->rxq_workqueue) vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work); else softint_schedule(rxq->rxq_handle_si); } /* free all the mbufs; called from if_stop(disable) */ static void vioif_rx_drain(struct vioif_rxqueue *rxq) { struct virtqueue *vq = rxq->rxq_vq; int i; for (i = 0; i < vq->vq_num; i++) { if (rxq->rxq_mbufs[i] == NULL) continue; vioif_free_rx_mbuf(rxq, i); } } /* * Transmition implementation */ /* actual transmission is done in if_start */ /* tx interrupt; dequeue and free mbufs */ /* * tx interrupt is actually disabled; this should be called upon * tx vq full and watchdog */ static void vioif_tx_handle_locked(struct vioif_txqueue *txq, u_int limit) { struct virtqueue *vq = txq->txq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); struct ifnet *ifp = &sc->sc_ethercom.ec_if; bool more; KASSERT(!txq->txq_stopping); more = vioif_tx_deq_locked(sc, vsc, txq, limit); if (more) { vioif_tx_sched_handle(sc, txq); return; } if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) more = virtio_postpone_intr_smart(vsc, vq); else more = virtio_start_vq_intr(vsc, vq); if (more) { vioif_tx_sched_handle(sc, txq); return; } atomic_store_relaxed(&txq->txq_active, false); /* for ALTQ */ if (txq == &sc->sc_txq[0]) { if_schedule_deferred_start(ifp); ifp->if_flags &= ~IFF_OACTIVE; } softint_schedule(txq->txq_deferred_transmit); } static int vioif_tx_intr(void *arg) { struct vioif_txqueue *txq = arg; struct virtqueue *vq = txq->txq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; limit = sc->sc_tx_intr_process_limit; if (atomic_load_relaxed(&txq->txq_active) == true) return 1; mutex_enter(txq->txq_lock); if (!txq->txq_stopping) { txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl; virtio_stop_vq_intr(vsc, vq); atomic_store_relaxed(&txq->txq_active, true); vioif_tx_handle_locked(txq, limit); } mutex_exit(txq->txq_lock); return 1; } static void vioif_tx_handle(void *xtxq) { struct vioif_txqueue *txq = xtxq; struct virtqueue *vq = txq->txq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit; limit = sc->sc_tx_process_limit; mutex_enter(txq->txq_lock); if (!txq->txq_stopping) vioif_tx_handle_locked(txq, limit); mutex_exit(txq->txq_lock); } static void vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq) { KASSERT(mutex_owned(txq->txq_lock)); if (txq->txq_stopping) return; if (txq->txq_workqueue) vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work); else softint_schedule(txq->txq_handle_si); } static void vioif_tx_queue_clear(struct vioif_txqueue *txq) { struct virtqueue *vq = txq->txq_vq; struct virtio_softc *vsc = vq->vq_owner; struct vioif_softc *sc = device_private(virtio_child(vsc)); u_int limit = UINT_MAX; bool more; mutex_enter(txq->txq_lock); for (;;) { more = vioif_tx_deq_locked(sc, vsc, txq, limit); if (more == false) break; } mutex_exit(txq->txq_lock); } static bool vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, struct vioif_txqueue *txq, u_int limit) { struct virtqueue *vq = txq->txq_vq; struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct mbuf *m; int slot, len; bool more = false; KASSERT(mutex_owned(txq->txq_lock)); if (virtio_vq_is_enqueued(vsc, vq) == false) return false; for (;;) { if (limit-- == 0) { more = true; break; } if (virtio_dequeue(vsc, vq, &slot, &len) != 0) break; bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 0, sc->sc_hdr_size, BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 0, txq->txq_dmamaps[slot]->dm_mapsize, BUS_DMASYNC_POSTWRITE); m = txq->txq_mbufs[slot]; bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); txq->txq_mbufs[slot] = NULL; virtio_dequeue_commit(vsc, vq, slot); if_statinc(ifp, if_opackets); m_freem(m); } return more; } /* free all the mbufs already put on vq; called from if_stop(disable) */ static void vioif_tx_drain(struct vioif_txqueue *txq) { struct virtqueue *vq = txq->txq_vq; struct virtio_softc *vsc = vq->vq_owner; int i; KASSERT(txq->txq_stopping); for (i = 0; i < vq->vq_num; i++) { if (txq->txq_mbufs[i] == NULL) continue; bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); m_freem(txq->txq_mbufs[i]); txq->txq_mbufs[i] = NULL; } } /* * Control vq */ /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ static void vioif_ctrl_acquire(struct vioif_softc *sc) { struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; mutex_enter(&ctrlq->ctrlq_wait_lock); while (ctrlq->ctrlq_inuse != FREE) cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); ctrlq->ctrlq_inuse = INUSE; ctrlq->ctrlq_owner = curlwp; mutex_exit(&ctrlq->ctrlq_wait_lock); } static void vioif_ctrl_release(struct vioif_softc *sc) { struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; KASSERT(ctrlq->ctrlq_inuse != FREE); KASSERT(ctrlq->ctrlq_owner == curlwp); mutex_enter(&ctrlq->ctrlq_wait_lock); ctrlq->ctrlq_inuse = FREE; ctrlq->ctrlq_owner = NULL; cv_signal(&ctrlq->ctrlq_wait); mutex_exit(&ctrlq->ctrlq_wait_lock); } static int vioif_ctrl_load_cmdspec(struct vioif_softc *sc, struct vioif_ctrl_cmdspec *specs, int nspecs) { struct virtio_softc *vsc = sc->sc_virtio; int i, r, loaded; loaded = 0; for (i = 0; i < nspecs; i++) { r = bus_dmamap_load(virtio_dmat(vsc), specs[i].dmamap, specs[i].buf, specs[i].bufsize, NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); if (r) { sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++; goto err; } loaded++; } return r; err: for (i = 0; i < loaded; i++) { bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); } return r; } static void vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, struct vioif_ctrl_cmdspec *specs, int nspecs) { struct virtio_softc *vsc = sc->sc_virtio; int i; for (i = 0; i < nspecs; i++) { bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); } } static int vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, struct vioif_ctrl_cmdspec *specs, int nspecs) { struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; struct virtqueue *vq = ctrlq->ctrlq_vq; struct virtio_softc *vsc = sc->sc_virtio; int i, r, slot; ctrlq->ctrlq_cmd->class = class; ctrlq->ctrlq_cmd->command = cmd; bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); for (i = 0; i < nspecs; i++) { bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); } bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); /* we need to explicitly (re)start vq intr when using RING EVENT IDX */ if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); r = virtio_enqueue_prep(vsc, vq, &slot); if (r != 0) panic("%s: control vq busy!?", device_xname(sc->sc_dev)); r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); if (r != 0) panic("%s: control vq busy!?", device_xname(sc->sc_dev)); virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); for (i = 0; i < nspecs; i++) { virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); } virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); virtio_enqueue_commit(vsc, vq, slot, true); /* wait for done */ mutex_enter(&ctrlq->ctrlq_wait_lock); while (ctrlq->ctrlq_inuse != DONE) cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); mutex_exit(&ctrlq->ctrlq_wait_lock); /* already dequeueued */ bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); for (i = 0; i < nspecs; i++) { bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, specs[i].bufsize, BUS_DMASYNC_POSTWRITE); } bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) r = 0; else { device_printf(sc->sc_dev, "failed setting rx mode\n"); sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++; r = EIO; } return r; } static int vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) { struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; struct vioif_ctrl_cmdspec specs[1]; int r; if (!sc->sc_has_ctrl) return ENOTSUP; vioif_ctrl_acquire(sc); rx->onoff = onoff; specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; specs[0].buf = rx; specs[0].bufsize = sizeof(*rx); r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, specs, __arraycount(specs)); vioif_ctrl_release(sc); return r; } static int vioif_set_promisc(struct vioif_softc *sc, bool onoff) { return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); } static int vioif_set_allmulti(struct vioif_softc *sc, bool onoff) { return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); } /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ static int vioif_set_rx_filter(struct vioif_softc *sc) { /* filter already set in ctrlq->ctrlq_mac_tbl */ struct virtio_softc *vsc = sc->sc_virtio; struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; struct vioif_ctrl_cmdspec specs[2]; int nspecs = __arraycount(specs); int r; mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; if (!sc->sc_has_ctrl) return ENOTSUP; vioif_ctrl_acquire(sc); specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; specs[0].buf = mac_tbl_uc; specs[0].bufsize = sizeof(*mac_tbl_uc) + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; specs[1].buf = mac_tbl_mc; specs[1].bufsize = sizeof(*mac_tbl_mc) + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); if (r != 0) goto out; r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, specs, nspecs); vioif_ctrl_unload_cmdspec(sc, specs, nspecs); out: vioif_ctrl_release(sc); return r; } static int vioif_set_mac_addr(struct vioif_softc *sc) { struct virtio_net_ctrl_mac_addr *ma = sc->sc_ctrlq.ctrlq_mac_addr; struct vioif_ctrl_cmdspec specs[1]; struct ifnet *ifp = &sc->sc_ethercom.ec_if; int nspecs = __arraycount(specs); int r; if (!sc->sc_has_ctrl) return ENOTSUP; vioif_ctrl_acquire(sc); memcpy(ma->mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap; specs[0].buf = ma; specs[0].bufsize = sizeof(*ma); r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET, specs, nspecs); vioif_ctrl_release(sc); return r; } static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) { struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; struct vioif_ctrl_cmdspec specs[1]; int r; if (!sc->sc_has_ctrl) return ENOTSUP; if (nvq_pairs <= 1) return EINVAL; vioif_ctrl_acquire(sc); mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; specs[0].buf = mq; specs[0].bufsize = sizeof(*mq); r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, specs, __arraycount(specs)); vioif_ctrl_release(sc); return r; } /* ctrl vq interrupt; wake up the command issuer */ static int vioif_ctrl_intr(void *arg) { struct vioif_ctrlqueue *ctrlq = arg; struct virtqueue *vq = ctrlq->ctrlq_vq; struct virtio_softc *vsc = vq->vq_owner; int r, slot; if (virtio_vq_is_enqueued(vsc, vq) == false) return 0; r = virtio_dequeue(vsc, vq, &slot, NULL); if (r == ENOENT) return 0; virtio_dequeue_commit(vsc, vq, slot); mutex_enter(&ctrlq->ctrlq_wait_lock); ctrlq->ctrlq_inuse = DONE; cv_signal(&ctrlq->ctrlq_wait); mutex_exit(&ctrlq->ctrlq_wait_lock); return 1; } static int vioif_ifflags(struct vioif_softc *sc) { struct ifnet *ifp = &sc->sc_ethercom.ec_if; bool onoff; int r; if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc and allmulti */ ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); return 0; } onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; r = vioif_set_allmulti(sc, onoff); if (r != 0) { log(LOG_WARNING, "%s: couldn't %sable ALLMULTI\n", ifp->if_xname, onoff ? "en" : "dis"); if (onoff == false) { ifp->if_flags |= IFF_ALLMULTI; } } onoff = ifp->if_flags & IFF_PROMISC ? true : false; r = vioif_set_promisc(sc, onoff); if (r != 0) { log(LOG_WARNING, "%s: couldn't %sable PROMISC\n", ifp->if_xname, onoff ? "en" : "dis"); if (onoff == false) { ifp->if_flags |= IFF_PROMISC; } } return 0; } static int vioif_ifflags_cb(struct ethercom *ec) { struct ifnet *ifp = &ec->ec_if; struct vioif_softc *sc = ifp->if_softc; return vioif_ifflags(sc); } /* * If multicast filter small enough (<=MAXENTRIES) set rx filter * If large multicast filter exist use ALLMULTI * If setting rx filter fails fall back to ALLMULTI */ static int vioif_rx_filter(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; struct ethercom *ec = &sc->sc_ethercom; struct ifnet *ifp = &ec->ec_if; struct ether_multi *enm; struct ether_multistep step; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; int nentries; bool allmulti = 0; int r; if (!sc->sc_has_ctrl) { goto set_ifflags; } memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0], CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); nentries = 0; allmulti = false; ETHER_LOCK(ec); for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; ETHER_NEXT_MULTI(step, enm)) { if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { allmulti = true; break; } if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { allmulti = true; break; } memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], enm->enm_addrlo, ETHER_ADDR_LEN); nentries++; } ETHER_UNLOCK(ec); r = vioif_set_mac_addr(sc); if (r != 0) { log(LOG_WARNING, "%s: couldn't set MAC address\n", ifp->if_xname); } if (!allmulti) { ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1); ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries); r = vioif_set_rx_filter(sc); if (r != 0) { allmulti = true; /* fallback */ } } if (allmulti) { ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0); ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0); r = vioif_set_rx_filter(sc); if (r != 0) { log(LOG_DEBUG, "%s: couldn't clear RX filter\n", ifp->if_xname); /* what to do on failure? */ } ifp->if_flags |= IFF_ALLMULTI; } set_ifflags: r = vioif_ifflags(sc); return r; } static bool vioif_is_link_up(struct vioif_softc *sc) { struct virtio_softc *vsc = sc->sc_virtio; uint16_t status; if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) status = virtio_read_device_config_2(vsc, VIRTIO_NET_CONFIG_STATUS); else status = VIRTIO_NET_S_LINK_UP; return ((status & VIRTIO_NET_S_LINK_UP) != 0); } /* change link status */ static void vioif_update_link_status(struct vioif_softc *sc) { struct ifnet *ifp = &sc->sc_ethercom.ec_if; struct vioif_txqueue *txq; bool active, changed; int link, i; mutex_enter(&sc->sc_lock); active = vioif_is_link_up(sc); changed = false; if (active) { if (!sc->sc_link_active) changed = true; link = LINK_STATE_UP; sc->sc_link_active = true; } else { if (sc->sc_link_active) changed = true; link = LINK_STATE_DOWN; sc->sc_link_active = false; } if (changed) { for (i = 0; i < sc->sc_act_nvq_pairs; i++) { txq = &sc->sc_txq[i]; mutex_enter(txq->txq_lock); txq->txq_link_active = sc->sc_link_active; mutex_exit(txq->txq_lock); } if_link_state_change(ifp, link); } mutex_exit(&sc->sc_lock); } static int vioif_config_change(struct virtio_softc *vsc) { struct vioif_softc *sc = device_private(virtio_child(vsc)); softint_schedule(sc->sc_ctl_softint); return 0; } static void vioif_ctl_softint(void *arg) { struct vioif_softc *sc = arg; struct ifnet *ifp = &sc->sc_ethercom.ec_if; vioif_update_link_status(sc); vioif_start(ifp); } static struct workqueue * vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) { struct workqueue *wq; int error; error = workqueue_create(&wq, name, vioif_workq_work, NULL, prio, ipl, flags); if (error) return NULL; return wq; } static void vioif_workq_destroy(struct workqueue *wq) { workqueue_destroy(wq); } static void vioif_workq_work(struct work *wk, void *context) { struct vioif_work *work; work = container_of(wk, struct vioif_work, cookie); atomic_store_relaxed(&work->added, 0); work->func(work->arg); } static void vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) { memset(work, 0, sizeof(*work)); work->func = func; work->arg = arg; } static void vioif_work_add(struct workqueue *wq, struct vioif_work *work) { if (atomic_load_relaxed(&work->added) != 0) return; atomic_store_relaxed(&work->added, 1); kpreempt_disable(); workqueue_enqueue(wq, &work->cookie, NULL); kpreempt_enable(); } static void vioif_work_wait(struct workqueue *wq, struct vioif_work *work) { workqueue_wait(wq, &work->cookie); } static int vioif_setup_sysctl(struct vioif_softc *sc) { const char *devname; struct sysctllog **log; const struct sysctlnode *rnode, *rxnode, *txnode; int error; log = &sc->sc_sysctllog; devname = device_xname(sc->sc_dev); error = sysctl_createv(log, 0, NULL, &rnode, 0, CTLTYPE_NODE, devname, SYSCTL_DESCR("virtio-net information and settings"), NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, NULL, CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, &rxnode, 0, CTLTYPE_NODE, "rx", SYSCTL_DESCR("virtio-net information and settings for Rx"), NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rxnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rxnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, &txnode, 0, CTLTYPE_NODE, "tx", SYSCTL_DESCR("virtio-net information and settings for Tx"), NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &txnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &txnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); out: if (error) sysctl_teardown(log); return error; } static void vioif_setup_stats(struct vioif_softc *sc) { struct vioif_rxqueue *rxq; struct vioif_txqueue *txq; int i; for (i = 0; i < sc->sc_max_nvq_pairs; i++) { rxq = &sc->sc_rxq[i]; txq = &sc->sc_txq[i]; snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d", device_xname(sc->sc_dev), i); evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC, NULL, txq->txq_evgroup, "tx m_defrag() failed"); evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC, NULL, txq->txq_evgroup, "tx dmamap load failed"); evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC, NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed"); snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d", device_xname(sc->sc_dev), i); evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC, NULL, rxq->rxq_evgroup, "rx mbuf allocation failed"); } evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, NULL, device_xname(sc->sc_dev), "control command failed"); } MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); #ifdef _MODULE #include "ioconf.c" #endif static int if_vioif_modcmd(modcmd_t cmd, void *opaque) { int error = 0; #ifdef _MODULE switch (cmd) { case MODULE_CMD_INIT: error = config_init_component(cfdriver_ioconf_if_vioif, cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); break; case MODULE_CMD_FINI: error = config_fini_component(cfdriver_ioconf_if_vioif, cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); break; default: error = ENOTTY; break; } #endif return error; }