1227652Sgrehan/*- 2253132Sbryanv * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3227652Sgrehan * All rights reserved. 4227652Sgrehan * 5227652Sgrehan * Redistribution and use in source and binary forms, with or without 6227652Sgrehan * modification, are permitted provided that the following conditions 7227652Sgrehan * are met: 8227652Sgrehan * 1. Redistributions of source code must retain the above copyright 9227652Sgrehan * notice unmodified, this list of conditions, and the following 10227652Sgrehan * disclaimer. 11227652Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 12227652Sgrehan * notice, this list of conditions and the following disclaimer in the 13227652Sgrehan * documentation and/or other materials provided with the distribution. 14227652Sgrehan * 15227652Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16227652Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17227652Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18227652Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19227652Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20227652Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21227652Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22227652Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23227652Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24227652Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25227652Sgrehan */ 26227652Sgrehan 27227652Sgrehan/* Driver for the VirtIO PCI interface. */ 28227652Sgrehan 29227652Sgrehan#include <sys/cdefs.h> 30227652Sgrehan__FBSDID("$FreeBSD$"); 31227652Sgrehan 32227652Sgrehan#include <sys/param.h> 33227652Sgrehan#include <sys/systm.h> 34227652Sgrehan#include <sys/bus.h> 35227652Sgrehan#include <sys/kernel.h> 36227652Sgrehan#include <sys/module.h> 37227652Sgrehan#include <sys/malloc.h> 38227652Sgrehan 39227652Sgrehan#include <machine/bus.h> 40227652Sgrehan#include <machine/resource.h> 41227652Sgrehan#include <sys/bus.h> 42227652Sgrehan#include <sys/rman.h> 43227652Sgrehan 44227652Sgrehan#include <dev/pci/pcivar.h> 45227652Sgrehan#include <dev/pci/pcireg.h> 46227652Sgrehan 47227652Sgrehan#include <dev/virtio/virtio.h> 48227652Sgrehan#include <dev/virtio/virtqueue.h> 49227652Sgrehan#include <dev/virtio/pci/virtio_pci.h> 50227652Sgrehan 51227652Sgrehan#include "virtio_bus_if.h" 52227652Sgrehan#include "virtio_if.h" 53227652Sgrehan 54253132Sbryanvstruct vtpci_interrupt { 55253132Sbryanv struct resource *vti_irq; 56253132Sbryanv int vti_rid; 57253132Sbryanv void *vti_handler; 58253132Sbryanv}; 59253132Sbryanv 60253132Sbryanvstruct vtpci_virtqueue { 61253132Sbryanv struct virtqueue *vtv_vq; 62253132Sbryanv int vtv_no_intr; 63253132Sbryanv}; 64253132Sbryanv 65227652Sgrehanstruct vtpci_softc { 66227652Sgrehan device_t vtpci_dev; 67227652Sgrehan struct resource *vtpci_res; 68227652Sgrehan struct resource *vtpci_msix_res; 69227652Sgrehan uint64_t vtpci_features; 70227652Sgrehan uint32_t vtpci_flags; 71246582Sbryanv#define VTPCI_FLAG_NO_MSI 0x0001 72246582Sbryanv#define VTPCI_FLAG_NO_MSIX 0x0002 73246582Sbryanv#define VTPCI_FLAG_LEGACY 0x1000 74246582Sbryanv#define VTPCI_FLAG_MSI 0x2000 75246582Sbryanv#define VTPCI_FLAG_MSIX 0x4000 76246582Sbryanv#define VTPCI_FLAG_SHARED_MSIX 0x8000 77246582Sbryanv#define VTPCI_FLAG_ITYPE_MASK 0xF000 78227652Sgrehan 79246582Sbryanv /* This "bus" will only ever have one child. */ 80227652Sgrehan device_t vtpci_child_dev; 81227652Sgrehan struct virtio_feature_desc *vtpci_child_feat_desc; 82227652Sgrehan 83227652Sgrehan int vtpci_nvqs; 84253132Sbryanv struct vtpci_virtqueue *vtpci_vqs; 85227652Sgrehan 86227652Sgrehan /* 87253132Sbryanv * Ideally, each virtqueue that the driver provides a callback for will 88253132Sbryanv * receive its own MSIX vector. If there are not sufficient vectors 89253132Sbryanv * available, then attempt to have all the VQs share one vector. For 90253132Sbryanv * MSIX, the configuration changed notifications must be on their own 91253132Sbryanv * vector. 92227652Sgrehan * 93253132Sbryanv * If MSIX is not available, we will attempt to have the whole device 94253132Sbryanv * share one MSI vector, and then, finally, one legacy interrupt. 95227652Sgrehan */ 96253132Sbryanv struct vtpci_interrupt vtpci_device_interrupt; 97253132Sbryanv struct vtpci_interrupt *vtpci_msix_vq_interrupts; 98253132Sbryanv int vtpci_nmsix_resources; 99227652Sgrehan}; 100227652Sgrehan 101227652Sgrehanstatic int vtpci_probe(device_t); 102227652Sgrehanstatic int vtpci_attach(device_t); 103227652Sgrehanstatic int vtpci_detach(device_t); 104227652Sgrehanstatic int vtpci_suspend(device_t); 105227652Sgrehanstatic int vtpci_resume(device_t); 106227652Sgrehanstatic int vtpci_shutdown(device_t); 107227652Sgrehanstatic void vtpci_driver_added(device_t, driver_t *); 108227652Sgrehanstatic void vtpci_child_detached(device_t, device_t); 109227652Sgrehanstatic int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 110227652Sgrehanstatic int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 111227652Sgrehan 112227652Sgrehanstatic uint64_t vtpci_negotiate_features(device_t, uint64_t); 113227652Sgrehanstatic int vtpci_with_feature(device_t, uint64_t); 114227652Sgrehanstatic int vtpci_alloc_virtqueues(device_t, int, int, 115227652Sgrehan struct vq_alloc_info *); 116227652Sgrehanstatic int vtpci_setup_intr(device_t, enum intr_type); 117227652Sgrehanstatic void vtpci_stop(device_t); 118227652Sgrehanstatic int vtpci_reinit(device_t, uint64_t); 119227652Sgrehanstatic void vtpci_reinit_complete(device_t); 120227652Sgrehanstatic void vtpci_notify_virtqueue(device_t, uint16_t); 121227652Sgrehanstatic uint8_t vtpci_get_status(device_t); 122227652Sgrehanstatic void vtpci_set_status(device_t, uint8_t); 123227652Sgrehanstatic void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 124227652Sgrehanstatic void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 125227652Sgrehan 126227652Sgrehanstatic void vtpci_describe_features(struct vtpci_softc *, const char *, 127227652Sgrehan uint64_t); 128227652Sgrehanstatic void vtpci_probe_and_attach_child(struct vtpci_softc *); 129227652Sgrehan 130253132Sbryanvstatic int vtpci_alloc_msix(struct vtpci_softc *, int); 131253132Sbryanvstatic int vtpci_alloc_msi(struct vtpci_softc *); 132253132Sbryanvstatic int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); 133253132Sbryanvstatic int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); 134253132Sbryanvstatic int vtpci_alloc_intr_msi(struct vtpci_softc *); 135253132Sbryanvstatic int vtpci_alloc_intr_legacy(struct vtpci_softc *); 136253132Sbryanvstatic int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, 137253132Sbryanv struct vtpci_interrupt *); 138246582Sbryanvstatic int vtpci_alloc_intr_resources(struct vtpci_softc *); 139246582Sbryanv 140253132Sbryanvstatic int vtpci_setup_legacy_interrupt(struct vtpci_softc *, 141246582Sbryanv enum intr_type); 142253132Sbryanvstatic int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, 143246582Sbryanv enum intr_type); 144253132Sbryanvstatic int vtpci_setup_msix_interrupts(struct vtpci_softc *, 145253132Sbryanv enum intr_type); 146253132Sbryanvstatic int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); 147246582Sbryanv 148253132Sbryanvstatic int vtpci_register_msix_vector(struct vtpci_softc *, int, 149253132Sbryanv struct vtpci_interrupt *); 150253132Sbryanvstatic int vtpci_set_host_msix_vectors(struct vtpci_softc *); 151253132Sbryanvstatic int vtpci_reinit_virtqueue(struct vtpci_softc *, int); 152227652Sgrehan 153253132Sbryanvstatic void vtpci_free_interrupt(struct vtpci_softc *, 154253132Sbryanv struct vtpci_interrupt *); 155227652Sgrehanstatic void vtpci_free_interrupts(struct vtpci_softc *); 156227652Sgrehanstatic void vtpci_free_virtqueues(struct vtpci_softc *); 157227652Sgrehanstatic void vtpci_release_child_resources(struct vtpci_softc *); 158253132Sbryanvstatic void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); 159227652Sgrehanstatic void vtpci_reset(struct vtpci_softc *); 160227652Sgrehan 161246582Sbryanvstatic void vtpci_select_virtqueue(struct vtpci_softc *, int); 162246582Sbryanv 163253132Sbryanvstatic void vtpci_legacy_intr(void *); 164253132Sbryanvstatic int vtpci_vq_shared_intr_filter(void *); 165253132Sbryanvstatic void vtpci_vq_shared_intr(void *); 166253132Sbryanvstatic int vtpci_vq_intr_filter(void *); 167253132Sbryanvstatic void vtpci_vq_intr(void *); 168253132Sbryanvstatic void vtpci_config_intr(void *); 169227652Sgrehan 170246582Sbryanv#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt 171246582Sbryanv 172227652Sgrehan/* 173227652Sgrehan * I/O port read/write wrappers. 174227652Sgrehan */ 175227652Sgrehan#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 176227652Sgrehan#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 177227652Sgrehan#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 178227652Sgrehan#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 179227652Sgrehan#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 180227652Sgrehan#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 181227652Sgrehan 182227652Sgrehan/* Tunables. */ 183227652Sgrehanstatic int vtpci_disable_msix = 0; 184227652SgrehanTUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 185227652Sgrehan 186227652Sgrehanstatic device_method_t vtpci_methods[] = { 187227652Sgrehan /* Device interface. */ 188227652Sgrehan DEVMETHOD(device_probe, vtpci_probe), 189227652Sgrehan DEVMETHOD(device_attach, vtpci_attach), 190227652Sgrehan DEVMETHOD(device_detach, vtpci_detach), 191227652Sgrehan DEVMETHOD(device_suspend, vtpci_suspend), 192227652Sgrehan DEVMETHOD(device_resume, vtpci_resume), 193227652Sgrehan DEVMETHOD(device_shutdown, vtpci_shutdown), 194227652Sgrehan 195227652Sgrehan /* Bus interface. */ 196227652Sgrehan DEVMETHOD(bus_driver_added, vtpci_driver_added), 197227652Sgrehan DEVMETHOD(bus_child_detached, vtpci_child_detached), 198227652Sgrehan DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 199227652Sgrehan DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 200227652Sgrehan 201227652Sgrehan /* VirtIO bus interface. */ 202227652Sgrehan DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 203227652Sgrehan DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 204227652Sgrehan DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 205227652Sgrehan DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 206227652Sgrehan DEVMETHOD(virtio_bus_stop, vtpci_stop), 207227652Sgrehan DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 208227652Sgrehan DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 209227652Sgrehan DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 210227652Sgrehan DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 211227652Sgrehan DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 212227652Sgrehan 213239472Semaste DEVMETHOD_END 214227652Sgrehan}; 215227652Sgrehan 216227652Sgrehanstatic driver_t vtpci_driver = { 217227652Sgrehan "virtio_pci", 218227652Sgrehan vtpci_methods, 219227652Sgrehan sizeof(struct vtpci_softc) 220227652Sgrehan}; 221227652Sgrehan 222227652Sgrehandevclass_t vtpci_devclass; 223227652Sgrehan 224227652SgrehanDRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 225227652SgrehanMODULE_VERSION(virtio_pci, 1); 226227652SgrehanMODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 227227652SgrehanMODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 228227652Sgrehan 229227652Sgrehanstatic int 230227652Sgrehanvtpci_probe(device_t dev) 231227652Sgrehan{ 232227652Sgrehan char desc[36]; 233227652Sgrehan const char *name; 234227652Sgrehan 235227652Sgrehan if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 236227652Sgrehan return (ENXIO); 237227652Sgrehan 238227652Sgrehan if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 239227652Sgrehan pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 240227652Sgrehan return (ENXIO); 241227652Sgrehan 242227652Sgrehan if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 243227652Sgrehan return (ENXIO); 244227652Sgrehan 245227652Sgrehan name = virtio_device_name(pci_get_subdevice(dev)); 246227652Sgrehan if (name == NULL) 247227652Sgrehan name = "Unknown"; 248227652Sgrehan 249227652Sgrehan snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 250227652Sgrehan device_set_desc_copy(dev, desc); 251227652Sgrehan 252227652Sgrehan return (BUS_PROBE_DEFAULT); 253227652Sgrehan} 254227652Sgrehan 255227652Sgrehanstatic int 256227652Sgrehanvtpci_attach(device_t dev) 257227652Sgrehan{ 258227652Sgrehan struct vtpci_softc *sc; 259227652Sgrehan device_t child; 260227652Sgrehan int rid; 261227652Sgrehan 262227652Sgrehan sc = device_get_softc(dev); 263227652Sgrehan sc->vtpci_dev = dev; 264227652Sgrehan 265227652Sgrehan pci_enable_busmaster(dev); 266227652Sgrehan 267227652Sgrehan rid = PCIR_BAR(0); 268227652Sgrehan sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 269227652Sgrehan RF_ACTIVE); 270227652Sgrehan if (sc->vtpci_res == NULL) { 271227652Sgrehan device_printf(dev, "cannot map I/O space\n"); 272227652Sgrehan return (ENXIO); 273227652Sgrehan } 274227652Sgrehan 275227652Sgrehan if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0) 276246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 277227652Sgrehan 278227652Sgrehan if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) { 279227652Sgrehan rid = PCIR_BAR(1); 280227652Sgrehan sc->vtpci_msix_res = bus_alloc_resource_any(dev, 281227652Sgrehan SYS_RES_MEMORY, &rid, RF_ACTIVE); 282227652Sgrehan } 283227652Sgrehan 284227652Sgrehan if (sc->vtpci_msix_res == NULL) 285246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 286227652Sgrehan 287227652Sgrehan vtpci_reset(sc); 288227652Sgrehan 289227652Sgrehan /* Tell the host we've noticed this device. */ 290227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 291227652Sgrehan 292227652Sgrehan if ((child = device_add_child(dev, NULL, -1)) == NULL) { 293227652Sgrehan device_printf(dev, "cannot create child device\n"); 294227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 295227652Sgrehan vtpci_detach(dev); 296227652Sgrehan return (ENOMEM); 297227652Sgrehan } 298227652Sgrehan 299227652Sgrehan sc->vtpci_child_dev = child; 300227652Sgrehan vtpci_probe_and_attach_child(sc); 301227652Sgrehan 302227652Sgrehan return (0); 303227652Sgrehan} 304227652Sgrehan 305227652Sgrehanstatic int 306227652Sgrehanvtpci_detach(device_t dev) 307227652Sgrehan{ 308227652Sgrehan struct vtpci_softc *sc; 309227652Sgrehan device_t child; 310227652Sgrehan int error; 311227652Sgrehan 312227652Sgrehan sc = device_get_softc(dev); 313227652Sgrehan 314227652Sgrehan if ((child = sc->vtpci_child_dev) != NULL) { 315227652Sgrehan error = device_delete_child(dev, child); 316227652Sgrehan if (error) 317227652Sgrehan return (error); 318227652Sgrehan sc->vtpci_child_dev = NULL; 319227652Sgrehan } 320227652Sgrehan 321227652Sgrehan vtpci_reset(sc); 322227652Sgrehan 323227652Sgrehan if (sc->vtpci_msix_res != NULL) { 324227652Sgrehan bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 325227652Sgrehan sc->vtpci_msix_res); 326227652Sgrehan sc->vtpci_msix_res = NULL; 327227652Sgrehan } 328227652Sgrehan 329227652Sgrehan if (sc->vtpci_res != NULL) { 330227652Sgrehan bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 331227652Sgrehan sc->vtpci_res); 332227652Sgrehan sc->vtpci_res = NULL; 333227652Sgrehan } 334227652Sgrehan 335227652Sgrehan return (0); 336227652Sgrehan} 337227652Sgrehan 338227652Sgrehanstatic int 339227652Sgrehanvtpci_suspend(device_t dev) 340227652Sgrehan{ 341227652Sgrehan 342227652Sgrehan return (bus_generic_suspend(dev)); 343227652Sgrehan} 344227652Sgrehan 345227652Sgrehanstatic int 346227652Sgrehanvtpci_resume(device_t dev) 347227652Sgrehan{ 348227652Sgrehan 349227652Sgrehan return (bus_generic_resume(dev)); 350227652Sgrehan} 351227652Sgrehan 352227652Sgrehanstatic int 353227652Sgrehanvtpci_shutdown(device_t dev) 354227652Sgrehan{ 355227652Sgrehan 356227652Sgrehan (void) bus_generic_shutdown(dev); 357227652Sgrehan /* Forcibly stop the host device. */ 358227652Sgrehan vtpci_stop(dev); 359227652Sgrehan 360227652Sgrehan return (0); 361227652Sgrehan} 362227652Sgrehan 363227652Sgrehanstatic void 364227652Sgrehanvtpci_driver_added(device_t dev, driver_t *driver) 365227652Sgrehan{ 366227652Sgrehan struct vtpci_softc *sc; 367227652Sgrehan 368227652Sgrehan sc = device_get_softc(dev); 369227652Sgrehan 370227652Sgrehan vtpci_probe_and_attach_child(sc); 371227652Sgrehan} 372227652Sgrehan 373227652Sgrehanstatic void 374227652Sgrehanvtpci_child_detached(device_t dev, device_t child) 375227652Sgrehan{ 376227652Sgrehan struct vtpci_softc *sc; 377227652Sgrehan 378227652Sgrehan sc = device_get_softc(dev); 379227652Sgrehan 380227652Sgrehan vtpci_reset(sc); 381227652Sgrehan vtpci_release_child_resources(sc); 382227652Sgrehan} 383227652Sgrehan 384227652Sgrehanstatic int 385227652Sgrehanvtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 386227652Sgrehan{ 387227652Sgrehan struct vtpci_softc *sc; 388227652Sgrehan 389227652Sgrehan sc = device_get_softc(dev); 390227652Sgrehan 391227652Sgrehan if (sc->vtpci_child_dev != child) 392227652Sgrehan return (ENOENT); 393227652Sgrehan 394227652Sgrehan switch (index) { 395227652Sgrehan case VIRTIO_IVAR_DEVTYPE: 396246582Sbryanv case VIRTIO_IVAR_SUBDEVICE: 397227652Sgrehan *result = pci_get_subdevice(dev); 398227652Sgrehan break; 399246582Sbryanv case VIRTIO_IVAR_VENDOR: 400246582Sbryanv *result = pci_get_vendor(dev); 401246582Sbryanv break; 402246582Sbryanv case VIRTIO_IVAR_DEVICE: 403246582Sbryanv *result = pci_get_device(dev); 404246582Sbryanv break; 405246582Sbryanv case VIRTIO_IVAR_SUBVENDOR: 406246582Sbryanv *result = pci_get_subdevice(dev); 407246582Sbryanv break; 408227652Sgrehan default: 409227652Sgrehan return (ENOENT); 410227652Sgrehan } 411227652Sgrehan 412227652Sgrehan return (0); 413227652Sgrehan} 414227652Sgrehan 415227652Sgrehanstatic int 416227652Sgrehanvtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 417227652Sgrehan{ 418227652Sgrehan struct vtpci_softc *sc; 419227652Sgrehan 420227652Sgrehan sc = device_get_softc(dev); 421227652Sgrehan 422227652Sgrehan if (sc->vtpci_child_dev != child) 423227652Sgrehan return (ENOENT); 424227652Sgrehan 425227652Sgrehan switch (index) { 426227652Sgrehan case VIRTIO_IVAR_FEATURE_DESC: 427227652Sgrehan sc->vtpci_child_feat_desc = (void *) value; 428227652Sgrehan break; 429227652Sgrehan default: 430227652Sgrehan return (ENOENT); 431227652Sgrehan } 432227652Sgrehan 433227652Sgrehan return (0); 434227652Sgrehan} 435227652Sgrehan 436227652Sgrehanstatic uint64_t 437227652Sgrehanvtpci_negotiate_features(device_t dev, uint64_t child_features) 438227652Sgrehan{ 439227652Sgrehan struct vtpci_softc *sc; 440227652Sgrehan uint64_t host_features, features; 441227652Sgrehan 442227652Sgrehan sc = device_get_softc(dev); 443227652Sgrehan 444227652Sgrehan host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 445227652Sgrehan vtpci_describe_features(sc, "host", host_features); 446227652Sgrehan 447227652Sgrehan /* 448227652Sgrehan * Limit negotiated features to what the driver, virtqueue, and 449227652Sgrehan * host all support. 450227652Sgrehan */ 451227652Sgrehan features = host_features & child_features; 452227652Sgrehan features = virtqueue_filter_features(features); 453227652Sgrehan sc->vtpci_features = features; 454227652Sgrehan 455227652Sgrehan vtpci_describe_features(sc, "negotiated", features); 456227652Sgrehan vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 457227652Sgrehan 458227652Sgrehan return (features); 459227652Sgrehan} 460227652Sgrehan 461227652Sgrehanstatic int 462227652Sgrehanvtpci_with_feature(device_t dev, uint64_t feature) 463227652Sgrehan{ 464227652Sgrehan struct vtpci_softc *sc; 465227652Sgrehan 466227652Sgrehan sc = device_get_softc(dev); 467227652Sgrehan 468227652Sgrehan return ((sc->vtpci_features & feature) != 0); 469227652Sgrehan} 470227652Sgrehan 471227652Sgrehanstatic int 472227652Sgrehanvtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 473227652Sgrehan struct vq_alloc_info *vq_info) 474227652Sgrehan{ 475227652Sgrehan struct vtpci_softc *sc; 476246582Sbryanv struct virtqueue *vq; 477227652Sgrehan struct vtpci_virtqueue *vqx; 478227652Sgrehan struct vq_alloc_info *info; 479246582Sbryanv int idx, error; 480246582Sbryanv uint16_t size; 481227652Sgrehan 482227652Sgrehan sc = device_get_softc(dev); 483227652Sgrehan 484246582Sbryanv if (sc->vtpci_nvqs != 0) 485246582Sbryanv return (EALREADY); 486253132Sbryanv if (nvqs <= 0) 487227652Sgrehan return (EINVAL); 488227652Sgrehan 489253132Sbryanv sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 490253132Sbryanv M_DEVBUF, M_NOWAIT | M_ZERO); 491253132Sbryanv if (sc->vtpci_vqs == NULL) 492253132Sbryanv return (ENOMEM); 493227652Sgrehan 494246582Sbryanv for (idx = 0; idx < nvqs; idx++) { 495253132Sbryanv vqx = &sc->vtpci_vqs[idx]; 496246582Sbryanv info = &vq_info[idx]; 497227652Sgrehan 498246582Sbryanv vtpci_select_virtqueue(sc, idx); 499246582Sbryanv size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 500227652Sgrehan 501246582Sbryanv error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 502246582Sbryanv 0xFFFFFFFFUL, info, &vq); 503246582Sbryanv if (error) { 504246582Sbryanv device_printf(dev, 505246582Sbryanv "cannot allocate virtqueue %d: %d\n", idx, error); 506246582Sbryanv break; 507227652Sgrehan } 508227652Sgrehan 509227652Sgrehan vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 510246582Sbryanv virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 511227652Sgrehan 512253132Sbryanv vqx->vtv_vq = *info->vqai_vq = vq; 513253132Sbryanv vqx->vtv_no_intr = info->vqai_intr == NULL; 514246582Sbryanv 515227652Sgrehan sc->vtpci_nvqs++; 516227652Sgrehan } 517227652Sgrehan 518253132Sbryanv if (error) 519253132Sbryanv vtpci_free_virtqueues(sc); 520253132Sbryanv 521246582Sbryanv return (error); 522227652Sgrehan} 523227652Sgrehan 524227652Sgrehanstatic int 525227652Sgrehanvtpci_setup_intr(device_t dev, enum intr_type type) 526227652Sgrehan{ 527227652Sgrehan struct vtpci_softc *sc; 528246582Sbryanv int attempt, error; 529227652Sgrehan 530227652Sgrehan sc = device_get_softc(dev); 531227652Sgrehan 532246582Sbryanv for (attempt = 0; attempt < 5; attempt++) { 533246582Sbryanv /* 534246582Sbryanv * Start with the most desirable interrupt configuration and 535246582Sbryanv * fallback towards less desirable ones. 536246582Sbryanv */ 537246582Sbryanv switch (attempt) { 538246582Sbryanv case 0: 539246582Sbryanv error = vtpci_alloc_intr_msix_pervq(sc); 540246582Sbryanv break; 541246582Sbryanv case 1: 542246582Sbryanv error = vtpci_alloc_intr_msix_shared(sc); 543246582Sbryanv break; 544246582Sbryanv case 2: 545246582Sbryanv error = vtpci_alloc_intr_msi(sc); 546246582Sbryanv break; 547246582Sbryanv case 3: 548246582Sbryanv error = vtpci_alloc_intr_legacy(sc); 549246582Sbryanv break; 550246582Sbryanv default: 551246582Sbryanv device_printf(dev, 552246582Sbryanv "exhausted all interrupt allocation attempts\n"); 553246582Sbryanv return (ENXIO); 554246582Sbryanv } 555227652Sgrehan 556246582Sbryanv if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 557246582Sbryanv break; 558227652Sgrehan 559246582Sbryanv vtpci_cleanup_setup_intr_attempt(sc); 560227652Sgrehan } 561227652Sgrehan 562246582Sbryanv if (bootverbose) { 563246582Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 564246582Sbryanv device_printf(dev, "using legacy interrupt\n"); 565246582Sbryanv else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 566246582Sbryanv device_printf(dev, "using MSI interrupt\n"); 567246582Sbryanv else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 568246582Sbryanv device_printf(dev, "using shared MSIX interrupts\n"); 569246582Sbryanv else 570246582Sbryanv device_printf(dev, "using per VQ MSIX interrupts\n"); 571227652Sgrehan } 572227652Sgrehan 573227652Sgrehan return (0); 574227652Sgrehan} 575227652Sgrehan 576227652Sgrehanstatic void 577227652Sgrehanvtpci_stop(device_t dev) 578227652Sgrehan{ 579227652Sgrehan 580227652Sgrehan vtpci_reset(device_get_softc(dev)); 581227652Sgrehan} 582227652Sgrehan 583227652Sgrehanstatic int 584227652Sgrehanvtpci_reinit(device_t dev, uint64_t features) 585227652Sgrehan{ 586227652Sgrehan struct vtpci_softc *sc; 587246582Sbryanv int idx, error; 588227652Sgrehan 589227652Sgrehan sc = device_get_softc(dev); 590227652Sgrehan 591227652Sgrehan /* 592246582Sbryanv * Redrive the device initialization. This is a bit of an abuse of 593246582Sbryanv * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 594246582Sbryanv * play nice. 595246582Sbryanv * 596246582Sbryanv * We do not allow the host device to change from what was originally 597246582Sbryanv * negotiated beyond what the guest driver changed. MSIX state should 598246582Sbryanv * not change, number of virtqueues and their size remain the same, etc. 599246582Sbryanv * This will need to be rethought when we want to support migration. 600227652Sgrehan */ 601227652Sgrehan 602227652Sgrehan if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 603227652Sgrehan vtpci_stop(dev); 604227652Sgrehan 605227652Sgrehan /* 606227652Sgrehan * Quickly drive the status through ACK and DRIVER. The device 607227652Sgrehan * does not become usable again until vtpci_reinit_complete(). 608227652Sgrehan */ 609227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 610227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 611227652Sgrehan 612227652Sgrehan vtpci_negotiate_features(dev, features); 613227652Sgrehan 614246582Sbryanv for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 615246582Sbryanv error = vtpci_reinit_virtqueue(sc, idx); 616227652Sgrehan if (error) 617227652Sgrehan return (error); 618227652Sgrehan } 619227652Sgrehan 620246582Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 621246582Sbryanv error = vtpci_set_host_msix_vectors(sc); 622227652Sgrehan if (error) 623227652Sgrehan return (error); 624227652Sgrehan } 625227652Sgrehan 626227652Sgrehan return (0); 627227652Sgrehan} 628227652Sgrehan 629227652Sgrehanstatic void 630227652Sgrehanvtpci_reinit_complete(device_t dev) 631227652Sgrehan{ 632227652Sgrehan 633227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 634227652Sgrehan} 635227652Sgrehan 636227652Sgrehanstatic void 637227652Sgrehanvtpci_notify_virtqueue(device_t dev, uint16_t queue) 638227652Sgrehan{ 639227652Sgrehan struct vtpci_softc *sc; 640227652Sgrehan 641227652Sgrehan sc = device_get_softc(dev); 642227652Sgrehan 643227652Sgrehan vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 644227652Sgrehan} 645227652Sgrehan 646227652Sgrehanstatic uint8_t 647227652Sgrehanvtpci_get_status(device_t dev) 648227652Sgrehan{ 649227652Sgrehan struct vtpci_softc *sc; 650227652Sgrehan 651227652Sgrehan sc = device_get_softc(dev); 652227652Sgrehan 653227652Sgrehan return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 654227652Sgrehan} 655227652Sgrehan 656227652Sgrehanstatic void 657227652Sgrehanvtpci_set_status(device_t dev, uint8_t status) 658227652Sgrehan{ 659227652Sgrehan struct vtpci_softc *sc; 660227652Sgrehan 661227652Sgrehan sc = device_get_softc(dev); 662227652Sgrehan 663227652Sgrehan if (status != VIRTIO_CONFIG_STATUS_RESET) 664227652Sgrehan status |= vtpci_get_status(dev); 665227652Sgrehan 666227652Sgrehan vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 667227652Sgrehan} 668227652Sgrehan 669227652Sgrehanstatic void 670227652Sgrehanvtpci_read_dev_config(device_t dev, bus_size_t offset, 671227652Sgrehan void *dst, int length) 672227652Sgrehan{ 673227652Sgrehan struct vtpci_softc *sc; 674227652Sgrehan bus_size_t off; 675227652Sgrehan uint8_t *d; 676227652Sgrehan int size; 677227652Sgrehan 678227652Sgrehan sc = device_get_softc(dev); 679227652Sgrehan off = VIRTIO_PCI_CONFIG(sc) + offset; 680227652Sgrehan 681227652Sgrehan for (d = dst; length > 0; d += size, off += size, length -= size) { 682227652Sgrehan if (length >= 4) { 683227652Sgrehan size = 4; 684227652Sgrehan *(uint32_t *)d = vtpci_read_config_4(sc, off); 685227652Sgrehan } else if (length >= 2) { 686227652Sgrehan size = 2; 687227652Sgrehan *(uint16_t *)d = vtpci_read_config_2(sc, off); 688227652Sgrehan } else { 689227652Sgrehan size = 1; 690227652Sgrehan *d = vtpci_read_config_1(sc, off); 691227652Sgrehan } 692227652Sgrehan } 693227652Sgrehan} 694227652Sgrehan 695227652Sgrehanstatic void 696227652Sgrehanvtpci_write_dev_config(device_t dev, bus_size_t offset, 697227652Sgrehan void *src, int length) 698227652Sgrehan{ 699227652Sgrehan struct vtpci_softc *sc; 700227652Sgrehan bus_size_t off; 701227652Sgrehan uint8_t *s; 702227652Sgrehan int size; 703227652Sgrehan 704227652Sgrehan sc = device_get_softc(dev); 705227652Sgrehan off = VIRTIO_PCI_CONFIG(sc) + offset; 706227652Sgrehan 707227652Sgrehan for (s = src; length > 0; s += size, off += size, length -= size) { 708227652Sgrehan if (length >= 4) { 709227652Sgrehan size = 4; 710227652Sgrehan vtpci_write_config_4(sc, off, *(uint32_t *)s); 711227652Sgrehan } else if (length >= 2) { 712227652Sgrehan size = 2; 713227652Sgrehan vtpci_write_config_2(sc, off, *(uint16_t *)s); 714227652Sgrehan } else { 715227652Sgrehan size = 1; 716227652Sgrehan vtpci_write_config_1(sc, off, *s); 717227652Sgrehan } 718227652Sgrehan } 719227652Sgrehan} 720227652Sgrehan 721227652Sgrehanstatic void 722227652Sgrehanvtpci_describe_features(struct vtpci_softc *sc, const char *msg, 723227652Sgrehan uint64_t features) 724227652Sgrehan{ 725227652Sgrehan device_t dev, child; 726227652Sgrehan 727227652Sgrehan dev = sc->vtpci_dev; 728227652Sgrehan child = sc->vtpci_child_dev; 729227652Sgrehan 730227652Sgrehan if (device_is_attached(child) && bootverbose == 0) 731227652Sgrehan return; 732227652Sgrehan 733227652Sgrehan virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 734227652Sgrehan} 735227652Sgrehan 736227652Sgrehanstatic void 737227652Sgrehanvtpci_probe_and_attach_child(struct vtpci_softc *sc) 738227652Sgrehan{ 739227652Sgrehan device_t dev, child; 740227652Sgrehan 741227652Sgrehan dev = sc->vtpci_dev; 742227652Sgrehan child = sc->vtpci_child_dev; 743227652Sgrehan 744227652Sgrehan if (child == NULL) 745227652Sgrehan return; 746227652Sgrehan 747227652Sgrehan if (device_get_state(child) != DS_NOTPRESENT) 748227652Sgrehan return; 749227652Sgrehan 750227652Sgrehan if (device_probe(child) != 0) 751227652Sgrehan return; 752227652Sgrehan 753227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 754227652Sgrehan if (device_attach(child) != 0) { 755227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 756227652Sgrehan vtpci_reset(sc); 757227652Sgrehan vtpci_release_child_resources(sc); 758227652Sgrehan /* Reset status for future attempt. */ 759227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 760265298Sbryanv } else { 761227652Sgrehan vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 762265298Sbryanv VIRTIO_ATTACH_COMPLETED(child); 763265298Sbryanv } 764227652Sgrehan} 765227652Sgrehan 766227652Sgrehanstatic int 767246582Sbryanvvtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 768227652Sgrehan{ 769246582Sbryanv device_t dev; 770246582Sbryanv int nmsix, cnt, required; 771246582Sbryanv 772246582Sbryanv dev = sc->vtpci_dev; 773246582Sbryanv 774246582Sbryanv /* Allocate an additional vector for the config changes. */ 775246582Sbryanv required = nvectors + 1; 776246582Sbryanv 777246582Sbryanv nmsix = pci_msix_count(dev); 778246582Sbryanv if (nmsix < required) 779246582Sbryanv return (1); 780246582Sbryanv 781246582Sbryanv cnt = required; 782246582Sbryanv if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 783253132Sbryanv sc->vtpci_nmsix_resources = required; 784246582Sbryanv return (0); 785246582Sbryanv } 786246582Sbryanv 787246582Sbryanv pci_release_msi(dev); 788246582Sbryanv 789246582Sbryanv return (1); 790246582Sbryanv} 791246582Sbryanv 792246582Sbryanvstatic int 793246582Sbryanvvtpci_alloc_msi(struct vtpci_softc *sc) 794246582Sbryanv{ 795246582Sbryanv device_t dev; 796246582Sbryanv int nmsi, cnt, required; 797246582Sbryanv 798246582Sbryanv dev = sc->vtpci_dev; 799246582Sbryanv required = 1; 800246582Sbryanv 801246582Sbryanv nmsi = pci_msi_count(dev); 802246582Sbryanv if (nmsi < required) 803246582Sbryanv return (1); 804246582Sbryanv 805246582Sbryanv cnt = required; 806253132Sbryanv if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 807246582Sbryanv return (0); 808246582Sbryanv 809246582Sbryanv pci_release_msi(dev); 810246582Sbryanv 811246582Sbryanv return (1); 812246582Sbryanv} 813246582Sbryanv 814246582Sbryanvstatic int 815246582Sbryanvvtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 816246582Sbryanv{ 817227652Sgrehan int i, nvectors, error; 818227652Sgrehan 819246582Sbryanv if (vtpci_disable_msix != 0 || 820246582Sbryanv sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 821246582Sbryanv return (ENOTSUP); 822246582Sbryanv 823246582Sbryanv for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 824253132Sbryanv if (sc->vtpci_vqs[i].vtv_no_intr == 0) 825227652Sgrehan nvectors++; 826246582Sbryanv } 827227652Sgrehan 828246582Sbryanv error = vtpci_alloc_msix(sc, nvectors); 829246582Sbryanv if (error) 830246582Sbryanv return (error); 831246582Sbryanv 832246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_MSIX; 833246582Sbryanv 834246582Sbryanv return (0); 835246582Sbryanv} 836246582Sbryanv 837246582Sbryanvstatic int 838246582Sbryanvvtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 839246582Sbryanv{ 840246582Sbryanv int error; 841246582Sbryanv 842227652Sgrehan if (vtpci_disable_msix != 0 || 843246582Sbryanv sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 844246582Sbryanv return (ENOTSUP); 845227652Sgrehan 846246582Sbryanv error = vtpci_alloc_msix(sc, 1); 847246582Sbryanv if (error) 848246582Sbryanv return (error); 849227652Sgrehan 850246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 851227652Sgrehan 852246582Sbryanv return (0); 853227652Sgrehan} 854227652Sgrehan 855227652Sgrehanstatic int 856246582Sbryanvvtpci_alloc_intr_msi(struct vtpci_softc *sc) 857227652Sgrehan{ 858246582Sbryanv int error; 859246582Sbryanv 860246582Sbryanv /* Only BHyVe supports MSI. */ 861246582Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 862246582Sbryanv return (ENOTSUP); 863246582Sbryanv 864246582Sbryanv error = vtpci_alloc_msi(sc); 865246582Sbryanv if (error) 866246582Sbryanv return (error); 867246582Sbryanv 868246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_MSI; 869246582Sbryanv 870246582Sbryanv return (0); 871246582Sbryanv} 872246582Sbryanv 873246582Sbryanvstatic int 874246582Sbryanvvtpci_alloc_intr_legacy(struct vtpci_softc *sc) 875246582Sbryanv{ 876246582Sbryanv 877246582Sbryanv sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 878246582Sbryanv 879246582Sbryanv return (0); 880246582Sbryanv} 881246582Sbryanv 882246582Sbryanvstatic int 883253132Sbryanvvtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 884253132Sbryanv struct vtpci_interrupt *intr) 885246582Sbryanv{ 886227652Sgrehan struct resource *irq; 887227652Sgrehan 888253132Sbryanv irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 889253132Sbryanv if (irq == NULL) 890253132Sbryanv return (ENXIO); 891227652Sgrehan 892253132Sbryanv intr->vti_irq = irq; 893253132Sbryanv intr->vti_rid = rid; 894227652Sgrehan 895253132Sbryanv return (0); 896253132Sbryanv} 897227652Sgrehan 898253132Sbryanvstatic int 899253132Sbryanvvtpci_alloc_intr_resources(struct vtpci_softc *sc) 900253132Sbryanv{ 901253132Sbryanv struct vtpci_interrupt *intr; 902253132Sbryanv int i, rid, flags, nvq_intrs, error; 903227652Sgrehan 904253132Sbryanv rid = 0; 905253132Sbryanv flags = RF_ACTIVE; 906253132Sbryanv 907253132Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 908253132Sbryanv flags |= RF_SHAREABLE; 909253132Sbryanv else 910253132Sbryanv rid = 1; 911253132Sbryanv 912227652Sgrehan /* 913253132Sbryanv * For legacy and MSI interrupts, this single resource handles all 914253132Sbryanv * interrupts. For MSIX, this resource is used for the configuration 915253132Sbryanv * changed interrupt. 916227652Sgrehan */ 917253132Sbryanv intr = &sc->vtpci_device_interrupt; 918253132Sbryanv error = vtpci_alloc_interrupt(sc, rid, flags, intr); 919253132Sbryanv if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 920253132Sbryanv return (error); 921227652Sgrehan 922253132Sbryanv /* Subtract one for the configuration changed interrupt. */ 923253132Sbryanv nvq_intrs = sc->vtpci_nmsix_resources - 1; 924253132Sbryanv 925253132Sbryanv intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 926253132Sbryanv sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 927253132Sbryanv if (sc->vtpci_msix_vq_interrupts == NULL) 928253132Sbryanv return (ENOMEM); 929253132Sbryanv 930253132Sbryanv for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 931253132Sbryanv error = vtpci_alloc_interrupt(sc, rid, flags, intr); 932253132Sbryanv if (error) 933253132Sbryanv return (error); 934227652Sgrehan } 935227652Sgrehan 936227652Sgrehan return (0); 937227652Sgrehan} 938227652Sgrehan 939227652Sgrehanstatic int 940246582Sbryanvvtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 941227652Sgrehan{ 942253132Sbryanv struct vtpci_interrupt *intr; 943246582Sbryanv int error; 944227652Sgrehan 945253132Sbryanv intr = &sc->vtpci_device_interrupt; 946253132Sbryanv error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 947253132Sbryanv vtpci_legacy_intr, sc, &intr->vti_handler); 948227652Sgrehan 949246582Sbryanv return (error); 950227652Sgrehan} 951227652Sgrehan 952227652Sgrehanstatic int 953253132Sbryanvvtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 954227652Sgrehan{ 955246582Sbryanv struct vtpci_virtqueue *vqx; 956253132Sbryanv struct vtpci_interrupt *intr; 957246582Sbryanv int i, error; 958227652Sgrehan 959253132Sbryanv intr = sc->vtpci_msix_vq_interrupts; 960227652Sgrehan 961253132Sbryanv for (i = 0; i < sc->vtpci_nvqs; i++) { 962253132Sbryanv vqx = &sc->vtpci_vqs[i]; 963227652Sgrehan 964253132Sbryanv if (vqx->vtv_no_intr) 965253132Sbryanv continue; 966227652Sgrehan 967253132Sbryanv error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 968253132Sbryanv vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 969253132Sbryanv &intr->vti_handler); 970246582Sbryanv if (error) 971246582Sbryanv return (error); 972227652Sgrehan 973253132Sbryanv intr++; 974227652Sgrehan } 975227652Sgrehan 976253132Sbryanv return (0); 977253132Sbryanv} 978253132Sbryanv 979253132Sbryanvstatic int 980253132Sbryanvvtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 981253132Sbryanv{ 982253132Sbryanv device_t dev; 983253132Sbryanv struct vtpci_interrupt *intr; 984253132Sbryanv int error; 985253132Sbryanv 986253132Sbryanv dev = sc->vtpci_dev; 987253132Sbryanv intr = &sc->vtpci_device_interrupt; 988253132Sbryanv 989253132Sbryanv error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 990253132Sbryanv vtpci_config_intr, sc, &intr->vti_handler); 991246582Sbryanv if (error) 992246582Sbryanv return (error); 993227652Sgrehan 994253132Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 995253132Sbryanv intr = sc->vtpci_msix_vq_interrupts; 996253132Sbryanv error = bus_setup_intr(dev, intr->vti_irq, type, 997253132Sbryanv vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 998253132Sbryanv &intr->vti_handler); 999253132Sbryanv } else 1000253132Sbryanv error = vtpci_setup_pervq_msix_interrupts(sc, type); 1001253132Sbryanv 1002253132Sbryanv return (error ? error : vtpci_set_host_msix_vectors(sc)); 1003246582Sbryanv} 1004227652Sgrehan 1005246582Sbryanvstatic int 1006246582Sbryanvvtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1007246582Sbryanv{ 1008246582Sbryanv int error; 1009227652Sgrehan 1010246582Sbryanv type |= INTR_MPSAFE; 1011246582Sbryanv KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1012253132Sbryanv ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1013246582Sbryanv 1014246582Sbryanv error = vtpci_alloc_intr_resources(sc); 1015246582Sbryanv if (error) 1016246582Sbryanv return (error); 1017246582Sbryanv 1018246582Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1019246582Sbryanv error = vtpci_setup_legacy_interrupt(sc, type); 1020246582Sbryanv else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1021246582Sbryanv error = vtpci_setup_msi_interrupt(sc, type); 1022246582Sbryanv else 1023246582Sbryanv error = vtpci_setup_msix_interrupts(sc, type); 1024246582Sbryanv 1025246582Sbryanv return (error); 1026227652Sgrehan} 1027227652Sgrehan 1028227652Sgrehanstatic int 1029253132Sbryanvvtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1030253132Sbryanv struct vtpci_interrupt *intr) 1031227652Sgrehan{ 1032227652Sgrehan device_t dev; 1033253132Sbryanv uint16_t vector; 1034227652Sgrehan 1035227652Sgrehan dev = sc->vtpci_dev; 1036227652Sgrehan 1037253132Sbryanv if (intr != NULL) { 1038246582Sbryanv /* Map from guest rid to host vector. */ 1039253132Sbryanv vector = intr->vti_rid - 1; 1040227652Sgrehan } else 1041227652Sgrehan vector = VIRTIO_MSI_NO_VECTOR; 1042227652Sgrehan 1043227652Sgrehan vtpci_write_config_2(sc, offset, vector); 1044227652Sgrehan 1045246582Sbryanv /* Read vector to determine if the host had sufficient resources. */ 1046253132Sbryanv if (vtpci_read_config_2(sc, offset) != vector) { 1047246582Sbryanv device_printf(dev, 1048246582Sbryanv "insufficient host resources for MSIX interrupts\n"); 1049227652Sgrehan return (ENODEV); 1050227652Sgrehan } 1051227652Sgrehan 1052227652Sgrehan return (0); 1053227652Sgrehan} 1054227652Sgrehan 1055246582Sbryanvstatic int 1056246582Sbryanvvtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1057246582Sbryanv{ 1058253132Sbryanv struct vtpci_interrupt *intr, *tintr; 1059253132Sbryanv int idx, offset, error; 1060246582Sbryanv 1061253132Sbryanv intr = &sc->vtpci_device_interrupt; 1062253132Sbryanv offset = VIRTIO_MSI_CONFIG_VECTOR; 1063253132Sbryanv 1064253132Sbryanv error = vtpci_register_msix_vector(sc, offset, intr); 1065246582Sbryanv if (error) 1066246582Sbryanv return (error); 1067246582Sbryanv 1068253132Sbryanv intr = sc->vtpci_msix_vq_interrupts; 1069253132Sbryanv offset = VIRTIO_MSI_QUEUE_VECTOR; 1070253132Sbryanv 1071246582Sbryanv for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1072253132Sbryanv vtpci_select_virtqueue(sc, idx); 1073246582Sbryanv 1074253132Sbryanv if (sc->vtpci_vqs[idx].vtv_no_intr) 1075253132Sbryanv tintr = NULL; 1076253132Sbryanv else 1077253132Sbryanv tintr = intr; 1078253132Sbryanv 1079253132Sbryanv error = vtpci_register_msix_vector(sc, offset, tintr); 1080246582Sbryanv if (error) 1081253132Sbryanv break; 1082253132Sbryanv 1083253132Sbryanv /* 1084253132Sbryanv * For shared MSIX, all the virtqueues share the first 1085253132Sbryanv * interrupt. 1086253132Sbryanv */ 1087253132Sbryanv if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1088253132Sbryanv intr++; 1089246582Sbryanv } 1090246582Sbryanv 1091253132Sbryanv return (error); 1092246582Sbryanv} 1093246582Sbryanv 1094246582Sbryanvstatic int 1095246582Sbryanvvtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1096246582Sbryanv{ 1097246582Sbryanv struct vtpci_virtqueue *vqx; 1098246582Sbryanv struct virtqueue *vq; 1099246582Sbryanv int error; 1100246582Sbryanv uint16_t size; 1101246582Sbryanv 1102253132Sbryanv vqx = &sc->vtpci_vqs[idx]; 1103253132Sbryanv vq = vqx->vtv_vq; 1104246582Sbryanv 1105253132Sbryanv KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1106246582Sbryanv 1107246582Sbryanv vtpci_select_virtqueue(sc, idx); 1108246582Sbryanv size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 1109246582Sbryanv 1110246582Sbryanv error = virtqueue_reinit(vq, size); 1111246582Sbryanv if (error) 1112246582Sbryanv return (error); 1113246582Sbryanv 1114246582Sbryanv vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 1115246582Sbryanv virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1116246582Sbryanv 1117246582Sbryanv return (0); 1118246582Sbryanv} 1119246582Sbryanv 1120227652Sgrehanstatic void 1121253132Sbryanvvtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1122227652Sgrehan{ 1123227652Sgrehan device_t dev; 1124227652Sgrehan 1125227652Sgrehan dev = sc->vtpci_dev; 1126227652Sgrehan 1127253132Sbryanv if (intr->vti_handler != NULL) { 1128253132Sbryanv bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1129253132Sbryanv intr->vti_handler = NULL; 1130253132Sbryanv } 1131227652Sgrehan 1132253132Sbryanv if (intr->vti_irq != NULL) { 1133253132Sbryanv bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1134253132Sbryanv intr->vti_irq); 1135253132Sbryanv intr->vti_irq = NULL; 1136253132Sbryanv intr->vti_rid = -1; 1137253132Sbryanv } 1138253132Sbryanv} 1139227652Sgrehan 1140253132Sbryanvstatic void 1141253132Sbryanvvtpci_free_interrupts(struct vtpci_softc *sc) 1142253132Sbryanv{ 1143253132Sbryanv struct vtpci_interrupt *intr; 1144253132Sbryanv int i, nvq_intrs; 1145253132Sbryanv 1146253132Sbryanv vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1147253132Sbryanv 1148253132Sbryanv if (sc->vtpci_nmsix_resources != 0) { 1149253132Sbryanv nvq_intrs = sc->vtpci_nmsix_resources - 1; 1150253132Sbryanv sc->vtpci_nmsix_resources = 0; 1151253132Sbryanv 1152253132Sbryanv intr = sc->vtpci_msix_vq_interrupts; 1153253132Sbryanv if (intr != NULL) { 1154253132Sbryanv for (i = 0; i < nvq_intrs; i++, intr++) 1155253132Sbryanv vtpci_free_interrupt(sc, intr); 1156253132Sbryanv 1157253132Sbryanv free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1158253132Sbryanv sc->vtpci_msix_vq_interrupts = NULL; 1159227652Sgrehan } 1160227652Sgrehan } 1161246582Sbryanv 1162246582Sbryanv if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1163253132Sbryanv pci_release_msi(sc->vtpci_dev); 1164246582Sbryanv 1165246582Sbryanv sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1166227652Sgrehan} 1167227652Sgrehan 1168227652Sgrehanstatic void 1169227652Sgrehanvtpci_free_virtqueues(struct vtpci_softc *sc) 1170227652Sgrehan{ 1171227652Sgrehan struct vtpci_virtqueue *vqx; 1172253132Sbryanv int idx; 1173227652Sgrehan 1174253132Sbryanv for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1175253132Sbryanv vqx = &sc->vtpci_vqs[idx]; 1176246582Sbryanv 1177253132Sbryanv vtpci_select_virtqueue(sc, idx); 1178253132Sbryanv vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1179253132Sbryanv 1180253132Sbryanv virtqueue_free(vqx->vtv_vq); 1181253132Sbryanv vqx->vtv_vq = NULL; 1182246582Sbryanv } 1183246582Sbryanv 1184253132Sbryanv free(sc->vtpci_vqs, M_DEVBUF); 1185253132Sbryanv sc->vtpci_vqs = NULL; 1186227652Sgrehan sc->vtpci_nvqs = 0; 1187246582Sbryanv} 1188227652Sgrehan 1189246582Sbryanvstatic void 1190253132Sbryanvvtpci_release_child_resources(struct vtpci_softc *sc) 1191253132Sbryanv{ 1192253132Sbryanv 1193253132Sbryanv vtpci_free_interrupts(sc); 1194253132Sbryanv vtpci_free_virtqueues(sc); 1195253132Sbryanv} 1196253132Sbryanv 1197253132Sbryanvstatic void 1198246582Sbryanvvtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1199246582Sbryanv{ 1200246582Sbryanv int idx; 1201227652Sgrehan 1202246582Sbryanv if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1203246582Sbryanv vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1204246582Sbryanv VIRTIO_MSI_NO_VECTOR); 1205246582Sbryanv 1206246582Sbryanv for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1207246582Sbryanv vtpci_select_virtqueue(sc, idx); 1208246582Sbryanv vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1209246582Sbryanv VIRTIO_MSI_NO_VECTOR); 1210227652Sgrehan } 1211227652Sgrehan } 1212246582Sbryanv 1213246582Sbryanv vtpci_free_interrupts(sc); 1214227652Sgrehan} 1215227652Sgrehan 1216227652Sgrehanstatic void 1217227652Sgrehanvtpci_reset(struct vtpci_softc *sc) 1218227652Sgrehan{ 1219227652Sgrehan 1220227652Sgrehan /* 1221227652Sgrehan * Setting the status to RESET sets the host device to 1222227652Sgrehan * the original, uninitialized state. 1223227652Sgrehan */ 1224227652Sgrehan vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1225227652Sgrehan} 1226227652Sgrehan 1227246582Sbryanvstatic void 1228246582Sbryanvvtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1229246582Sbryanv{ 1230246582Sbryanv 1231246582Sbryanv vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1232246582Sbryanv} 1233246582Sbryanv 1234253132Sbryanvstatic void 1235227652Sgrehanvtpci_legacy_intr(void *xsc) 1236227652Sgrehan{ 1237227652Sgrehan struct vtpci_softc *sc; 1238227652Sgrehan struct vtpci_virtqueue *vqx; 1239227652Sgrehan int i; 1240227652Sgrehan uint8_t isr; 1241227652Sgrehan 1242227652Sgrehan sc = xsc; 1243253132Sbryanv vqx = &sc->vtpci_vqs[0]; 1244227652Sgrehan 1245227652Sgrehan /* Reading the ISR also clears it. */ 1246227652Sgrehan isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1247227652Sgrehan 1248227652Sgrehan if (isr & VIRTIO_PCI_ISR_CONFIG) 1249227652Sgrehan vtpci_config_intr(sc); 1250227652Sgrehan 1251253132Sbryanv if (isr & VIRTIO_PCI_ISR_INTR) { 1252253132Sbryanv for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1253253132Sbryanv if (vqx->vtv_no_intr == 0) 1254253132Sbryanv virtqueue_intr(vqx->vtv_vq); 1255253132Sbryanv } 1256253132Sbryanv } 1257227652Sgrehan} 1258227652Sgrehan 1259227652Sgrehanstatic int 1260253132Sbryanvvtpci_vq_shared_intr_filter(void *xsc) 1261227652Sgrehan{ 1262227652Sgrehan struct vtpci_softc *sc; 1263227652Sgrehan struct vtpci_virtqueue *vqx; 1264227652Sgrehan int i, rc; 1265227652Sgrehan 1266227652Sgrehan rc = 0; 1267227652Sgrehan sc = xsc; 1268253132Sbryanv vqx = &sc->vtpci_vqs[0]; 1269227652Sgrehan 1270253132Sbryanv for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1271253132Sbryanv if (vqx->vtv_no_intr == 0) 1272253132Sbryanv rc |= virtqueue_intr_filter(vqx->vtv_vq); 1273253132Sbryanv } 1274227652Sgrehan 1275253132Sbryanv return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1276227652Sgrehan} 1277227652Sgrehan 1278253132Sbryanvstatic void 1279253132Sbryanvvtpci_vq_shared_intr(void *xsc) 1280253132Sbryanv{ 1281253132Sbryanv struct vtpci_softc *sc; 1282253132Sbryanv struct vtpci_virtqueue *vqx; 1283253132Sbryanv int i; 1284253132Sbryanv 1285253132Sbryanv sc = xsc; 1286253132Sbryanv vqx = &sc->vtpci_vqs[0]; 1287253132Sbryanv 1288253132Sbryanv for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1289253132Sbryanv if (vqx->vtv_no_intr == 0) 1290253132Sbryanv virtqueue_intr(vqx->vtv_vq); 1291253132Sbryanv } 1292253132Sbryanv} 1293253132Sbryanv 1294227652Sgrehanstatic int 1295253132Sbryanvvtpci_vq_intr_filter(void *xvq) 1296227652Sgrehan{ 1297227652Sgrehan struct virtqueue *vq; 1298227652Sgrehan int rc; 1299227652Sgrehan 1300227652Sgrehan vq = xvq; 1301253132Sbryanv rc = virtqueue_intr_filter(vq); 1302227652Sgrehan 1303253132Sbryanv return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1304227652Sgrehan} 1305227652Sgrehan 1306253132Sbryanvstatic void 1307253132Sbryanvvtpci_vq_intr(void *xvq) 1308253132Sbryanv{ 1309253132Sbryanv struct virtqueue *vq; 1310253132Sbryanv 1311253132Sbryanv vq = xvq; 1312253132Sbryanv virtqueue_intr(vq); 1313253132Sbryanv} 1314253132Sbryanv 1315253132Sbryanvstatic void 1316227652Sgrehanvtpci_config_intr(void *xsc) 1317227652Sgrehan{ 1318227652Sgrehan struct vtpci_softc *sc; 1319227652Sgrehan device_t child; 1320227652Sgrehan 1321227652Sgrehan sc = xsc; 1322227652Sgrehan child = sc->vtpci_child_dev; 1323227652Sgrehan 1324227652Sgrehan if (child != NULL) 1325253132Sbryanv VIRTIO_CONFIG_CHANGE(child); 1326227652Sgrehan} 1327