1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* Driver for the modern VirtIO PCI interface. */ 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/bus.h> 34#include <sys/lock.h> 35#include <sys/kernel.h> 36#include <sys/module.h> 37 38#include <machine/bus.h> 39#include <machine/cpu.h> 40#include <machine/resource.h> 41#include <sys/bus.h> 42#include <sys/rman.h> 43 44#include <dev/pci/pcivar.h> 45#include <dev/pci/pcireg.h> 46 47#include <dev/virtio/virtio.h> 48#include <dev/virtio/virtqueue.h> 49#include <dev/virtio/pci/virtio_pci.h> 50#include <dev/virtio/pci/virtio_pci_modern_var.h> 51 52#include "virtio_bus_if.h" 53#include "virtio_pci_if.h" 54#include "virtio_if.h" 55 56struct vtpci_modern_resource_map { 57 struct resource_map vtrm_map; 58 int vtrm_cap_offset; 59 int vtrm_bar; 60 int vtrm_offset; 61 int vtrm_length; 62 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 63}; 64 65struct vtpci_modern_bar_resource { 66 struct resource *vtbr_res; 67 int vtbr_type; 68}; 69 70struct vtpci_modern_softc { 71 device_t vtpci_dev; 72 struct vtpci_common vtpci_common; 73 uint32_t vtpci_notify_offset_multiplier; 74 uint16_t vtpci_devid; 75 int vtpci_msix_bar; 76 struct resource *vtpci_msix_res; 77 78 struct vtpci_modern_resource_map vtpci_common_res_map; 79 struct vtpci_modern_resource_map vtpci_notify_res_map; 80 struct vtpci_modern_resource_map vtpci_isr_res_map; 81 struct vtpci_modern_resource_map vtpci_device_res_map; 82 83#define VTPCI_MODERN_MAX_BARS 6 84 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 85}; 86 87static int vtpci_modern_probe(device_t); 88static int vtpci_modern_attach(device_t); 89static int vtpci_modern_detach(device_t); 90static int vtpci_modern_suspend(device_t); 91static int vtpci_modern_resume(device_t); 92static int vtpci_modern_shutdown(device_t); 93 94static void vtpci_modern_driver_added(device_t, driver_t *); 95static void vtpci_modern_child_detached(device_t, device_t); 96static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 97static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 98 99static uint8_t vtpci_modern_read_isr(device_t); 100static uint16_t vtpci_modern_get_vq_size(device_t, int); 101static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 102static void vtpci_modern_set_vq(device_t, struct virtqueue *); 103static void vtpci_modern_disable_vq(device_t, int); 104static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 105 struct vtpci_interrupt *); 106static int vtpci_modern_register_cfg_msix(device_t, 107 struct vtpci_interrupt *); 108static int vtpci_modern_register_vq_msix(device_t, int idx, 109 struct vtpci_interrupt *); 110 111static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 112static int vtpci_modern_finalize_features(device_t); 113static bool vtpci_modern_with_feature(device_t, uint64_t); 114static int vtpci_modern_alloc_virtqueues(device_t, int, 115 struct vq_alloc_info *); 116static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 117static void vtpci_modern_stop(device_t); 118static int vtpci_modern_reinit(device_t, uint64_t); 119static void vtpci_modern_reinit_complete(device_t); 120static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 121static int vtpci_modern_config_generation(device_t); 122static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 123static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 124 125static int vtpci_modern_probe_configs(device_t); 126static int vtpci_modern_find_cap(device_t, uint8_t, int *); 127static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 128static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 129static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 130 uint8_t, int, int, struct vtpci_modern_resource_map *); 131static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 132static struct resource *vtpci_modern_get_bar_resource( 133 struct vtpci_modern_softc *, int, int); 134static struct resource *vtpci_modern_alloc_bar_resource( 135 struct vtpci_modern_softc *, int, int); 136static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 137static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 138 struct vtpci_modern_resource_map *); 139static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 140 struct vtpci_modern_resource_map *); 141static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 142static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 143 144static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 145 146static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 147static void vtpci_modern_write_features(struct vtpci_modern_softc *, 148 uint64_t); 149static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 150static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 151static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 152static void vtpci_modern_reset(struct vtpci_modern_softc *); 153static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 154 155static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 156 bus_size_t); 157static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 158 bus_size_t); 159static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 160 bus_size_t); 161static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 162 bus_size_t, uint8_t); 163static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 164 bus_size_t, uint16_t); 165static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 166 bus_size_t, uint32_t); 167static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 168 bus_size_t, uint64_t); 169static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 170 bus_size_t, uint16_t); 171static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 172 bus_size_t); 173static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 174 bus_size_t); 175static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 176 bus_size_t); 177static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 178 bus_size_t); 179static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 180 bus_size_t); 181static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 182 bus_size_t, uint8_t); 183static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 184 bus_size_t, uint16_t); 185static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 186 bus_size_t, uint32_t); 187static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 188 bus_size_t, uint64_t); 189 190/* Tunables. */ 191static int vtpci_modern_transitional = 0; 192TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 193 194static device_method_t vtpci_modern_methods[] = { 195 /* Device interface. */ 196 DEVMETHOD(device_probe, vtpci_modern_probe), 197 DEVMETHOD(device_attach, vtpci_modern_attach), 198 DEVMETHOD(device_detach, vtpci_modern_detach), 199 DEVMETHOD(device_suspend, vtpci_modern_suspend), 200 DEVMETHOD(device_resume, vtpci_modern_resume), 201 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 202 203 /* Bus interface. */ 204 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 205 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 206 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 207 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 208 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 209 210 /* VirtIO PCI interface. */ 211 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 212 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 213 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 214 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 215 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 216 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 217 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 218 219 /* VirtIO bus interface. */ 220 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 221 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 222 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 223 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 224 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 225 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 226 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 227 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 228 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 229 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 230 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 231 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 232 233 DEVMETHOD_END 234}; 235 236static driver_t vtpci_modern_driver = { 237 .name = "virtio_pci", 238 .methods = vtpci_modern_methods, 239 .size = sizeof(struct vtpci_modern_softc) 240}; 241 242DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); 243 244static int 245vtpci_modern_probe(device_t dev) 246{ 247 const char *name; 248 uint16_t devid; 249 250 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 251 return (ENXIO); 252 253 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 254 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 255 return (ENXIO); 256 257 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 258 if (!vtpci_modern_transitional) 259 return (ENXIO); 260 devid = pci_get_subdevice(dev); 261 } else 262 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 263 264 if (vtpci_modern_probe_configs(dev) != 0) 265 return (ENXIO); 266 267 name = virtio_device_name(devid); 268 if (name == NULL) 269 name = "Unknown"; 270 271 device_set_descf(dev, "VirtIO PCI (modern) %s adapter", name); 272 273 return (BUS_PROBE_DEFAULT); 274} 275 276static int 277vtpci_modern_attach(device_t dev) 278{ 279 struct vtpci_modern_softc *sc; 280 int error; 281 282 sc = device_get_softc(dev); 283 sc->vtpci_dev = dev; 284 vtpci_init(&sc->vtpci_common, dev, true); 285 286 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 287 sc->vtpci_devid = pci_get_subdevice(dev); 288 else 289 sc->vtpci_devid = pci_get_device(dev) - 290 VIRTIO_PCI_DEVICEID_MODERN_MIN; 291 292 error = vtpci_modern_map_configs(sc); 293 if (error) { 294 device_printf(dev, "cannot map configs\n"); 295 vtpci_modern_unmap_configs(sc); 296 return (error); 297 } 298 299 vtpci_modern_reset(sc); 300 301 /* Tell the host we've noticed this device. */ 302 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 303 304 error = vtpci_add_child(&sc->vtpci_common); 305 if (error) 306 goto fail; 307 308 vtpci_modern_probe_and_attach_child(sc); 309 310 return (0); 311 312fail: 313 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 314 vtpci_modern_detach(dev); 315 316 return (error); 317} 318 319static int 320vtpci_modern_detach(device_t dev) 321{ 322 struct vtpci_modern_softc *sc; 323 int error; 324 325 sc = device_get_softc(dev); 326 327 error = vtpci_delete_child(&sc->vtpci_common); 328 if (error) 329 return (error); 330 331 vtpci_modern_reset(sc); 332 vtpci_modern_unmap_configs(sc); 333 334 return (0); 335} 336 337static int 338vtpci_modern_suspend(device_t dev) 339{ 340 return (bus_generic_suspend(dev)); 341} 342 343static int 344vtpci_modern_resume(device_t dev) 345{ 346 return (bus_generic_resume(dev)); 347} 348 349static int 350vtpci_modern_shutdown(device_t dev) 351{ 352 (void) bus_generic_shutdown(dev); 353 /* Forcibly stop the host device. */ 354 vtpci_modern_stop(dev); 355 356 return (0); 357} 358 359static void 360vtpci_modern_driver_added(device_t dev, driver_t *driver) 361{ 362 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 363} 364 365static void 366vtpci_modern_child_detached(device_t dev, device_t child) 367{ 368 struct vtpci_modern_softc *sc; 369 370 sc = device_get_softc(dev); 371 372 vtpci_modern_reset(sc); 373 vtpci_child_detached(&sc->vtpci_common); 374 375 /* After the reset, retell the host we've noticed this device. */ 376 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 377} 378 379static int 380vtpci_modern_read_ivar(device_t dev, device_t child, int index, 381 uintptr_t *result) 382{ 383 struct vtpci_modern_softc *sc; 384 struct vtpci_common *cn; 385 386 sc = device_get_softc(dev); 387 cn = &sc->vtpci_common; 388 389 if (vtpci_child_device(cn) != child) 390 return (ENOENT); 391 392 switch (index) { 393 case VIRTIO_IVAR_DEVTYPE: 394 *result = sc->vtpci_devid; 395 break; 396 default: 397 return (vtpci_read_ivar(cn, index, result)); 398 } 399 400 return (0); 401} 402 403static int 404vtpci_modern_write_ivar(device_t dev, device_t child, int index, 405 uintptr_t value) 406{ 407 struct vtpci_modern_softc *sc; 408 struct vtpci_common *cn; 409 410 sc = device_get_softc(dev); 411 cn = &sc->vtpci_common; 412 413 if (vtpci_child_device(cn) != child) 414 return (ENOENT); 415 416 switch (index) { 417 default: 418 return (vtpci_write_ivar(cn, index, value)); 419 } 420 421 return (0); 422} 423 424static uint64_t 425vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 426{ 427 struct vtpci_modern_softc *sc; 428 uint64_t host_features, features; 429 430 sc = device_get_softc(dev); 431 host_features = vtpci_modern_read_features(sc); 432 433 /* 434 * Since the driver was added as a child of the modern PCI bus, 435 * always add the V1 flag. 436 */ 437 child_features |= VIRTIO_F_VERSION_1; 438 439 features = vtpci_negotiate_features(&sc->vtpci_common, 440 child_features, host_features); 441 vtpci_modern_write_features(sc, features); 442 443 return (features); 444} 445 446static int 447vtpci_modern_finalize_features(device_t dev) 448{ 449 struct vtpci_modern_softc *sc; 450 uint8_t status; 451 452 sc = device_get_softc(dev); 453 454 /* 455 * Must re-read the status after setting it to verify the negotiated 456 * features were accepted by the device. 457 */ 458 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 459 460 status = vtpci_modern_get_status(sc); 461 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 462 device_printf(dev, "desired features were not accepted\n"); 463 return (ENOTSUP); 464 } 465 466 return (0); 467} 468 469static bool 470vtpci_modern_with_feature(device_t dev, uint64_t feature) 471{ 472 struct vtpci_modern_softc *sc; 473 474 sc = device_get_softc(dev); 475 476 return (vtpci_with_feature(&sc->vtpci_common, feature)); 477} 478 479static uint64_t 480vtpci_modern_read_features(struct vtpci_modern_softc *sc) 481{ 482 uint32_t features0, features1; 483 484 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 485 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 486 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 487 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 488 489 return (((uint64_t) features1 << 32) | features0); 490} 491 492static void 493vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 494{ 495 uint32_t features0, features1; 496 497 features0 = features; 498 features1 = features >> 32; 499 500 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 501 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 502 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 504} 505 506static int 507vtpci_modern_alloc_virtqueues(device_t dev, int nvqs, 508 struct vq_alloc_info *vq_info) 509{ 510 struct vtpci_modern_softc *sc; 511 struct vtpci_common *cn; 512 uint16_t max_nvqs; 513 514 sc = device_get_softc(dev); 515 cn = &sc->vtpci_common; 516 517 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 518 if (nvqs > max_nvqs) { 519 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 520 "exceeds max %d\n", nvqs, max_nvqs); 521 return (E2BIG); 522 } 523 524 return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); 525} 526 527static int 528vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 529{ 530 struct vtpci_modern_softc *sc; 531 int error; 532 533 sc = device_get_softc(dev); 534 535 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 536 if (error == 0) 537 vtpci_modern_enable_virtqueues(sc); 538 539 return (error); 540} 541 542static void 543vtpci_modern_stop(device_t dev) 544{ 545 vtpci_modern_reset(device_get_softc(dev)); 546} 547 548static int 549vtpci_modern_reinit(device_t dev, uint64_t features) 550{ 551 struct vtpci_modern_softc *sc; 552 struct vtpci_common *cn; 553 int error; 554 555 sc = device_get_softc(dev); 556 cn = &sc->vtpci_common; 557 558 /* 559 * Redrive the device initialization. This is a bit of an abuse of 560 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 561 * play nice. 562 * 563 * We do not allow the host device to change from what was originally 564 * negotiated beyond what the guest driver changed. MSIX state should 565 * not change, number of virtqueues and their size remain the same, etc. 566 * This will need to be rethought when we want to support migration. 567 */ 568 569 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 570 vtpci_modern_stop(dev); 571 572 /* 573 * Quickly drive the status through ACK and DRIVER. The device does 574 * not become usable again until DRIVER_OK in reinit complete. 575 */ 576 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 577 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 578 579 /* 580 * TODO: Check that features are not added as to what was 581 * originally negotiated. 582 */ 583 vtpci_modern_negotiate_features(dev, features); 584 error = vtpci_modern_finalize_features(dev); 585 if (error) { 586 device_printf(dev, "cannot finalize features during reinit\n"); 587 return (error); 588 } 589 590 error = vtpci_reinit(cn); 591 if (error) 592 return (error); 593 594 return (0); 595} 596 597static void 598vtpci_modern_reinit_complete(device_t dev) 599{ 600 struct vtpci_modern_softc *sc; 601 602 sc = device_get_softc(dev); 603 604 vtpci_modern_enable_virtqueues(sc); 605 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 606} 607 608static void 609vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 610{ 611 struct vtpci_modern_softc *sc; 612 613 sc = device_get_softc(dev); 614 615 vtpci_modern_write_notify_2(sc, offset, queue); 616} 617 618static uint8_t 619vtpci_modern_get_status(struct vtpci_modern_softc *sc) 620{ 621 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 622} 623 624static void 625vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 626{ 627 if (status != VIRTIO_CONFIG_STATUS_RESET) 628 status |= vtpci_modern_get_status(sc); 629 630 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 631} 632 633static int 634vtpci_modern_config_generation(device_t dev) 635{ 636 struct vtpci_modern_softc *sc; 637 uint8_t gen; 638 639 sc = device_get_softc(dev); 640 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 641 642 return (gen); 643} 644 645static void 646vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 647 int length) 648{ 649 struct vtpci_modern_softc *sc; 650 651 sc = device_get_softc(dev); 652 653 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 654 panic("%s: attempt to read dev config but not present", 655 __func__); 656 } 657 658 switch (length) { 659 case 1: 660 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 661 break; 662 case 2: 663 *(uint16_t *) dst = virtio_htog16(true, 664 vtpci_modern_read_device_2(sc, offset)); 665 break; 666 case 4: 667 *(uint32_t *) dst = virtio_htog32(true, 668 vtpci_modern_read_device_4(sc, offset)); 669 break; 670 case 8: 671 *(uint64_t *) dst = virtio_htog64(true, 672 vtpci_modern_read_device_8(sc, offset)); 673 break; 674 default: 675 panic("%s: device %s invalid device read length %d offset %d", 676 __func__, device_get_nameunit(dev), length, (int) offset); 677 } 678} 679 680static void 681vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 682 int length) 683{ 684 struct vtpci_modern_softc *sc; 685 686 sc = device_get_softc(dev); 687 688 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 689 panic("%s: attempt to write dev config but not present", 690 __func__); 691 } 692 693 switch (length) { 694 case 1: 695 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 696 break; 697 case 2: { 698 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 699 vtpci_modern_write_device_2(sc, offset, val); 700 break; 701 } 702 case 4: { 703 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 704 vtpci_modern_write_device_4(sc, offset, val); 705 break; 706 } 707 case 8: { 708 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 709 vtpci_modern_write_device_8(sc, offset, val); 710 break; 711 } 712 default: 713 panic("%s: device %s invalid device write length %d offset %d", 714 __func__, device_get_nameunit(dev), length, (int) offset); 715 } 716} 717 718static int 719vtpci_modern_probe_configs(device_t dev) 720{ 721 int error; 722 723 /* 724 * These config capabilities must be present. The DEVICE_CFG 725 * capability is only present if the device requires it. 726 */ 727 728 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 729 if (error) { 730 device_printf(dev, "cannot find COMMON_CFG capability\n"); 731 return (error); 732 } 733 734 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 735 if (error) { 736 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 737 return (error); 738 } 739 740 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 741 if (error) { 742 device_printf(dev, "cannot find ISR_CFG capability\n"); 743 return (error); 744 } 745 746 return (0); 747} 748 749static int 750vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 751{ 752 uint32_t type, bar; 753 int capreg, error; 754 755 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 756 error == 0; 757 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 758 759 type = pci_read_config(dev, capreg + 760 offsetof(struct virtio_pci_cap, cfg_type), 1); 761 bar = pci_read_config(dev, capreg + 762 offsetof(struct virtio_pci_cap, bar), 1); 763 764 /* Must ignore reserved BARs. */ 765 if (bar >= VTPCI_MODERN_MAX_BARS) 766 continue; 767 768 if (type == cfg_type) { 769 if (cap_offset != NULL) 770 *cap_offset = capreg; 771 break; 772 } 773 } 774 775 return (error); 776} 777 778static int 779vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 780{ 781 device_t dev; 782 int error; 783 784 dev = sc->vtpci_dev; 785 786 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 787 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 788 if (error) { 789 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 790 return (error); 791 } 792 793 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 794 if (error) { 795 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 796 return (error); 797 } 798 799 return (0); 800} 801 802static int 803vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 804{ 805 device_t dev; 806 int cap_offset, error; 807 808 dev = sc->vtpci_dev; 809 810 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 811 -1, 2, &sc->vtpci_notify_res_map); 812 if (error) { 813 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 814 return (error); 815 } 816 817 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 818 819 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 820 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 821 822 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 823 if (error) { 824 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 825 return (error); 826 } 827 828 return (0); 829} 830 831static int 832vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 833{ 834 device_t dev; 835 int error; 836 837 dev = sc->vtpci_dev; 838 839 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 840 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 841 if (error) { 842 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 843 return (error); 844 } 845 846 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 847 if (error) { 848 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 849 return (error); 850 } 851 852 return (0); 853} 854 855static int 856vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 857{ 858 device_t dev; 859 int error; 860 861 dev = sc->vtpci_dev; 862 863 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 864 -1, 4, &sc->vtpci_device_res_map); 865 if (error == ENOENT) { 866 /* Device configuration is optional depending on device. */ 867 return (0); 868 } else if (error) { 869 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 870 return (error); 871 } 872 873 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 874 if (error) { 875 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 876 return (error); 877 } 878 879 return (0); 880} 881 882static int 883vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 884{ 885 int error; 886 887 error = vtpci_modern_map_common_config(sc); 888 if (error) 889 return (error); 890 891 error = vtpci_modern_map_notify_config(sc); 892 if (error) 893 return (error); 894 895 error = vtpci_modern_map_isr_config(sc); 896 if (error) 897 return (error); 898 899 error = vtpci_modern_map_device_config(sc); 900 if (error) 901 return (error); 902 903 vtpci_modern_alloc_msix_resource(sc); 904 905 return (0); 906} 907 908static void 909vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 910{ 911 912 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 913 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 914 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 915 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 916 917 vtpci_modern_free_bar_resources(sc); 918 vtpci_modern_free_msix_resource(sc); 919 920 sc->vtpci_notify_offset_multiplier = 0; 921} 922 923static int 924vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 925 int min_size, int alignment, struct vtpci_modern_resource_map *res) 926{ 927 device_t dev; 928 int cap_offset, offset, length, error; 929 uint8_t bar, cap_length; 930 931 dev = sc->vtpci_dev; 932 933 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 934 if (error) 935 return (error); 936 937 cap_length = pci_read_config(dev, 938 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 939 940 if (cap_length < sizeof(struct virtio_pci_cap)) { 941 device_printf(dev, "cap %u length %d less than expected\n", 942 cfg_type, cap_length); 943 return (ENXIO); 944 } 945 946 bar = pci_read_config(dev, 947 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 948 offset = pci_read_config(dev, 949 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 950 length = pci_read_config(dev, 951 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 952 953 if (min_size != -1 && length < min_size) { 954 device_printf(dev, "cap %u struct length %d less than min %d\n", 955 cfg_type, length, min_size); 956 return (ENXIO); 957 } 958 959 if (offset % alignment) { 960 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 961 cfg_type, offset, alignment); 962 return (ENXIO); 963 } 964 965 /* BMV: TODO Can we determine the size of the BAR here? */ 966 967 res->vtrm_cap_offset = cap_offset; 968 res->vtrm_bar = bar; 969 res->vtrm_offset = offset; 970 res->vtrm_length = length; 971 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 972 973 return (0); 974} 975 976static int 977vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 978{ 979 uint32_t val; 980 981 /* 982 * The BAR described by a config capability may be either an IOPORT or 983 * MEM, but we must know the type when calling bus_alloc_resource(). 984 */ 985 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 986 if (PCI_BAR_IO(val)) 987 return (SYS_RES_IOPORT); 988 else 989 return (SYS_RES_MEMORY); 990} 991 992static struct resource * 993vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 994{ 995 struct resource *res; 996 997 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 998 res = sc->vtpci_bar_res[bar].vtbr_res; 999 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1000 1001 return (res); 1002} 1003 1004static struct resource * 1005vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1006 int type) 1007{ 1008 struct resource *res; 1009 int rid; 1010 1011 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1012 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1013 1014 res = sc->vtpci_bar_res[bar].vtbr_res; 1015 if (res != NULL) { 1016 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1017 return (res); 1018 } 1019 1020 rid = PCIR_BAR(bar); 1021 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1022 RF_ACTIVE | RF_UNMAPPED); 1023 if (res != NULL) { 1024 sc->vtpci_bar_res[bar].vtbr_res = res; 1025 sc->vtpci_bar_res[bar].vtbr_type = type; 1026 } 1027 1028 return (res); 1029} 1030 1031static void 1032vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1033{ 1034 device_t dev; 1035 struct resource *res; 1036 int bar, rid, type; 1037 1038 dev = sc->vtpci_dev; 1039 1040 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1041 res = sc->vtpci_bar_res[bar].vtbr_res; 1042 type = sc->vtpci_bar_res[bar].vtbr_type; 1043 1044 if (res != NULL) { 1045 rid = PCIR_BAR(bar); 1046 bus_release_resource(dev, type, rid, res); 1047 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1048 sc->vtpci_bar_res[bar].vtbr_type = 0; 1049 } 1050 } 1051} 1052 1053static int 1054vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1055 struct vtpci_modern_resource_map *map) 1056{ 1057 struct resource_map_request req; 1058 struct resource *res; 1059 int type; 1060 1061 type = map->vtrm_type; 1062 1063 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1064 if (res == NULL) 1065 return (ENXIO); 1066 1067 resource_init_map_request(&req); 1068 req.offset = map->vtrm_offset; 1069 req.length = map->vtrm_length; 1070 1071 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1072 &map->vtrm_map)); 1073} 1074 1075static void 1076vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1077 struct vtpci_modern_resource_map *map) 1078{ 1079 struct resource *res; 1080 int type; 1081 1082 type = map->vtrm_type; 1083 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1084 1085 if (res != NULL && map->vtrm_map.r_size != 0) { 1086 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1087 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1088 } 1089} 1090 1091static void 1092vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1093{ 1094 device_t dev; 1095 int bar; 1096 1097 dev = sc->vtpci_dev; 1098 1099 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1100 (bar = pci_msix_table_bar(dev)) == -1) 1101 return; 1102 1103 /* TODO: Can this BAR be in the 0-5 range? */ 1104 sc->vtpci_msix_bar = bar; 1105 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1106 &bar, RF_ACTIVE)) == NULL) 1107 device_printf(dev, "Unable to map MSIX table\n"); 1108} 1109 1110static void 1111vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1112{ 1113 device_t dev; 1114 1115 dev = sc->vtpci_dev; 1116 1117 if (sc->vtpci_msix_res != NULL) { 1118 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1119 sc->vtpci_msix_res); 1120 sc->vtpci_msix_bar = 0; 1121 sc->vtpci_msix_res = NULL; 1122 } 1123} 1124 1125static void 1126vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1127{ 1128 device_t dev, child; 1129 1130 dev = sc->vtpci_dev; 1131 child = vtpci_child_device(&sc->vtpci_common); 1132 1133 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1134 return; 1135 1136 if (device_probe(child) != 0) 1137 return; 1138 1139 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1140 1141 if (device_attach(child) != 0) { 1142 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1143 /* Reset state for later attempt. */ 1144 vtpci_modern_child_detached(dev, child); 1145 } else { 1146 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1147 VIRTIO_ATTACH_COMPLETED(child); 1148 } 1149} 1150 1151static int 1152vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1153 struct vtpci_interrupt *intr) 1154{ 1155 uint16_t vector; 1156 1157 if (intr != NULL) { 1158 /* Map from guest rid to host vector. */ 1159 vector = intr->vti_rid - 1; 1160 } else 1161 vector = VIRTIO_MSI_NO_VECTOR; 1162 1163 vtpci_modern_write_common_2(sc, offset, vector); 1164 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1165} 1166 1167static int 1168vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1169{ 1170 struct vtpci_modern_softc *sc; 1171 int error; 1172 1173 sc = device_get_softc(dev); 1174 1175 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1176 if (error) { 1177 device_printf(dev, 1178 "unable to register config MSIX interrupt\n"); 1179 return (error); 1180 } 1181 1182 return (0); 1183} 1184 1185static int 1186vtpci_modern_register_vq_msix(device_t dev, int idx, 1187 struct vtpci_interrupt *intr) 1188{ 1189 struct vtpci_modern_softc *sc; 1190 int error; 1191 1192 sc = device_get_softc(dev); 1193 1194 vtpci_modern_select_virtqueue(sc, idx); 1195 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1196 if (error) { 1197 device_printf(dev, 1198 "unable to register virtqueue MSIX interrupt\n"); 1199 return (error); 1200 } 1201 1202 return (0); 1203} 1204 1205static void 1206vtpci_modern_reset(struct vtpci_modern_softc *sc) 1207{ 1208 /* 1209 * Setting the status to RESET sets the host device to the 1210 * original, uninitialized state. Must poll the status until 1211 * the reset is complete. 1212 */ 1213 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1214 1215 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1216 cpu_spinwait(); 1217} 1218 1219static void 1220vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1221{ 1222 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1223} 1224 1225static uint8_t 1226vtpci_modern_read_isr(device_t dev) 1227{ 1228 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1229} 1230 1231static uint16_t 1232vtpci_modern_get_vq_size(device_t dev, int idx) 1233{ 1234 struct vtpci_modern_softc *sc; 1235 1236 sc = device_get_softc(dev); 1237 1238 vtpci_modern_select_virtqueue(sc, idx); 1239 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1240} 1241 1242static bus_size_t 1243vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1244{ 1245 struct vtpci_modern_softc *sc; 1246 uint16_t q_notify_off; 1247 1248 sc = device_get_softc(dev); 1249 1250 vtpci_modern_select_virtqueue(sc, idx); 1251 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1252 1253 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1254} 1255 1256static void 1257vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1258{ 1259 struct vtpci_modern_softc *sc; 1260 1261 sc = device_get_softc(dev); 1262 1263 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1264 1265 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1266 vtpci_modern_write_common_2(sc, 1267 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1268 1269 vtpci_modern_write_common_8(sc, 1270 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1271 vtpci_modern_write_common_8(sc, 1272 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1273 vtpci_modern_write_common_8(sc, 1274 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1275} 1276 1277static void 1278vtpci_modern_disable_vq(device_t dev, int idx) 1279{ 1280 struct vtpci_modern_softc *sc; 1281 1282 sc = device_get_softc(dev); 1283 1284 vtpci_modern_select_virtqueue(sc, idx); 1285 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1286 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1287 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1288} 1289 1290static void 1291vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1292{ 1293 int idx; 1294 1295 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1296 vtpci_modern_select_virtqueue(sc, idx); 1297 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1298 } 1299} 1300 1301static uint8_t 1302vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1303{ 1304 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1305} 1306 1307static uint16_t 1308vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1309{ 1310 return virtio_htog16(true, 1311 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1312} 1313 1314static uint32_t 1315vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1316{ 1317 return virtio_htog32(true, 1318 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1319} 1320 1321static void 1322vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1323 uint8_t val) 1324{ 1325 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1326} 1327 1328static void 1329vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1330 uint16_t val) 1331{ 1332 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1333 off, virtio_gtoh16(true, val)); 1334} 1335 1336static void 1337vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1338 uint32_t val) 1339{ 1340 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1341 off, virtio_gtoh32(true, val)); 1342} 1343 1344static void 1345vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1346 uint64_t val) 1347{ 1348 uint32_t val0, val1; 1349 1350 val0 = (uint32_t) val; 1351 val1 = val >> 32; 1352 1353 vtpci_modern_write_common_4(sc, off, val0); 1354 vtpci_modern_write_common_4(sc, off + 4, val1); 1355} 1356 1357static void 1358vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1359 uint16_t val) 1360{ 1361 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1362} 1363 1364static uint8_t 1365vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1366{ 1367 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1368} 1369 1370static uint8_t 1371vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1372{ 1373 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1374} 1375 1376static uint16_t 1377vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1378{ 1379 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1380} 1381 1382static uint32_t 1383vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1384{ 1385 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1386} 1387 1388static uint64_t 1389vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1390{ 1391 device_t dev; 1392 int gen; 1393 uint32_t val0, val1; 1394 1395 dev = sc->vtpci_dev; 1396 1397 /* 1398 * Treat the 64-bit field as two 32-bit fields. Use the generation 1399 * to ensure a consistent read. 1400 */ 1401 do { 1402 gen = vtpci_modern_config_generation(dev); 1403 val0 = vtpci_modern_read_device_4(sc, off); 1404 val1 = vtpci_modern_read_device_4(sc, off + 4); 1405 } while (gen != vtpci_modern_config_generation(dev)); 1406 1407 return (((uint64_t) val1 << 32) | val0); 1408} 1409 1410static void 1411vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1412 uint8_t val) 1413{ 1414 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1415} 1416 1417static void 1418vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1419 uint16_t val) 1420{ 1421 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1422} 1423 1424static void 1425vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1426 uint32_t val) 1427{ 1428 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1429} 1430 1431static void 1432vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1433 uint64_t val) 1434{ 1435 uint32_t val0, val1; 1436 1437 val0 = (uint32_t) val; 1438 val1 = val >> 32; 1439 1440 vtpci_modern_write_device_4(sc, off, val0); 1441 vtpci_modern_write_device_4(sc, off + 4, val1); 1442} 1443