1/*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35/* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40/* 41 * FDT example: 42 * virtio_block@1000 { 43 * compatible = "virtio,mmio"; 44 * reg = <0x1000 0x100>; 45 * interrupts = <63>; 46 * interrupt-parent = <&GIC>; 47 * }; 48 */ 49 50#include <sys/cdefs.h> 51__FBSDID("$FreeBSD$"); 52 53#include <sys/param.h> 54#include <sys/systm.h> 55#include <sys/bus.h> 56#include <sys/kernel.h> 57#include <sys/module.h> 58#include <sys/malloc.h> 59#include <sys/rman.h> 60 61#include <machine/bus.h> 62#include <machine/resource.h> 63 64#include <dev/fdt/fdt_common.h> 65#include <dev/ofw/openfirm.h> 66#include <dev/ofw/ofw_bus.h> 67#include <dev/ofw/ofw_bus_subr.h> 68 69#include <dev/virtio/virtio.h> 70#include <dev/virtio/virtqueue.h> 71#include <dev/virtio/mmio/virtio_mmio.h> 72 73#include "virtio_mmio_if.h" 74#include "virtio_bus_if.h" 75#include "virtio_if.h" 76 77#define PAGE_SHIFT 12 78 79struct vtmmio_virtqueue { 80 struct virtqueue *vtv_vq; 81 int vtv_no_intr; 82}; 83 84struct vtmmio_softc { 85 device_t dev; 86 device_t platform; 87 struct resource *res[2]; 88 89 uint64_t vtmmio_features; 90 uint32_t vtmmio_flags; 91 92 /* This "bus" will only ever have one child. */ 93 device_t vtmmio_child_dev; 94 struct virtio_feature_desc *vtmmio_child_feat_desc; 95 96 int vtmmio_nvqs; 97 struct vtmmio_virtqueue *vtmmio_vqs; 98 void *ih; 99}; 100 101static int vtmmio_probe(device_t); 102static int vtmmio_attach(device_t); 103static int vtmmio_detach(device_t); 104static int vtmmio_suspend(device_t); 105static int vtmmio_resume(device_t); 106static int vtmmio_shutdown(device_t); 107static void vtmmio_driver_added(device_t, driver_t *); 108static void vtmmio_child_detached(device_t, device_t); 109static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 110static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 111static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 112static int vtmmio_with_feature(device_t, uint64_t); 113static int vtmmio_alloc_virtqueues(device_t, int, int, 114 struct vq_alloc_info *); 115static int vtmmio_setup_intr(device_t, enum intr_type); 116static void vtmmio_stop(device_t); 117static void vtmmio_poll(device_t); 118static int vtmmio_reinit(device_t, uint64_t); 119static void vtmmio_reinit_complete(device_t); 120static void vtmmio_notify_virtqueue(device_t, uint16_t); 121static uint8_t vtmmio_get_status(device_t); 122static void vtmmio_set_status(device_t, uint8_t); 123static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 124static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 125static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 126 uint64_t); 127static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 128static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 129static void vtmmio_free_interrupts(struct vtmmio_softc *); 130static void vtmmio_free_virtqueues(struct vtmmio_softc *); 131static void vtmmio_release_child_resources(struct vtmmio_softc *); 132static void vtmmio_reset(struct vtmmio_softc *); 133static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 134static void vtmmio_vq_intr(void *); 135 136/* 137 * I/O port read/write wrappers. 138 */ 139#define vtmmio_write_config_1(sc, o, v) \ 140do { \ 141 if (sc->platform != NULL) \ 142 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 143 bus_write_1((sc)->res[0], (o), (v)); \ 144 if (sc->platform != NULL) \ 145 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 146} while (0) 147#define vtmmio_write_config_2(sc, o, v) \ 148do { \ 149 if (sc->platform != NULL) \ 150 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 151 bus_write_2((sc)->res[0], (o), (v)); \ 152 if (sc->platform != NULL) \ 153 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 154} while (0) 155#define vtmmio_write_config_4(sc, o, v) \ 156do { \ 157 if (sc->platform != NULL) \ 158 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 159 bus_write_4((sc)->res[0], (o), (v)); \ 160 if (sc->platform != NULL) \ 161 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 162} while (0) 163 164#define vtmmio_read_config_1(sc, o) \ 165 bus_read_1((sc)->res[0], (o)) 166#define vtmmio_read_config_2(sc, o) \ 167 bus_read_2((sc)->res[0], (o)) 168#define vtmmio_read_config_4(sc, o) \ 169 bus_read_4((sc)->res[0], (o)) 170 171static device_method_t vtmmio_methods[] = { 172 /* Device interface. */ 173 DEVMETHOD(device_probe, vtmmio_probe), 174 DEVMETHOD(device_attach, vtmmio_attach), 175 DEVMETHOD(device_detach, vtmmio_detach), 176 DEVMETHOD(device_suspend, vtmmio_suspend), 177 DEVMETHOD(device_resume, vtmmio_resume), 178 DEVMETHOD(device_shutdown, vtmmio_shutdown), 179 180 /* Bus interface. */ 181 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 182 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 183 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 184 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 185 186 /* VirtIO bus interface. */ 187 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 188 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 189 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 190 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 191 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 192 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 193 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 194 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 195 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 196 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 197 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 198 199 DEVMETHOD_END 200}; 201 202static driver_t vtmmio_driver = { 203 "virtio_mmio", 204 vtmmio_methods, 205 sizeof(struct vtmmio_softc) 206}; 207 208devclass_t vtmmio_devclass; 209 210DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0); 211DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0); 212MODULE_VERSION(virtio_mmio, 1); 213MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1); 214MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1); 215 216static int 217vtmmio_setup_intr(device_t dev, enum intr_type type) 218{ 219 struct vtmmio_softc *sc; 220 int rid; 221 int err; 222 223 sc = device_get_softc(dev); 224 225 if (sc->platform != NULL) { 226 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 227 vtmmio_vq_intr, sc); 228 if (err == 0) { 229 /* Okay we have backend-specific interrupts */ 230 return (0); 231 } 232 } 233 234 rid = 0; 235 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 236 RF_ACTIVE); 237 if (!sc->res[1]) { 238 device_printf(dev, "Can't allocate interrupt\n"); 239 return (ENXIO); 240 } 241 242 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 243 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 244 device_printf(dev, "Can't setup the interrupt\n"); 245 return (ENXIO); 246 } 247 248 return (0); 249} 250 251static int 252vtmmio_probe(device_t dev) 253{ 254 255 if (!ofw_bus_status_okay(dev)) 256 return (ENXIO); 257 258 if (!ofw_bus_is_compatible(dev, "virtio,mmio")) 259 return (ENXIO); 260 261 device_set_desc(dev, "VirtIO MMIO adapter"); 262 return (BUS_PROBE_DEFAULT); 263} 264 265static int 266vtmmio_setup_platform(struct vtmmio_softc *sc) 267{ 268 phandle_t platform_node; 269 struct fdt_ic *ic; 270 phandle_t xref; 271 phandle_t node; 272 273 sc->platform = NULL; 274 275 if ((node = ofw_bus_get_node(sc->dev)) == -1) 276 return (ENXIO); 277 278 if (OF_searchencprop(node, "platform", &xref, 279 sizeof(xref)) == -1) { 280 return (ENXIO); 281 } 282 283 platform_node = OF_node_from_xref(xref); 284 285 SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) { 286 if (ic->iph == platform_node) { 287 sc->platform = ic->dev; 288 break; 289 } 290 } 291 292 if (sc->platform == NULL) { 293 /* No platform-specific device. Ignore it. */ 294 } 295 296 return (0); 297} 298 299static int 300vtmmio_attach(device_t dev) 301{ 302 struct vtmmio_softc *sc; 303 device_t child; 304 int rid; 305 306 sc = device_get_softc(dev); 307 sc->dev = dev; 308 309 vtmmio_setup_platform(sc); 310 311 rid = 0; 312 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 313 RF_ACTIVE); 314 if (!sc->res[0]) { 315 device_printf(dev, "Cannot allocate memory window.\n"); 316 return (ENXIO); 317 } 318 319 vtmmio_reset(sc); 320 321 /* Tell the host we've noticed this device. */ 322 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 323 324 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 325 device_printf(dev, "Cannot create child device.\n"); 326 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 327 vtmmio_detach(dev); 328 return (ENOMEM); 329 } 330 331 sc->vtmmio_child_dev = child; 332 vtmmio_probe_and_attach_child(sc); 333 334 return (0); 335} 336 337static int 338vtmmio_detach(device_t dev) 339{ 340 struct vtmmio_softc *sc; 341 device_t child; 342 int error; 343 344 sc = device_get_softc(dev); 345 346 if ((child = sc->vtmmio_child_dev) != NULL) { 347 error = device_delete_child(dev, child); 348 if (error) 349 return (error); 350 sc->vtmmio_child_dev = NULL; 351 } 352 353 vtmmio_reset(sc); 354 355 if (sc->res[0] != NULL) { 356 bus_release_resource(dev, SYS_RES_MEMORY, 0, 357 sc->res[0]); 358 sc->res[0] = NULL; 359 } 360 361 return (0); 362} 363 364static int 365vtmmio_suspend(device_t dev) 366{ 367 368 return (bus_generic_suspend(dev)); 369} 370 371static int 372vtmmio_resume(device_t dev) 373{ 374 375 return (bus_generic_resume(dev)); 376} 377 378static int 379vtmmio_shutdown(device_t dev) 380{ 381 382 (void) bus_generic_shutdown(dev); 383 384 /* Forcibly stop the host device. */ 385 vtmmio_stop(dev); 386 387 return (0); 388} 389 390static void 391vtmmio_driver_added(device_t dev, driver_t *driver) 392{ 393 struct vtmmio_softc *sc; 394 395 sc = device_get_softc(dev); 396 397 vtmmio_probe_and_attach_child(sc); 398} 399 400static void 401vtmmio_child_detached(device_t dev, device_t child) 402{ 403 struct vtmmio_softc *sc; 404 405 sc = device_get_softc(dev); 406 407 vtmmio_reset(sc); 408 vtmmio_release_child_resources(sc); 409} 410 411static int 412vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 413{ 414 struct vtmmio_softc *sc; 415 416 sc = device_get_softc(dev); 417 418 if (sc->vtmmio_child_dev != child) 419 return (ENOENT); 420 421 switch (index) { 422 case VIRTIO_IVAR_DEVTYPE: 423 case VIRTIO_IVAR_SUBDEVICE: 424 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 425 break; 426 case VIRTIO_IVAR_VENDOR: 427 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 428 break; 429 default: 430 return (ENOENT); 431 } 432 433 return (0); 434} 435 436static int 437vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 438{ 439 struct vtmmio_softc *sc; 440 441 sc = device_get_softc(dev); 442 443 if (sc->vtmmio_child_dev != child) 444 return (ENOENT); 445 446 switch (index) { 447 case VIRTIO_IVAR_FEATURE_DESC: 448 sc->vtmmio_child_feat_desc = (void *) value; 449 break; 450 default: 451 return (ENOENT); 452 } 453 454 return (0); 455} 456 457static uint64_t 458vtmmio_negotiate_features(device_t dev, uint64_t child_features) 459{ 460 struct vtmmio_softc *sc; 461 uint64_t host_features, features; 462 463 sc = device_get_softc(dev); 464 465 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 466 vtmmio_describe_features(sc, "host", host_features); 467 468 /* 469 * Limit negotiated features to what the driver, virtqueue, and 470 * host all support. 471 */ 472 features = host_features & child_features; 473 features = virtqueue_filter_features(features); 474 sc->vtmmio_features = features; 475 476 vtmmio_describe_features(sc, "negotiated", features); 477 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 478 479 return (features); 480} 481 482static int 483vtmmio_with_feature(device_t dev, uint64_t feature) 484{ 485 struct vtmmio_softc *sc; 486 487 sc = device_get_softc(dev); 488 489 return ((sc->vtmmio_features & feature) != 0); 490} 491 492static int 493vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 494 struct vq_alloc_info *vq_info) 495{ 496 struct vtmmio_virtqueue *vqx; 497 struct vq_alloc_info *info; 498 struct vtmmio_softc *sc; 499 struct virtqueue *vq; 500 uint32_t size; 501 int idx, error; 502 503 sc = device_get_softc(dev); 504 505 if (sc->vtmmio_nvqs != 0) 506 return (EALREADY); 507 if (nvqs <= 0) 508 return (EINVAL); 509 510 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 511 M_DEVBUF, M_NOWAIT | M_ZERO); 512 if (sc->vtmmio_vqs == NULL) 513 return (ENOMEM); 514 515 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 516 (1 << PAGE_SHIFT)); 517 518 for (idx = 0; idx < nvqs; idx++) { 519 vqx = &sc->vtmmio_vqs[idx]; 520 info = &vq_info[idx]; 521 522 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 523 524 vtmmio_select_virtqueue(sc, idx); 525 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 526 527 error = virtqueue_alloc(dev, idx, size, 528 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq); 529 if (error) { 530 device_printf(dev, 531 "cannot allocate virtqueue %d: %d\n", 532 idx, error); 533 break; 534 } 535 536 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 537 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 538 VIRTIO_MMIO_VRING_ALIGN); 539#if 0 540 device_printf(dev, "virtqueue paddr 0x%08lx\n", 541 (uint64_t)virtqueue_paddr(vq)); 542#endif 543 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 544 virtqueue_paddr(vq) >> PAGE_SHIFT); 545 546 vqx->vtv_vq = *info->vqai_vq = vq; 547 vqx->vtv_no_intr = info->vqai_intr == NULL; 548 549 sc->vtmmio_nvqs++; 550 } 551 552 if (error) 553 vtmmio_free_virtqueues(sc); 554 555 return (error); 556} 557 558static void 559vtmmio_stop(device_t dev) 560{ 561 562 vtmmio_reset(device_get_softc(dev)); 563} 564 565static void 566vtmmio_poll(device_t dev) 567{ 568 struct vtmmio_softc *sc; 569 570 sc = device_get_softc(dev); 571 572 if (sc->platform != NULL) 573 VIRTIO_MMIO_POLL(sc->platform); 574} 575 576static int 577vtmmio_reinit(device_t dev, uint64_t features) 578{ 579 struct vtmmio_softc *sc; 580 int idx, error; 581 582 sc = device_get_softc(dev); 583 584 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 585 vtmmio_stop(dev); 586 587 /* 588 * Quickly drive the status through ACK and DRIVER. The device 589 * does not become usable again until vtmmio_reinit_complete(). 590 */ 591 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 592 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 593 594 vtmmio_negotiate_features(dev, features); 595 596 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 597 (1 << PAGE_SHIFT)); 598 599 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 600 error = vtmmio_reinit_virtqueue(sc, idx); 601 if (error) 602 return (error); 603 } 604 605 return (0); 606} 607 608static void 609vtmmio_reinit_complete(device_t dev) 610{ 611 612 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 613} 614 615static void 616vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 617{ 618 struct vtmmio_softc *sc; 619 620 sc = device_get_softc(dev); 621 622 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 623} 624 625static uint8_t 626vtmmio_get_status(device_t dev) 627{ 628 struct vtmmio_softc *sc; 629 630 sc = device_get_softc(dev); 631 632 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 633} 634 635static void 636vtmmio_set_status(device_t dev, uint8_t status) 637{ 638 struct vtmmio_softc *sc; 639 640 sc = device_get_softc(dev); 641 642 if (status != VIRTIO_CONFIG_STATUS_RESET) 643 status |= vtmmio_get_status(dev); 644 645 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 646} 647 648static void 649vtmmio_read_dev_config(device_t dev, bus_size_t offset, 650 void *dst, int length) 651{ 652 struct vtmmio_softc *sc; 653 bus_size_t off; 654 uint8_t *d; 655 int size; 656 657 sc = device_get_softc(dev); 658 off = VIRTIO_MMIO_CONFIG + offset; 659 660 for (d = dst; length > 0; d += size, off += size, length -= size) { 661#ifdef ALLOW_WORD_ALIGNED_ACCESS 662 if (length >= 4) { 663 size = 4; 664 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 665 } else if (length >= 2) { 666 size = 2; 667 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 668 } else 669#endif 670 { 671 size = 1; 672 *d = vtmmio_read_config_1(sc, off); 673 } 674 } 675} 676 677static void 678vtmmio_write_dev_config(device_t dev, bus_size_t offset, 679 void *src, int length) 680{ 681 struct vtmmio_softc *sc; 682 bus_size_t off; 683 uint8_t *s; 684 int size; 685 686 sc = device_get_softc(dev); 687 off = VIRTIO_MMIO_CONFIG + offset; 688 689 for (s = src; length > 0; s += size, off += size, length -= size) { 690#ifdef ALLOW_WORD_ALIGNED_ACCESS 691 if (length >= 4) { 692 size = 4; 693 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 694 } else if (length >= 2) { 695 size = 2; 696 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 697 } else 698#endif 699 { 700 size = 1; 701 vtmmio_write_config_1(sc, off, *s); 702 } 703 } 704} 705 706static void 707vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 708 uint64_t features) 709{ 710 device_t dev, child; 711 712 dev = sc->dev; 713 child = sc->vtmmio_child_dev; 714 715 if (device_is_attached(child) || bootverbose == 0) 716 return; 717 718 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 719} 720 721static void 722vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 723{ 724 device_t dev, child; 725 726 dev = sc->dev; 727 child = sc->vtmmio_child_dev; 728 729 if (child == NULL) 730 return; 731 732 if (device_get_state(child) != DS_NOTPRESENT) { 733 return; 734 } 735 736 if (device_probe(child) != 0) { 737 return; 738 } 739 740 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 741 if (device_attach(child) != 0) { 742 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 743 vtmmio_reset(sc); 744 vtmmio_release_child_resources(sc); 745 /* Reset status for future attempt. */ 746 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 747 } else { 748 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 749 VIRTIO_ATTACH_COMPLETED(child); 750 } 751} 752 753static int 754vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 755{ 756 struct vtmmio_virtqueue *vqx; 757 struct virtqueue *vq; 758 int error; 759 uint16_t size; 760 761 vqx = &sc->vtmmio_vqs[idx]; 762 vq = vqx->vtv_vq; 763 764 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 765 766 vtmmio_select_virtqueue(sc, idx); 767 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 768 769 error = virtqueue_reinit(vq, size); 770 if (error) 771 return (error); 772 773 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 774 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 775 VIRTIO_MMIO_VRING_ALIGN); 776#if 0 777 device_printf(sc->dev, "virtqueue paddr 0x%08lx\n", 778 (uint64_t)virtqueue_paddr(vq)); 779#endif 780 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 781 virtqueue_paddr(vq) >> PAGE_SHIFT); 782 783 return (0); 784} 785 786static void 787vtmmio_free_interrupts(struct vtmmio_softc *sc) 788{ 789 790 if (sc->ih != NULL) 791 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 792 793 if (sc->res[1] != NULL) 794 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 795} 796 797static void 798vtmmio_free_virtqueues(struct vtmmio_softc *sc) 799{ 800 struct vtmmio_virtqueue *vqx; 801 int idx; 802 803 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 804 vqx = &sc->vtmmio_vqs[idx]; 805 806 vtmmio_select_virtqueue(sc, idx); 807 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 808 809 virtqueue_free(vqx->vtv_vq); 810 vqx->vtv_vq = NULL; 811 } 812 813 free(sc->vtmmio_vqs, M_DEVBUF); 814 sc->vtmmio_vqs = NULL; 815 sc->vtmmio_nvqs = 0; 816} 817 818static void 819vtmmio_release_child_resources(struct vtmmio_softc *sc) 820{ 821 822 vtmmio_free_interrupts(sc); 823 vtmmio_free_virtqueues(sc); 824} 825 826static void 827vtmmio_reset(struct vtmmio_softc *sc) 828{ 829 830 /* 831 * Setting the status to RESET sets the host device to 832 * the original, uninitialized state. 833 */ 834 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 835} 836 837static void 838vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 839{ 840 841 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 842} 843 844static void 845vtmmio_vq_intr(void *arg) 846{ 847 struct vtmmio_virtqueue *vqx; 848 struct vtmmio_softc *sc; 849 struct virtqueue *vq; 850 uint32_t status; 851 int idx; 852 853 sc = arg; 854 855 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 856 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 857 858 /* The config changed */ 859 if (status & VIRTIO_MMIO_INT_CONFIG) 860 if (sc->vtmmio_child_dev != NULL) 861 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 862 863 /* Notify all virtqueues. */ 864 if (status & VIRTIO_MMIO_INT_VRING) { 865 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 866 vqx = &sc->vtmmio_vqs[idx]; 867 if (vqx->vtv_no_intr == 0) { 868 vq = vqx->vtv_vq; 869 virtqueue_intr(vq); 870 } 871 } 872 } 873} 874