175/* 176 * I/O port read/write wrappers. 177 */ 178#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 179#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 180#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 181#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 182#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 183#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 184 185/* Tunables. */ 186static int vtpci_disable_msix = 0; 187TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 188 189static device_method_t vtpci_methods[] = { 190 /* Device interface. */ 191 DEVMETHOD(device_probe, vtpci_probe), 192 DEVMETHOD(device_attach, vtpci_attach), 193 DEVMETHOD(device_detach, vtpci_detach), 194 DEVMETHOD(device_suspend, vtpci_suspend), 195 DEVMETHOD(device_resume, vtpci_resume), 196 DEVMETHOD(device_shutdown, vtpci_shutdown), 197 198 /* Bus interface. */ 199 DEVMETHOD(bus_driver_added, vtpci_driver_added), 200 DEVMETHOD(bus_child_detached, vtpci_child_detached), 201 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 202 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 203 204 /* VirtIO bus interface. */ 205 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 206 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 207 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 208 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 209 DEVMETHOD(virtio_bus_stop, vtpci_stop), 210 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 211 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 212 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 213 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 214 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 215 216 DEVMETHOD_END 217}; 218 219static driver_t vtpci_driver = { 220 "virtio_pci", 221 vtpci_methods, 222 sizeof(struct vtpci_softc) 223}; 224 225devclass_t vtpci_devclass; 226 227DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 228MODULE_VERSION(virtio_pci, 1); 229MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 230MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 231 232static int 233vtpci_probe(device_t dev) 234{ 235 char desc[36]; 236 const char *name; 237 238 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 239 return (ENXIO); 240 241 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 242 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 243 return (ENXIO); 244 245 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 246 return (ENXIO); 247 248 name = virtio_device_name(pci_get_subdevice(dev)); 249 if (name == NULL) 250 name = "Unknown"; 251 252 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 253 device_set_desc_copy(dev, desc); 254 255 return (BUS_PROBE_DEFAULT); 256} 257 258static int 259vtpci_attach(device_t dev) 260{ 261 struct vtpci_softc *sc; 262 device_t child; 263 int rid; 264 265 sc = device_get_softc(dev); 266 sc->vtpci_dev = dev; 267 268 pci_enable_busmaster(dev); 269 270 rid = PCIR_BAR(0); 271 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 272 RF_ACTIVE); 273 if (sc->vtpci_res == NULL) { 274 device_printf(dev, "cannot map I/O space\n"); 275 return (ENXIO); 276 } 277 278 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 279 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 280 281 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 282 rid = PCIR_BAR(1); 283 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 284 SYS_RES_MEMORY, &rid, RF_ACTIVE); 285 } 286 287 if (sc->vtpci_msix_res == NULL) 288 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 289 290 vtpci_reset(sc); 291 292 /* Tell the host we've noticed this device. */ 293 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 294 295 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 296 device_printf(dev, "cannot create child device\n"); 297 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 298 vtpci_detach(dev); 299 return (ENOMEM); 300 } 301 302 sc->vtpci_child_dev = child; 303 vtpci_probe_and_attach_child(sc); 304 305 return (0); 306} 307 308static int 309vtpci_detach(device_t dev) 310{ 311 struct vtpci_softc *sc; 312 device_t child; 313 int error; 314 315 sc = device_get_softc(dev); 316 317 if ((child = sc->vtpci_child_dev) != NULL) { 318 error = device_delete_child(dev, child); 319 if (error) 320 return (error); 321 sc->vtpci_child_dev = NULL; 322 } 323 324 vtpci_reset(sc); 325 326 if (sc->vtpci_msix_res != NULL) { 327 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 328 sc->vtpci_msix_res); 329 sc->vtpci_msix_res = NULL; 330 } 331 332 if (sc->vtpci_res != NULL) { 333 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 334 sc->vtpci_res); 335 sc->vtpci_res = NULL; 336 } 337 338 return (0); 339} 340 341static int 342vtpci_suspend(device_t dev) 343{ 344 345 return (bus_generic_suspend(dev)); 346} 347 348static int 349vtpci_resume(device_t dev) 350{ 351 352 return (bus_generic_resume(dev)); 353} 354 355static int 356vtpci_shutdown(device_t dev) 357{ 358 359 (void) bus_generic_shutdown(dev); 360 /* Forcibly stop the host device. */ 361 vtpci_stop(dev); 362 363 return (0); 364} 365 366static void 367vtpci_driver_added(device_t dev, driver_t *driver) 368{ 369 struct vtpci_softc *sc; 370 371 sc = device_get_softc(dev); 372 373 vtpci_probe_and_attach_child(sc); 374} 375 376static void 377vtpci_child_detached(device_t dev, device_t child) 378{ 379 struct vtpci_softc *sc; 380 381 sc = device_get_softc(dev); 382 383 vtpci_reset(sc); 384 vtpci_release_child_resources(sc); 385} 386 387static int 388vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 389{ 390 struct vtpci_softc *sc; 391 392 sc = device_get_softc(dev); 393 394 if (sc->vtpci_child_dev != child) 395 return (ENOENT); 396 397 switch (index) { 398 case VIRTIO_IVAR_DEVTYPE: 399 case VIRTIO_IVAR_SUBDEVICE: 400 *result = pci_get_subdevice(dev); 401 break; 402 case VIRTIO_IVAR_VENDOR: 403 *result = pci_get_vendor(dev); 404 break; 405 case VIRTIO_IVAR_DEVICE: 406 *result = pci_get_device(dev); 407 break; 408 case VIRTIO_IVAR_SUBVENDOR: 409 *result = pci_get_subdevice(dev); 410 break; 411 default: 412 return (ENOENT); 413 } 414 415 return (0); 416} 417 418static int 419vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 420{ 421 struct vtpci_softc *sc; 422 423 sc = device_get_softc(dev); 424 425 if (sc->vtpci_child_dev != child) 426 return (ENOENT); 427 428 switch (index) { 429 case VIRTIO_IVAR_FEATURE_DESC: 430 sc->vtpci_child_feat_desc = (void *) value; 431 break; 432 default: 433 return (ENOENT); 434 } 435 436 return (0); 437} 438 439static uint64_t 440vtpci_negotiate_features(device_t dev, uint64_t child_features) 441{ 442 struct vtpci_softc *sc; 443 uint64_t host_features, features; 444 445 sc = device_get_softc(dev); 446 447 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 448 vtpci_describe_features(sc, "host", host_features); 449 450 /* 451 * Limit negotiated features to what the driver, virtqueue, and 452 * host all support. 453 */ 454 features = host_features & child_features; 455 features = virtqueue_filter_features(features); 456 sc->vtpci_features = features; 457 458 vtpci_describe_features(sc, "negotiated", features); 459 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 460 461 return (features); 462} 463 464static int 465vtpci_with_feature(device_t dev, uint64_t feature) 466{ 467 struct vtpci_softc *sc; 468 469 sc = device_get_softc(dev); 470 471 return ((sc->vtpci_features & feature) != 0); 472} 473 474static int 475vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 476 struct vq_alloc_info *vq_info) 477{ 478 struct vtpci_softc *sc; 479 struct virtqueue *vq; 480 struct vtpci_virtqueue *vqx; 481 struct vq_alloc_info *info; 482 int idx, error; 483 uint16_t size; 484 485 sc = device_get_softc(dev); 486 487 if (sc->vtpci_nvqs != 0) 488 return (EALREADY); 489 if (nvqs <= 0) 490 return (EINVAL); 491 492 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 493 M_DEVBUF, M_NOWAIT | M_ZERO); 494 if (sc->vtpci_vqs == NULL) 495 return (ENOMEM); 496 497 for (idx = 0; idx < nvqs; idx++) { 498 vqx = &sc->vtpci_vqs[idx]; 499 info = &vq_info[idx]; 500 501 vtpci_select_virtqueue(sc, idx); 502 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 503 504 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 505 0xFFFFFFFFUL, info, &vq); 506 if (error) { 507 device_printf(dev, 508 "cannot allocate virtqueue %d: %d\n", idx, error); 509 break; 510 } 511 512 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 513 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 514 515 vqx->vtv_vq = *info->vqai_vq = vq; 516 vqx->vtv_no_intr = info->vqai_intr == NULL; 517 518 sc->vtpci_nvqs++; 519 } 520 521 if (error) 522 vtpci_free_virtqueues(sc); 523 524 return (error); 525} 526 527static int 528vtpci_setup_intr(device_t dev, enum intr_type type) 529{ 530 struct vtpci_softc *sc; 531 int attempt, error; 532 533 sc = device_get_softc(dev); 534 535 for (attempt = 0; attempt < 5; attempt++) { 536 /* 537 * Start with the most desirable interrupt configuration and 538 * fallback towards less desirable ones. 539 */ 540 switch (attempt) { 541 case 0: 542 error = vtpci_alloc_intr_msix_pervq(sc); 543 break; 544 case 1: 545 error = vtpci_alloc_intr_msix_shared(sc); 546 break; 547 case 2: 548 error = vtpci_alloc_intr_msi(sc); 549 break; 550 case 3: 551 error = vtpci_alloc_intr_legacy(sc); 552 break; 553 default: 554 device_printf(dev, 555 "exhausted all interrupt allocation attempts\n"); 556 return (ENXIO); 557 } 558 559 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 560 break; 561 562 vtpci_cleanup_setup_intr_attempt(sc); 563 } 564 565 if (bootverbose) { 566 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 567 device_printf(dev, "using legacy interrupt\n"); 568 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 569 device_printf(dev, "using MSI interrupt\n"); 570 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 571 device_printf(dev, "using shared MSIX interrupts\n"); 572 else 573 device_printf(dev, "using per VQ MSIX interrupts\n"); 574 } 575 576 return (0); 577} 578 579static void 580vtpci_stop(device_t dev) 581{ 582 583 vtpci_reset(device_get_softc(dev)); 584} 585 586static int 587vtpci_reinit(device_t dev, uint64_t features) 588{ 589 struct vtpci_softc *sc; 590 int idx, error; 591 592 sc = device_get_softc(dev); 593 594 /* 595 * Redrive the device initialization. This is a bit of an abuse of 596 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 597 * play nice. 598 * 599 * We do not allow the host device to change from what was originally 600 * negotiated beyond what the guest driver changed. MSIX state should 601 * not change, number of virtqueues and their size remain the same, etc. 602 * This will need to be rethought when we want to support migration. 603 */ 604 605 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 606 vtpci_stop(dev); 607 608 /* 609 * Quickly drive the status through ACK and DRIVER. The device 610 * does not become usable again until vtpci_reinit_complete(). 611 */ 612 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 613 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 614 615 vtpci_negotiate_features(dev, features); 616 617 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 618 error = vtpci_reinit_virtqueue(sc, idx); 619 if (error) 620 return (error); 621 } 622 623 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 624 error = vtpci_set_host_msix_vectors(sc); 625 if (error) 626 return (error); 627 } 628 629 return (0); 630} 631 632static void 633vtpci_reinit_complete(device_t dev) 634{ 635 636 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 637} 638 639static void 640vtpci_notify_virtqueue(device_t dev, uint16_t queue) 641{ 642 struct vtpci_softc *sc; 643 644 sc = device_get_softc(dev); 645 646 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 647} 648 649static uint8_t 650vtpci_get_status(device_t dev) 651{ 652 struct vtpci_softc *sc; 653 654 sc = device_get_softc(dev); 655 656 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 657} 658 659static void 660vtpci_set_status(device_t dev, uint8_t status) 661{ 662 struct vtpci_softc *sc; 663 664 sc = device_get_softc(dev); 665 666 if (status != VIRTIO_CONFIG_STATUS_RESET) 667 status |= vtpci_get_status(dev); 668 669 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 670} 671 672static void 673vtpci_read_dev_config(device_t dev, bus_size_t offset, 674 void *dst, int length) 675{ 676 struct vtpci_softc *sc; 677 bus_size_t off; 678 uint8_t *d; 679 int size; 680 681 sc = device_get_softc(dev); 682 off = VIRTIO_PCI_CONFIG(sc) + offset; 683 684 for (d = dst; length > 0; d += size, off += size, length -= size) { 685 if (length >= 4) { 686 size = 4; 687 *(uint32_t *)d = vtpci_read_config_4(sc, off); 688 } else if (length >= 2) { 689 size = 2; 690 *(uint16_t *)d = vtpci_read_config_2(sc, off); 691 } else { 692 size = 1; 693 *d = vtpci_read_config_1(sc, off); 694 } 695 } 696} 697 698static void 699vtpci_write_dev_config(device_t dev, bus_size_t offset, 700 void *src, int length) 701{ 702 struct vtpci_softc *sc; 703 bus_size_t off; 704 uint8_t *s; 705 int size; 706 707 sc = device_get_softc(dev); 708 off = VIRTIO_PCI_CONFIG(sc) + offset; 709 710 for (s = src; length > 0; s += size, off += size, length -= size) { 711 if (length >= 4) { 712 size = 4; 713 vtpci_write_config_4(sc, off, *(uint32_t *)s); 714 } else if (length >= 2) { 715 size = 2; 716 vtpci_write_config_2(sc, off, *(uint16_t *)s); 717 } else { 718 size = 1; 719 vtpci_write_config_1(sc, off, *s); 720 } 721 } 722} 723 724static void 725vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 726 uint64_t features) 727{ 728 device_t dev, child; 729 730 dev = sc->vtpci_dev; 731 child = sc->vtpci_child_dev; 732 733 if (device_is_attached(child) && bootverbose == 0) 734 return; 735 736 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 737} 738 739static void 740vtpci_probe_and_attach_child(struct vtpci_softc *sc) 741{ 742 device_t dev, child; 743 744 dev = sc->vtpci_dev; 745 child = sc->vtpci_child_dev; 746 747 if (child == NULL) 748 return; 749 750 if (device_get_state(child) != DS_NOTPRESENT) 751 return; 752 753 if (device_probe(child) != 0) 754 return; 755 756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 757 if (device_attach(child) != 0) { 758 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 759 vtpci_reset(sc); 760 vtpci_release_child_resources(sc); 761 /* Reset status for future attempt. */ 762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 763 } else { 764 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 765 VIRTIO_ATTACH_COMPLETED(child); 766 } 767} 768 769static int 770vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 771{ 772 device_t dev; 773 int nmsix, cnt, required; 774 775 dev = sc->vtpci_dev; 776 777 /* Allocate an additional vector for the config changes. */ 778 required = nvectors + 1; 779 780 nmsix = pci_msix_count(dev); 781 if (nmsix < required) 782 return (1); 783 784 cnt = required; 785 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 786 sc->vtpci_nmsix_resources = required; 787 return (0); 788 } 789 790 pci_release_msi(dev); 791 792 return (1); 793} 794 795static int 796vtpci_alloc_msi(struct vtpci_softc *sc) 797{ 798 device_t dev; 799 int nmsi, cnt, required; 800 801 dev = sc->vtpci_dev; 802 required = 1; 803 804 nmsi = pci_msi_count(dev); 805 if (nmsi < required) 806 return (1); 807 808 cnt = required; 809 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 810 return (0); 811 812 pci_release_msi(dev); 813 814 return (1); 815} 816 817static int 818vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 819{ 820 int i, nvectors, error; 821 822 if (vtpci_disable_msix != 0 || 823 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 824 return (ENOTSUP); 825 826 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 827 if (sc->vtpci_vqs[i].vtv_no_intr == 0) 828 nvectors++; 829 } 830 831 error = vtpci_alloc_msix(sc, nvectors); 832 if (error) 833 return (error); 834 835 sc->vtpci_flags |= VTPCI_FLAG_MSIX; 836 837 return (0); 838} 839 840static int 841vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 842{ 843 int error; 844 845 if (vtpci_disable_msix != 0 || 846 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 847 return (ENOTSUP); 848 849 error = vtpci_alloc_msix(sc, 1); 850 if (error) 851 return (error); 852 853 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 854 855 return (0); 856} 857 858static int 859vtpci_alloc_intr_msi(struct vtpci_softc *sc) 860{ 861 int error; 862 863 /* Only BHyVe supports MSI. */ 864 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 865 return (ENOTSUP); 866 867 error = vtpci_alloc_msi(sc); 868 if (error) 869 return (error); 870 871 sc->vtpci_flags |= VTPCI_FLAG_MSI; 872 873 return (0); 874} 875 876static int 877vtpci_alloc_intr_legacy(struct vtpci_softc *sc) 878{ 879 880 sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 881 882 return (0); 883} 884 885static int 886vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 887 struct vtpci_interrupt *intr) 888{ 889 struct resource *irq; 890 891 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 892 if (irq == NULL) 893 return (ENXIO); 894 895 intr->vti_irq = irq; 896 intr->vti_rid = rid; 897 898 return (0); 899} 900 901static int 902vtpci_alloc_intr_resources(struct vtpci_softc *sc) 903{ 904 struct vtpci_interrupt *intr; 905 int i, rid, flags, nvq_intrs, error; 906 907 rid = 0; 908 flags = RF_ACTIVE; 909 910 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 911 flags |= RF_SHAREABLE; 912 else 913 rid = 1; 914 915 /* 916 * For legacy and MSI interrupts, this single resource handles all 917 * interrupts. For MSIX, this resource is used for the configuration 918 * changed interrupt. 919 */ 920 intr = &sc->vtpci_device_interrupt; 921 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 922 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 923 return (error); 924 925 /* Subtract one for the configuration changed interrupt. */ 926 nvq_intrs = sc->vtpci_nmsix_resources - 1; 927 928 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 929 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 930 if (sc->vtpci_msix_vq_interrupts == NULL) 931 return (ENOMEM); 932 933 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 934 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 935 if (error) 936 return (error); 937 } 938 939 return (0); 940} 941 942static int 943vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 944{ 945 struct vtpci_interrupt *intr; 946 int error; 947 948 intr = &sc->vtpci_device_interrupt; 949 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 950 vtpci_legacy_intr, sc, &intr->vti_handler); 951 952 return (error); 953} 954 955static int 956vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 957{ 958 struct vtpci_virtqueue *vqx; 959 struct vtpci_interrupt *intr; 960 int i, error; 961 962 intr = sc->vtpci_msix_vq_interrupts; 963 964 for (i = 0; i < sc->vtpci_nvqs; i++) { 965 vqx = &sc->vtpci_vqs[i]; 966 967 if (vqx->vtv_no_intr) 968 continue; 969 970 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 971 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 972 &intr->vti_handler); 973 if (error) 974 return (error); 975 976 intr++; 977 } 978 979 return (0); 980} 981 982static int 983vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 984{ 985 device_t dev; 986 struct vtpci_interrupt *intr; 987 int error; 988 989 dev = sc->vtpci_dev; 990 intr = &sc->vtpci_device_interrupt; 991 992 error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 993 vtpci_config_intr, sc, &intr->vti_handler); 994 if (error) 995 return (error); 996 997 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 998 intr = sc->vtpci_msix_vq_interrupts; 999 error = bus_setup_intr(dev, intr->vti_irq, type, 1000 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 1001 &intr->vti_handler); 1002 } else 1003 error = vtpci_setup_pervq_msix_interrupts(sc, type); 1004 1005 return (error ? error : vtpci_set_host_msix_vectors(sc)); 1006} 1007 1008static int 1009vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1010{ 1011 int error; 1012 1013 type |= INTR_MPSAFE; 1014 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1015 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1016 1017 error = vtpci_alloc_intr_resources(sc); 1018 if (error) 1019 return (error); 1020 1021 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1022 error = vtpci_setup_legacy_interrupt(sc, type); 1023 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1024 error = vtpci_setup_msi_interrupt(sc, type); 1025 else 1026 error = vtpci_setup_msix_interrupts(sc, type); 1027 1028 return (error); 1029} 1030 1031static int 1032vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1033 struct vtpci_interrupt *intr) 1034{ 1035 device_t dev; 1036 uint16_t vector; 1037 1038 dev = sc->vtpci_dev; 1039 1040 if (intr != NULL) { 1041 /* Map from guest rid to host vector. */ 1042 vector = intr->vti_rid - 1; 1043 } else 1044 vector = VIRTIO_MSI_NO_VECTOR; 1045 1046 vtpci_write_config_2(sc, offset, vector); 1047 1048 /* Read vector to determine if the host had sufficient resources. */ 1049 if (vtpci_read_config_2(sc, offset) != vector) { 1050 device_printf(dev, 1051 "insufficient host resources for MSIX interrupts\n"); 1052 return (ENODEV); 1053 } 1054 1055 return (0); 1056} 1057 1058static int 1059vtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1060{ 1061 struct vtpci_interrupt *intr, *tintr; 1062 int idx, offset, error; 1063 1064 intr = &sc->vtpci_device_interrupt; 1065 offset = VIRTIO_MSI_CONFIG_VECTOR; 1066 1067 error = vtpci_register_msix_vector(sc, offset, intr); 1068 if (error) 1069 return (error); 1070 1071 intr = sc->vtpci_msix_vq_interrupts; 1072 offset = VIRTIO_MSI_QUEUE_VECTOR; 1073 1074 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1075 vtpci_select_virtqueue(sc, idx); 1076 1077 if (sc->vtpci_vqs[idx].vtv_no_intr) 1078 tintr = NULL; 1079 else 1080 tintr = intr; 1081 1082 error = vtpci_register_msix_vector(sc, offset, tintr); 1083 if (error) 1084 break; 1085 1086 /* 1087 * For shared MSIX, all the virtqueues share the first 1088 * interrupt. 1089 */ 1090 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1091 intr++; 1092 } 1093 1094 return (error); 1095} 1096 1097static int 1098vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1099{ 1100 struct vtpci_virtqueue *vqx; 1101 struct virtqueue *vq; 1102 int error; 1103 uint16_t size; 1104 1105 vqx = &sc->vtpci_vqs[idx]; 1106 vq = vqx->vtv_vq; 1107 1108 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1109 1110 vtpci_select_virtqueue(sc, idx); 1111 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 1112 1113 error = virtqueue_reinit(vq, size); 1114 if (error) 1115 return (error); 1116 1117 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 1118 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1119 1120 return (0); 1121} 1122 1123static void 1124vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1125{ 1126 device_t dev; 1127 1128 dev = sc->vtpci_dev; 1129 1130 if (intr->vti_handler != NULL) { 1131 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1132 intr->vti_handler = NULL; 1133 } 1134 1135 if (intr->vti_irq != NULL) { 1136 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1137 intr->vti_irq); 1138 intr->vti_irq = NULL; 1139 intr->vti_rid = -1; 1140 } 1141} 1142 1143static void 1144vtpci_free_interrupts(struct vtpci_softc *sc) 1145{ 1146 struct vtpci_interrupt *intr; 1147 int i, nvq_intrs; 1148 1149 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1150 1151 if (sc->vtpci_nmsix_resources != 0) { 1152 nvq_intrs = sc->vtpci_nmsix_resources - 1; 1153 sc->vtpci_nmsix_resources = 0; 1154 1155 intr = sc->vtpci_msix_vq_interrupts; 1156 if (intr != NULL) { 1157 for (i = 0; i < nvq_intrs; i++, intr++) 1158 vtpci_free_interrupt(sc, intr); 1159 1160 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1161 sc->vtpci_msix_vq_interrupts = NULL; 1162 } 1163 } 1164 1165 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1166 pci_release_msi(sc->vtpci_dev); 1167 1168 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1169} 1170 1171static void 1172vtpci_free_virtqueues(struct vtpci_softc *sc) 1173{ 1174 struct vtpci_virtqueue *vqx; 1175 int idx; 1176 1177 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1178 vqx = &sc->vtpci_vqs[idx]; 1179 1180 vtpci_select_virtqueue(sc, idx); 1181 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1182 1183 virtqueue_free(vqx->vtv_vq); 1184 vqx->vtv_vq = NULL; 1185 } 1186 1187 free(sc->vtpci_vqs, M_DEVBUF); 1188 sc->vtpci_vqs = NULL; 1189 sc->vtpci_nvqs = 0; 1190} 1191 1192static void 1193vtpci_release_child_resources(struct vtpci_softc *sc) 1194{ 1195 1196 vtpci_free_interrupts(sc); 1197 vtpci_free_virtqueues(sc); 1198} 1199 1200static void 1201vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1202{ 1203 int idx; 1204 1205 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1206 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1207 VIRTIO_MSI_NO_VECTOR); 1208 1209 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1210 vtpci_select_virtqueue(sc, idx); 1211 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1212 VIRTIO_MSI_NO_VECTOR); 1213 } 1214 } 1215 1216 vtpci_free_interrupts(sc); 1217} 1218 1219static void 1220vtpci_reset(struct vtpci_softc *sc) 1221{ 1222 1223 /* 1224 * Setting the status to RESET sets the host device to 1225 * the original, uninitialized state. 1226 */ 1227 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1228} 1229 1230static void 1231vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1232{ 1233 1234 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1235} 1236 1237static void 1238vtpci_legacy_intr(void *xsc) 1239{ 1240 struct vtpci_softc *sc; 1241 struct vtpci_virtqueue *vqx; 1242 int i; 1243 uint8_t isr; 1244 1245 sc = xsc; 1246 vqx = &sc->vtpci_vqs[0]; 1247 1248 /* Reading the ISR also clears it. */ 1249 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1250 1251 if (isr & VIRTIO_PCI_ISR_CONFIG) 1252 vtpci_config_intr(sc); 1253 1254 if (isr & VIRTIO_PCI_ISR_INTR) { 1255 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1256 if (vqx->vtv_no_intr == 0) 1257 virtqueue_intr(vqx->vtv_vq); 1258 } 1259 } 1260} 1261 1262static int 1263vtpci_vq_shared_intr_filter(void *xsc) 1264{ 1265 struct vtpci_softc *sc; 1266 struct vtpci_virtqueue *vqx; 1267 int i, rc; 1268 1269 rc = 0; 1270 sc = xsc; 1271 vqx = &sc->vtpci_vqs[0]; 1272 1273 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1274 if (vqx->vtv_no_intr == 0) 1275 rc |= virtqueue_intr_filter(vqx->vtv_vq); 1276 } 1277 1278 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1279} 1280 1281static void 1282vtpci_vq_shared_intr(void *xsc) 1283{ 1284 struct vtpci_softc *sc; 1285 struct vtpci_virtqueue *vqx; 1286 int i; 1287 1288 sc = xsc; 1289 vqx = &sc->vtpci_vqs[0]; 1290 1291 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1292 if (vqx->vtv_no_intr == 0) 1293 virtqueue_intr(vqx->vtv_vq); 1294 } 1295} 1296 1297static int 1298vtpci_vq_intr_filter(void *xvq) 1299{ 1300 struct virtqueue *vq; 1301 int rc; 1302 1303 vq = xvq; 1304 rc = virtqueue_intr_filter(vq); 1305 1306 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1307} 1308 1309static void 1310vtpci_vq_intr(void *xvq) 1311{ 1312 struct virtqueue *vq; 1313 1314 vq = xvq; 1315 virtqueue_intr(vq); 1316} 1317 1318static void 1319vtpci_config_intr(void *xsc) 1320{ 1321 struct vtpci_softc *sc; 1322 device_t child; 1323 1324 sc = xsc; 1325 child = sc->vtpci_child_dev; 1326 1327 if (child != NULL) 1328 VIRTIO_CONFIG_CHANGE(child); 1329}
|