1/*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7/* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */ 8 9#include <sys/cdefs.h> 10__FBSDID("$FreeBSD$"); 11 12#include <sys/types.h> 13#include <sys/bus.h> 14#include <sys/kernel.h> 15#include <sys/malloc.h> 16#include <sys/module.h> 17#include <sys/rman.h> 18#include <sys/systm.h> 19 20#include <dev/pci/pcireg.h> 21#include <dev/pci/pcivar.h> 22 23#include <machine/bus.h> 24 25#include "vmci.h" 26#include "vmci_doorbell.h" 27#include "vmci_driver.h" 28#include "vmci_kernel_defs.h" 29#include "vmci_queue_pair.h" 30 31static int vmci_probe(device_t); 32static int vmci_attach(device_t); 33static int vmci_detach(device_t); 34static int vmci_shutdown(device_t); 35 36static int vmci_map_bars(struct vmci_softc *); 37static void vmci_unmap_bars(struct vmci_softc *); 38 39static int vmci_config_capabilities(struct vmci_softc *); 40 41static int vmci_dma_malloc_int(struct vmci_softc *, bus_size_t, 42 bus_size_t, struct vmci_dma_alloc *); 43static void vmci_dma_free_int(struct vmci_softc *, 44 struct vmci_dma_alloc *); 45 46static int vmci_config_interrupts(struct vmci_softc *); 47static int vmci_config_interrupt(struct vmci_softc *); 48static int vmci_check_intr_cnt(struct vmci_softc *); 49static int vmci_allocate_interrupt_resources(struct vmci_softc *); 50static int vmci_setup_interrupts(struct vmci_softc *); 51static void vmci_dismantle_interrupts(struct vmci_softc *); 52static void vmci_interrupt(void *); 53static void vmci_interrupt_bm(void *); 54static void dispatch_datagrams(void *, int); 55static void process_bitmap(void *, int); 56 57static void vmci_delayed_work_fn_cb(void *context, int data); 58 59static device_method_t vmci_methods[] = { 60 /* Device interface. */ 61 DEVMETHOD(device_probe, vmci_probe), 62 DEVMETHOD(device_attach, vmci_attach), 63 DEVMETHOD(device_detach, vmci_detach), 64 DEVMETHOD(device_shutdown, vmci_shutdown), 65 66 DEVMETHOD_END 67}; 68 69static driver_t vmci_driver = { 70 "vmci", vmci_methods, sizeof(struct vmci_softc) 71}; 72 73static devclass_t vmci_devclass; 74DRIVER_MODULE(vmci, pci, vmci_driver, vmci_devclass, 0, 0); 75MODULE_VERSION(vmci, VMCI_VERSION); 76const struct { 77 uint16_t vendor; 78 uint16_t device; 79 const char *desc; 80} vmci_ids[] = { 81 { VMCI_VMWARE_VENDOR_ID, VMCI_VMWARE_DEVICE_ID, 82 "VMware Virtual Machine Communication Interface" }, 83}; 84MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmci, vmci_ids, 85 nitems(vmci_ids)); 86 87MODULE_DEPEND(vmci, pci, 1, 1, 1); 88 89static struct vmci_softc *vmci_sc; 90 91#define LGPFX "vmci: " 92/* 93 * Allocate a buffer for incoming datagrams globally to avoid repeated 94 * allocation in the interrupt handler's atomic context. 95 */ 96static uint8_t *data_buffer = NULL; 97static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE; 98 99struct vmci_delayed_work_info { 100 vmci_work_fn *work_fn; 101 void *data; 102 vmci_list_item(vmci_delayed_work_info) entry; 103}; 104 105/* 106 *------------------------------------------------------------------------------ 107 * 108 * vmci_probe -- 109 * 110 * Probe to see if the VMCI device is present. 111 * 112 * Results: 113 * BUS_PROBE_DEFAULT if device exists, ENXIO otherwise. 114 * 115 * Side effects: 116 * None. 117 * 118 *------------------------------------------------------------------------------ 119 */ 120 121static int 122vmci_probe(device_t dev) 123{ 124 125 if (pci_get_vendor(dev) == vmci_ids[0].vendor && 126 pci_get_device(dev) == vmci_ids[0].device) { 127 device_set_desc(dev, vmci_ids[0].desc); 128 129 return (BUS_PROBE_DEFAULT); 130 } 131 132 return (ENXIO); 133} 134 135/* 136 *------------------------------------------------------------------------------ 137 * 138 * vmci_attach -- 139 * 140 * Attach VMCI device to the system after vmci_probe() has been called and 141 * the device has been detected. 142 * 143 * Results: 144 * 0 if success, ENXIO otherwise. 145 * 146 * Side effects: 147 * None. 148 * 149 *------------------------------------------------------------------------------ 150 */ 151 152static int 153vmci_attach(device_t dev) 154{ 155 struct vmci_softc *sc; 156 int error, i; 157 158 sc = device_get_softc(dev); 159 sc->vmci_dev = dev; 160 vmci_sc = sc; 161 162 data_buffer = NULL; 163 sc->vmci_num_intr = 0; 164 for (i = 0; i < VMCI_MAX_INTRS; i++) { 165 sc->vmci_intrs[i].vmci_irq = NULL; 166 sc->vmci_intrs[i].vmci_handler = NULL; 167 } 168 169 TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc); 170 TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc); 171 172 TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc); 173 174 pci_enable_busmaster(dev); 175 176 mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN); 177 mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock", 178 NULL, MTX_DEF); 179 180 error = vmci_map_bars(sc); 181 if (error) { 182 VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n"); 183 goto fail; 184 } 185 186 error = vmci_config_capabilities(sc); 187 if (error) { 188 VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n"); 189 goto fail; 190 } 191 192 vmci_list_init(&sc->vmci_delayed_work_infos); 193 194 vmci_components_init(); 195 vmci_util_init(); 196 error = vmci_qp_guest_endpoints_init(); 197 if (error) { 198 VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n"); 199 goto fail; 200 } 201 202 error = vmci_config_interrupts(sc); 203 if (error) 204 VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n"); 205 206fail: 207 if (error) { 208 vmci_detach(dev); 209 return (ENXIO); 210 } 211 212 return (0); 213} 214 215/* 216 *------------------------------------------------------------------------------ 217 * 218 * vmci_detach -- 219 * 220 * Detach the VMCI device. 221 * 222 * Results: 223 * 0 224 * 225 * Side effects: 226 * None. 227 * 228 *------------------------------------------------------------------------------ 229 */ 230 231static int 232vmci_detach(device_t dev) 233{ 234 struct vmci_softc *sc; 235 236 sc = device_get_softc(dev); 237 238 vmci_qp_guest_endpoints_exit(); 239 vmci_util_exit(); 240 241 vmci_dismantle_interrupts(sc); 242 243 vmci_components_cleanup(); 244 245 taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task); 246 mtx_destroy(&sc->vmci_delayed_work_lock); 247 248 if (sc->vmci_res0 != NULL) 249 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 250 VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET); 251 252 if (sc->vmci_notifications_bitmap.dma_vaddr != NULL) 253 vmci_dma_free(&sc->vmci_notifications_bitmap); 254 255 vmci_unmap_bars(sc); 256 257 mtx_destroy(&sc->vmci_spinlock); 258 259 pci_disable_busmaster(dev); 260 261 return (0); 262} 263 264/* 265 *------------------------------------------------------------------------------ 266 * 267 * vmci_shutdown -- 268 * 269 * This function is called during system shutdown. We don't do anything. 270 * 271 * Results: 272 * 0 273 * 274 * Side effects: 275 * None. 276 * 277 *------------------------------------------------------------------------------ 278 */ 279 280static int 281vmci_shutdown(device_t dev) 282{ 283 284 return (0); 285} 286 287/* 288 *------------------------------------------------------------------------------ 289 * 290 * vmci_map_bars -- 291 * 292 * Maps the PCI I/O and MMIO BARs. 293 * 294 * Results: 295 * 0 on success, ENXIO otherwise. 296 * 297 * Side effects: 298 * None. 299 * 300 *------------------------------------------------------------------------------ 301 */ 302 303static int 304vmci_map_bars(struct vmci_softc *sc) 305{ 306 int rid; 307 308 /* Map the PCI I/O BAR: BAR0 */ 309 rid = PCIR_BAR(0); 310 sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT, 311 &rid, RF_ACTIVE); 312 if (sc->vmci_res0 == NULL) { 313 VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n"); 314 return (ENXIO); 315 } 316 317 sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0); 318 sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0); 319 sc->vmci_ioaddr = rman_get_start(sc->vmci_res0); 320 321 /* Map the PCI MMIO BAR: BAR1 */ 322 rid = PCIR_BAR(1); 323 sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY, 324 &rid, RF_ACTIVE); 325 if (sc->vmci_res1 == NULL) { 326 VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n"); 327 return (ENXIO); 328 } 329 330 sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1); 331 sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1); 332 333 return (0); 334} 335 336/* 337 *------------------------------------------------------------------------------ 338 * 339 * vmci_unmap_bars -- 340 * 341 * Unmaps the VMCI PCI I/O and MMIO BARs. 342 * 343 * Results: 344 * None. 345 * 346 * Side effects: 347 * None. 348 * 349 *------------------------------------------------------------------------------ 350 */ 351 352static void 353vmci_unmap_bars(struct vmci_softc *sc) 354{ 355 int rid; 356 357 if (sc->vmci_res0 != NULL) { 358 rid = PCIR_BAR(0); 359 bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid, 360 sc->vmci_res0); 361 sc->vmci_res0 = NULL; 362 } 363 364 if (sc->vmci_res1 != NULL) { 365 rid = PCIR_BAR(1); 366 bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid, 367 sc->vmci_res1); 368 sc->vmci_res1 = NULL; 369 } 370} 371 372/* 373 *------------------------------------------------------------------------------ 374 * 375 * vmci_config_capabilities -- 376 * 377 * Check the VMCI device capabilities and configure the device accordingly. 378 * 379 * Results: 380 * 0 if success, ENODEV otherwise. 381 * 382 * Side effects: 383 * Device capabilities are enabled. 384 * 385 *------------------------------------------------------------------------------ 386 */ 387 388static int 389vmci_config_capabilities(struct vmci_softc *sc) 390{ 391 unsigned long bitmap_PPN; 392 int error; 393 394 /* 395 * Verify that the VMCI device supports the capabilities that we 396 * need. Datagrams are necessary and notifications will be used 397 * if the device supports it. 398 */ 399 sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0, 400 VMCI_CAPS_ADDR); 401 402 if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) { 403 VMCI_LOG_ERROR(LGPFX"VMCI device does not support " 404 "datagrams.\n"); 405 return (ENODEV); 406 } 407 408 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) { 409 sc->capabilities = VMCI_CAPS_DATAGRAM; 410 error = vmci_dma_malloc(PAGE_SIZE, 1, 411 &sc->vmci_notifications_bitmap); 412 if (error) 413 VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for " 414 "notification bitmap.\n"); 415 else { 416 memset(sc->vmci_notifications_bitmap.dma_vaddr, 0, 417 PAGE_SIZE); 418 sc->capabilities |= VMCI_CAPS_NOTIFICATIONS; 419 } 420 } else 421 sc->capabilities = VMCI_CAPS_DATAGRAM; 422 423 /* Let the host know which capabilities we intend to use. */ 424 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 425 VMCI_CAPS_ADDR, sc->capabilities); 426 427 /* 428 * Register notification bitmap with device if that capability is 429 * used. 430 */ 431 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) { 432 bitmap_PPN = 433 sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT; 434 vmci_register_notification_bitmap(bitmap_PPN); 435 } 436 437 /* Check host capabilities. */ 438 if (!vmci_check_host_capabilities()) 439 return (ENODEV); 440 441 return (0); 442} 443 444/* 445 *------------------------------------------------------------------------------ 446 * 447 * vmci_dmamap_cb -- 448 * 449 * Callback to receive mapping information resulting from the load of a 450 * bus_dmamap_t via bus_dmamap_load() 451 * 452 * Results: 453 * None. 454 * 455 * Side effects: 456 * None. 457 * 458 *------------------------------------------------------------------------------ 459 */ 460 461static void 462vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 463{ 464 bus_addr_t *baddr = arg; 465 466 if (error == 0) 467 *baddr = segs->ds_addr; 468} 469 470/* 471 *------------------------------------------------------------------------------ 472 * 473 * vmci_dma_malloc_int -- 474 * 475 * Internal function that allocates DMA memory. 476 * 477 * Results: 478 * 0 if success. 479 * ENOMEM if insufficient memory. 480 * EINPROGRESS if mapping is deferred. 481 * EINVAL if the request was invalid. 482 * 483 * Side effects: 484 * DMA memory is allocated. 485 * 486 *------------------------------------------------------------------------------ 487 */ 488 489static int 490vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align, 491 struct vmci_dma_alloc *dma) 492{ 493 int error; 494 495 bzero(dma, sizeof(struct vmci_dma_alloc)); 496 497 error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev), 498 align, 0, /* alignment, bounds */ 499 BUS_SPACE_MAXADDR, /* lowaddr */ 500 BUS_SPACE_MAXADDR, /* highaddr */ 501 NULL, NULL, /* filter, filterarg */ 502 size, /* maxsize */ 503 1, /* nsegments */ 504 size, /* maxsegsize */ 505 BUS_DMA_ALLOCNOW, /* flags */ 506 NULL, /* lockfunc */ 507 NULL, /* lockfuncarg */ 508 &dma->dma_tag); 509 if (error) { 510 VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error); 511 goto fail; 512 } 513 514 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 515 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 516 if (error) { 517 VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error); 518 goto fail; 519 } 520 521 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 522 size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 523 if (error) { 524 VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error); 525 goto fail; 526 } 527 528 dma->dma_size = size; 529 530fail: 531 if (error) 532 vmci_dma_free(dma); 533 534 return (error); 535} 536 537/* 538 *------------------------------------------------------------------------------ 539 * 540 * vmci_dma_malloc -- 541 * 542 * This function is a wrapper around vmci_dma_malloc_int for callers 543 * outside of this module. Since we only support a single VMCI device, this 544 * wrapper provides access to the device softc structure. 545 * 546 * Results: 547 * 0 if success. 548 * ENOMEM if insufficient memory. 549 * EINPROGRESS if mapping is deferred. 550 * EINVAL if the request was invalid. 551 * 552 * Side effects: 553 * DMA memory is allocated. 554 * 555 *------------------------------------------------------------------------------ 556 */ 557 558int 559vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma) 560{ 561 562 return (vmci_dma_malloc_int(vmci_sc, size, align, dma)); 563} 564 565/* 566 *------------------------------------------------------------------------------ 567 * 568 * vmci_dma_free_int -- 569 * 570 * Internal function that frees DMA memory. 571 * 572 * Results: 573 * None. 574 * 575 * Side effects: 576 * Frees DMA memory. 577 * 578 *------------------------------------------------------------------------------ 579 */ 580 581static void 582vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma) 583{ 584 585 if (dma->dma_tag != NULL) { 586 if (dma->dma_paddr != 0) { 587 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 588 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 589 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 590 } 591 592 if (dma->dma_vaddr != NULL) 593 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 594 dma->dma_map); 595 596 bus_dma_tag_destroy(dma->dma_tag); 597 } 598 bzero(dma, sizeof(struct vmci_dma_alloc)); 599} 600 601/* 602 *------------------------------------------------------------------------------ 603 * 604 * vmci_dma_free -- 605 * 606 * This function is a wrapper around vmci_dma_free_int for callers outside 607 * of this module. Since we only support a single VMCI device, this wrapper 608 * provides access to the device softc structure. 609 * 610 * Results: 611 * None. 612 * 613 * Side effects: 614 * Frees DMA memory. 615 * 616 *------------------------------------------------------------------------------ 617 */ 618 619void 620vmci_dma_free(struct vmci_dma_alloc *dma) 621{ 622 623 vmci_dma_free_int(vmci_sc, dma); 624} 625 626/* 627 *------------------------------------------------------------------------------ 628 * 629 * vmci_config_interrupts -- 630 * 631 * Configures and enables interrupts. Try to configure MSI-X. If this fails, 632 * try to configure MSI. If even this fails, try legacy interrupts. 633 * 634 * Results: 635 * 0 if success. 636 * ENOMEM if insufficient memory. 637 * ENODEV if the device doesn't support interrupts. 638 * ENXIO if the device configuration failed. 639 * 640 * Side effects: 641 * Interrupts get enabled if successful. 642 * 643 *------------------------------------------------------------------------------ 644 */ 645 646static int 647vmci_config_interrupts(struct vmci_softc *sc) 648{ 649 int error; 650 651 data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT); 652 if (data_buffer == NULL) 653 return (ENOMEM); 654 655 sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX; 656 error = vmci_config_interrupt(sc); 657 if (error) { 658 sc->vmci_intr_type = VMCI_INTR_TYPE_MSI; 659 error = vmci_config_interrupt(sc); 660 } 661 if (error) { 662 sc->vmci_intr_type = VMCI_INTR_TYPE_INTX; 663 error = vmci_config_interrupt(sc); 664 } 665 if (error) 666 return (error); 667 668 /* Enable specific interrupt bits. */ 669 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) 670 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 671 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION); 672 else 673 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 674 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM); 675 676 /* Enable interrupts. */ 677 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 678 VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE); 679 680 return (0); 681} 682 683/* 684 *------------------------------------------------------------------------------ 685 * 686 * vmci_config_interrupt -- 687 * 688 * Check the number of interrupts supported, allocate resources and setup 689 * interrupts. 690 * 691 * Results: 692 * 0 if success. 693 * ENOMEM if insufficient memory. 694 * ENODEV if the device doesn't support interrupts. 695 * ENXIO if the device configuration failed. 696 * 697 * Side effects: 698 * Resources get allocated and interrupts get setup (but not enabled) if 699 * successful. 700 * 701 *------------------------------------------------------------------------------ 702 */ 703 704static int 705vmci_config_interrupt(struct vmci_softc *sc) 706{ 707 int error; 708 709 error = vmci_check_intr_cnt(sc); 710 if (error) 711 return (error); 712 713 error = vmci_allocate_interrupt_resources(sc); 714 if (error) 715 return (error); 716 717 error = vmci_setup_interrupts(sc); 718 if (error) 719 return (error); 720 721 return (0); 722} 723 724/* 725 *------------------------------------------------------------------------------ 726 * 727 * vmci_check_intr_cnt -- 728 * 729 * Check the number of interrupts supported by the device and ask PCI bus 730 * to allocate appropriate number of interrupts. 731 * 732 * Results: 733 * 0 if success. 734 * ENODEV if the device doesn't support any interrupts. 735 * ENXIO if the device configuration failed. 736 * 737 * Side effects: 738 * Resources get allocated on success. 739 * 740 *------------------------------------------------------------------------------ 741 */ 742 743static int 744vmci_check_intr_cnt(struct vmci_softc *sc) 745{ 746 747 if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) { 748 sc->vmci_num_intr = 1; 749 return (0); 750 } 751 752 /* 753 * Make sure that the device supports the required number of MSI/MSI-X 754 * messages. We try for 2 MSI-X messages but 1 is good too. We need at 755 * least 1 MSI message. 756 */ 757 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ? 758 pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev); 759 760 if (!sc->vmci_num_intr) { 761 VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt" 762 " messages"); 763 return (ENODEV); 764 } 765 766 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ? 767 VMCI_MAX_INTRS : 1; 768 if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) { 769 if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr)) 770 return (ENXIO); 771 } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) { 772 if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr)) 773 return (ENXIO); 774 } 775 776 return (0); 777} 778 779/* 780 *------------------------------------------------------------------------------ 781 * 782 * vmci_allocate_interrupt_resources -- 783 * 784 * Allocate resources necessary for interrupts. 785 * 786 * Results: 787 * 0 if success, ENXIO otherwise. 788 * 789 * Side effects: 790 * Resources get allocated on success. 791 * 792 *------------------------------------------------------------------------------ 793 */ 794 795static int 796vmci_allocate_interrupt_resources(struct vmci_softc *sc) 797{ 798 struct resource *irq; 799 int flags, i, rid; 800 801 flags = RF_ACTIVE; 802 flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0; 803 rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1; 804 805 for (i = 0; i < sc->vmci_num_intr; i++, rid++) { 806 irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid, 807 flags); 808 if (irq == NULL) 809 return (ENXIO); 810 sc->vmci_intrs[i].vmci_irq = irq; 811 sc->vmci_intrs[i].vmci_rid = rid; 812 } 813 814 return (0); 815} 816 817/* 818 *------------------------------------------------------------------------------ 819 * 820 * vmci_setup_interrupts -- 821 * 822 * Sets up the interrupts. 823 * 824 * Results: 825 * 0 if success, appropriate error code from bus_setup_intr otherwise. 826 * 827 * Side effects: 828 * Interrupt handler gets attached. 829 * 830 *------------------------------------------------------------------------------ 831 */ 832 833static int 834vmci_setup_interrupts(struct vmci_softc *sc) 835{ 836 struct vmci_interrupt *intr; 837 int error, flags; 838 839 flags = INTR_TYPE_NET | INTR_MPSAFE; 840 if (sc->vmci_num_intr > 1) 841 flags |= INTR_EXCL; 842 843 intr = &sc->vmci_intrs[0]; 844 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL, 845 vmci_interrupt, NULL, &intr->vmci_handler); 846 if (error) 847 return (error); 848 bus_describe_intr(sc->vmci_dev, intr->vmci_irq, intr->vmci_handler, 849 "vmci_interrupt"); 850 851 if (sc->vmci_num_intr == 2) { 852 intr = &sc->vmci_intrs[1]; 853 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, 854 NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler); 855 if (error) 856 return (error); 857 bus_describe_intr(sc->vmci_dev, intr->vmci_irq, 858 intr->vmci_handler, "vmci_interrupt_bm"); 859 } 860 861 return (0); 862} 863 864/* 865 *------------------------------------------------------------------------------ 866 * 867 * vmci_interrupt -- 868 * 869 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X 870 * interrupt (vector VMCI_INTR_DATAGRAM). 871 * 872 * Results: 873 * None. 874 * 875 * Side effects: 876 * None. 877 * 878 *------------------------------------------------------------------------------ 879 */ 880 881static void 882vmci_interrupt(void *arg) 883{ 884 885 if (vmci_sc->vmci_num_intr == 2) 886 taskqueue_enqueue(taskqueue_swi, 887 &vmci_sc->vmci_interrupt_dq_task); 888 else { 889 unsigned int icr; 890 891 icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR); 892 if (icr == 0 || icr == 0xffffffff) 893 return; 894 if (icr & VMCI_ICR_DATAGRAM) { 895 taskqueue_enqueue(taskqueue_swi, 896 &vmci_sc->vmci_interrupt_dq_task); 897 icr &= ~VMCI_ICR_DATAGRAM; 898 } 899 if (icr & VMCI_ICR_NOTIFICATION) { 900 taskqueue_enqueue(taskqueue_swi, 901 &vmci_sc->vmci_interrupt_bm_task); 902 icr &= ~VMCI_ICR_NOTIFICATION; 903 } 904 if (icr != 0) 905 VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt " 906 "cause"); 907 } 908} 909 910/* 911 *------------------------------------------------------------------------------ 912 * 913 * vmci_interrupt_bm -- 914 * 915 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION, 916 * which is for the notification bitmap. Will only get called if we are 917 * using MSI-X with exclusive vectors. 918 * 919 * Results: 920 * None. 921 * 922 * Side effects: 923 * None. 924 * 925 *------------------------------------------------------------------------------ 926 */ 927 928static void 929vmci_interrupt_bm(void *arg) 930{ 931 932 ASSERT(vmci_sc->vmci_num_intr == 2); 933 taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task); 934} 935 936/* 937 *------------------------------------------------------------------------------ 938 * 939 * dispatch_datagrams -- 940 * 941 * Reads and dispatches incoming datagrams. 942 * 943 * Results: 944 * None. 945 * 946 * Side effects: 947 * Reads data from the device. 948 * 949 *------------------------------------------------------------------------------ 950 */ 951 952static void 953dispatch_datagrams(void *context, int data) 954{ 955 956 if (data_buffer == NULL) 957 VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer " 958 "present"); 959 960 vmci_read_datagrams_from_port((vmci_io_handle) 0, 961 vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR, 962 data_buffer, data_buffer_size); 963} 964 965/* 966 *------------------------------------------------------------------------------ 967 * 968 * process_bitmap -- 969 * 970 * Scans the notification bitmap for raised flags, clears them and handles 971 * the notifications. 972 * 973 * Results: 974 * None. 975 * 976 * Side effects: 977 * None. 978 * 979 *------------------------------------------------------------------------------ 980 */ 981 982static void 983process_bitmap(void *context, int data) 984{ 985 986 if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL) 987 VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present"); 988 989 vmci_scan_notification_bitmap( 990 vmci_sc->vmci_notifications_bitmap.dma_vaddr); 991} 992 993/* 994 *------------------------------------------------------------------------------ 995 * 996 * vmci_dismantle_interrupts -- 997 * 998 * Releases resources, detaches the interrupt handler and drains the task 999 * queue. 1000 * 1001 * Results: 1002 * None. 1003 * 1004 * Side effects: 1005 * No more interrupts. 1006 * 1007 *------------------------------------------------------------------------------ 1008 */ 1009 1010static void 1011vmci_dismantle_interrupts(struct vmci_softc *sc) 1012{ 1013 struct vmci_interrupt *intr; 1014 int i; 1015 1016 for (i = 0; i < sc->vmci_num_intr; i++) { 1017 intr = &sc->vmci_intrs[i]; 1018 if (intr->vmci_handler != NULL) { 1019 bus_teardown_intr(sc->vmci_dev, intr->vmci_irq, 1020 intr->vmci_handler); 1021 intr->vmci_handler = NULL; 1022 } 1023 if (intr->vmci_irq != NULL) { 1024 bus_release_resource(sc->vmci_dev, SYS_RES_IRQ, 1025 intr->vmci_rid, intr->vmci_irq); 1026 intr->vmci_irq = NULL; 1027 intr->vmci_rid = -1; 1028 } 1029 } 1030 1031 if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) && 1032 (sc->vmci_num_intr)) 1033 pci_release_msi(sc->vmci_dev); 1034 1035 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task); 1036 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task); 1037 1038 if (data_buffer != NULL) 1039 free(data_buffer, M_DEVBUF); 1040} 1041 1042/* 1043 *------------------------------------------------------------------------------ 1044 * 1045 * vmci_delayed_work_fn_cb -- 1046 * 1047 * Callback function that executes the queued up delayed work functions. 1048 * 1049 * Results: 1050 * None. 1051 * 1052 * Side effects: 1053 * None. 1054 * 1055 *------------------------------------------------------------------------------ 1056 */ 1057 1058static void 1059vmci_delayed_work_fn_cb(void *context, int data) 1060{ 1061 vmci_list(vmci_delayed_work_info) temp_list; 1062 1063 vmci_list_init(&temp_list); 1064 1065 /* 1066 * Swap vmci_delayed_work_infos list with the empty temp_list while 1067 * holding a lock. vmci_delayed_work_infos would then be an empty list 1068 * and temp_list would contain the elements from the original 1069 * vmci_delayed_work_infos. Finally, iterate through temp_list 1070 * executing the delayed callbacks. 1071 */ 1072 1073 mtx_lock(&vmci_sc->vmci_delayed_work_lock); 1074 vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos, 1075 vmci_delayed_work_info, entry); 1076 mtx_unlock(&vmci_sc->vmci_delayed_work_lock); 1077 1078 while (!vmci_list_empty(&temp_list)) { 1079 struct vmci_delayed_work_info *delayed_work_info = 1080 vmci_list_first(&temp_list); 1081 1082 delayed_work_info->work_fn(delayed_work_info->data); 1083 1084 vmci_list_remove(delayed_work_info, entry); 1085 vmci_free_kernel_mem(delayed_work_info, 1086 sizeof(*delayed_work_info)); 1087 } 1088} 1089 1090/* 1091 *------------------------------------------------------------------------------ 1092 * 1093 * vmci_schedule_delayed_work_fn -- 1094 * 1095 * Schedule the specified callback. 1096 * 1097 * Results: 1098 * 0 if success, error code otherwise. 1099 * 1100 * Side effects: 1101 * None. 1102 * 1103 *------------------------------------------------------------------------------ 1104 */ 1105 1106int 1107vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data) 1108{ 1109 struct vmci_delayed_work_info *delayed_work_info; 1110 1111 delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info), 1112 VMCI_MEMORY_ATOMIC); 1113 1114 if (!delayed_work_info) 1115 return (VMCI_ERROR_NO_MEM); 1116 1117 delayed_work_info->work_fn = work_fn; 1118 delayed_work_info->data = data; 1119 mtx_lock(&vmci_sc->vmci_delayed_work_lock); 1120 vmci_list_insert(&vmci_sc->vmci_delayed_work_infos, 1121 delayed_work_info, entry); 1122 mtx_unlock(&vmci_sc->vmci_delayed_work_lock); 1123 1124 taskqueue_enqueue(taskqueue_thread, 1125 &vmci_sc->vmci_delayed_work_task); 1126 1127 return (VMCI_SUCCESS); 1128} 1129 1130/* 1131 *------------------------------------------------------------------------------ 1132 * 1133 * vmci_send_datagram -- 1134 * 1135 * VM to hypervisor call mechanism. 1136 * 1137 * Results: 1138 * The result of the hypercall. 1139 * 1140 * Side effects: 1141 * None. 1142 * 1143 *------------------------------------------------------------------------------ 1144 */ 1145 1146int 1147vmci_send_datagram(struct vmci_datagram *dg) 1148{ 1149 int result; 1150 1151 if (dg == NULL) 1152 return (VMCI_ERROR_INVALID_ARGS); 1153 1154 /* 1155 * Need to acquire spinlock on the device because 1156 * the datagram data may be spread over multiple pages and the monitor 1157 * may interleave device user rpc calls from multiple VCPUs. Acquiring 1158 * the spinlock precludes that possibility. Disabling interrupts to 1159 * avoid incoming datagrams during a "rep out" and possibly landing up 1160 * in this function. 1161 */ 1162 mtx_lock_spin(&vmci_sc->vmci_spinlock); 1163 1164 /* 1165 * Send the datagram and retrieve the return value from the result 1166 * register. 1167 */ 1168 __asm__ __volatile__( 1169 "cld\n\t" 1170 "rep outsb\n\t" 1171 : /* No output. */ 1172 : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR), 1173 "c"(VMCI_DG_SIZE(dg)), "S"(dg) 1174 ); 1175 1176 /* 1177 * XXX: Should read result high port as well when updating handlers to 1178 * return 64bit. 1179 */ 1180 1181 result = bus_space_read_4(vmci_sc->vmci_iot0, 1182 vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR); 1183 mtx_unlock_spin(&vmci_sc->vmci_spinlock); 1184 1185 return (result); 1186} 1187