1/*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7/* VMCI QueuePair API implementation. */ 8 9#include <sys/cdefs.h> 10__FBSDID("$FreeBSD$"); 11 12#include "vmci.h" 13#include "vmci_driver.h" 14#include "vmci_event.h" 15#include "vmci_kernel_api.h" 16#include "vmci_kernel_defs.h" 17#include "vmci_queue_pair.h" 18 19#define LGPFX "vmci_queue_pair: " 20 21struct queue_pair_entry { 22 vmci_list_item(queue_pair_entry) list_item; 23 struct vmci_handle handle; 24 vmci_id peer; 25 uint32_t flags; 26 uint64_t produce_size; 27 uint64_t consume_size; 28 uint32_t ref_count; 29}; 30 31struct qp_guest_endpoint { 32 struct queue_pair_entry qp; 33 uint64_t num_ppns; 34 void *produce_q; 35 void *consume_q; 36 bool hibernate_failure; 37 struct ppn_set ppn_set; 38}; 39 40struct queue_pair_list { 41 vmci_list(queue_pair_entry) head; 42 volatile int hibernate; 43 vmci_mutex mutex; 44}; 45 46#define QPE_NUM_PAGES(_QPE) \ 47 ((uint32_t)(CEILING(_QPE.produce_size, PAGE_SIZE) + \ 48 CEILING(_QPE.consume_size, PAGE_SIZE) + 2)) 49 50static struct queue_pair_list qp_guest_endpoints; 51 52static struct queue_pair_entry *queue_pair_list_find_entry( 53 struct queue_pair_list *qp_list, struct vmci_handle handle); 54static void queue_pair_list_add_entry(struct queue_pair_list *qp_list, 55 struct queue_pair_entry *entry); 56static void queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 57 struct queue_pair_entry *entry); 58static struct queue_pair_entry *queue_pair_list_get_head( 59 struct queue_pair_list *qp_list); 60static int queue_pair_notify_peer_local(bool attach, 61 struct vmci_handle handle); 62static struct qp_guest_endpoint *qp_guest_endpoint_create( 63 struct vmci_handle handle, vmci_id peer, uint32_t flags, 64 uint64_t produce_size, uint64_t consume_size, 65 void *produce_q, void *consume_q); 66static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry); 67static int vmci_queue_pair_alloc_hypercall( 68 const struct qp_guest_endpoint *entry); 69static int vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 70 struct vmci_queue **produce_q, uint64_t produce_size, 71 struct vmci_queue **consume_q, uint64_t consume_size, 72 vmci_id peer, uint32_t flags, 73 vmci_privilege_flags priv_flags); 74static int vmci_queue_pair_detach_guest_work(struct vmci_handle handle); 75static int vmci_queue_pair_detach_hypercall(struct vmci_handle handle); 76 77/* 78 *------------------------------------------------------------------------------ 79 * 80 * vmci_queue_pair_alloc -- 81 * 82 * Allocates a VMCI QueuePair. Only checks validity of input arguments. The 83 * real work is done in the host or guest specific function. 84 * 85 * Results: 86 * VMCI_SUCCESS on success, appropriate error code otherwise. 87 * 88 * Side effects: 89 * None. 90 * 91 *------------------------------------------------------------------------------ 92 */ 93 94int 95vmci_queue_pair_alloc(struct vmci_handle *handle, struct vmci_queue **produce_q, 96 uint64_t produce_size, struct vmci_queue **consume_q, uint64_t consume_size, 97 vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags) 98{ 99 100 if (!handle || !produce_q || !consume_q || 101 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS)) 102 return (VMCI_ERROR_INVALID_ARGS); 103 104 return (vmci_queue_pair_alloc_guest_work(handle, produce_q, 105 produce_size, consume_q, consume_size, peer, flags, priv_flags)); 106} 107 108/* 109 *------------------------------------------------------------------------------ 110 * 111 * vmci_queue_pair_detach -- 112 * 113 * Detaches from a VMCI QueuePair. Only checks validity of input argument. 114 * Real work is done in the host or guest specific function. 115 * 116 * Results: 117 * Success or failure. 118 * 119 * Side effects: 120 * Memory is freed. 121 * 122 *------------------------------------------------------------------------------ 123 */ 124 125int 126vmci_queue_pair_detach(struct vmci_handle handle) 127{ 128 129 if (VMCI_HANDLE_INVALID(handle)) 130 return (VMCI_ERROR_INVALID_ARGS); 131 132 return (vmci_queue_pair_detach_guest_work(handle)); 133} 134 135/* 136 *------------------------------------------------------------------------------ 137 * 138 * queue_pair_list_init -- 139 * 140 * Initializes the list of QueuePairs. 141 * 142 * Results: 143 * Success or failure. 144 * 145 * Side effects: 146 * None. 147 * 148 *------------------------------------------------------------------------------ 149 */ 150 151static inline int 152queue_pair_list_init(struct queue_pair_list *qp_list) 153{ 154 int ret; 155 156 vmci_list_init(&qp_list->head); 157 atomic_store_int(&qp_list->hibernate, 0); 158 ret = vmci_mutex_init(&qp_list->mutex, "VMCI QP List lock"); 159 return (ret); 160} 161 162/* 163 *------------------------------------------------------------------------------ 164 * 165 * queue_pair_list_destroy -- 166 * 167 * Destroy the list's mutex. 168 * 169 * Results: 170 * None. 171 * 172 * Side effects: 173 * None. 174 * 175 *------------------------------------------------------------------------------ 176 */ 177 178static inline void 179queue_pair_list_destroy(struct queue_pair_list *qp_list) 180{ 181 182 vmci_mutex_destroy(&qp_list->mutex); 183 vmci_list_init(&qp_list->head); 184} 185 186/* 187 *------------------------------------------------------------------------------ 188 * 189 * queue_pair_list_find_entry -- 190 * 191 * Finds the entry in the list corresponding to a given handle. Assumes that 192 * the list is locked. 193 * 194 * Results: 195 * Pointer to entry. 196 * 197 * Side effects: 198 * None. 199 * 200 *------------------------------------------------------------------------------ 201 */ 202 203static struct queue_pair_entry * 204queue_pair_list_find_entry(struct queue_pair_list *qp_list, 205 struct vmci_handle handle) 206{ 207 struct queue_pair_entry *next; 208 209 if (VMCI_HANDLE_INVALID(handle)) 210 return (NULL); 211 212 vmci_list_scan(next, &qp_list->head, list_item) { 213 if (VMCI_HANDLE_EQUAL(next->handle, handle)) 214 return (next); 215 } 216 217 return (NULL); 218} 219 220/* 221 *------------------------------------------------------------------------------ 222 * 223 * queue_pair_list_add_entry -- 224 * 225 * Adds the given entry to the list. Assumes that the list is locked. 226 * 227 * Results: 228 * None. 229 * 230 * Side effects: 231 * None. 232 * 233 *------------------------------------------------------------------------------ 234 */ 235 236static void 237queue_pair_list_add_entry(struct queue_pair_list *qp_list, 238 struct queue_pair_entry *entry) 239{ 240 241 if (entry) 242 vmci_list_insert(&qp_list->head, entry, list_item); 243} 244 245/* 246 *------------------------------------------------------------------------------ 247 * 248 * queue_pair_list_remove_entry -- 249 * 250 * Removes the given entry from the list. Assumes that the list is locked. 251 * 252 * Results: 253 * None. 254 * 255 * Side effects: 256 * None. 257 * 258 *------------------------------------------------------------------------------ 259 */ 260 261static void 262queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 263 struct queue_pair_entry *entry) 264{ 265 266 if (entry) 267 vmci_list_remove(entry, list_item); 268} 269 270/* 271 *------------------------------------------------------------------------------ 272 * 273 * queue_pair_list_get_head -- 274 * 275 * Returns the entry from the head of the list. Assumes that the list is 276 * locked. 277 * 278 * Results: 279 * Pointer to entry. 280 * 281 * Side effects: 282 * None. 283 * 284 *------------------------------------------------------------------------------ 285 */ 286 287static struct queue_pair_entry * 288queue_pair_list_get_head(struct queue_pair_list *qp_list) 289{ 290 291 return (vmci_list_first(&qp_list->head)); 292} 293 294/* 295 *------------------------------------------------------------------------------ 296 * 297 * vmci_qp_guest_endpoints_init -- 298 * 299 * Initalizes data structure state keeping track of queue pair guest 300 * endpoints. 301 * 302 * Results: 303 * VMCI_SUCCESS on success and appropriate failure code otherwise. 304 * 305 * Side effects: 306 * None. 307 * 308 *------------------------------------------------------------------------------ 309 */ 310 311int 312vmci_qp_guest_endpoints_init(void) 313{ 314 315 return (queue_pair_list_init(&qp_guest_endpoints)); 316} 317 318/* 319 *------------------------------------------------------------------------------ 320 * 321 * vmci_qp_guest_endpoints_exit -- 322 * 323 * Destroys all guest queue pair endpoints. If active guest queue pairs 324 * still exist, hypercalls to attempt detach from these queue pairs will be 325 * made. Any failure to detach is silently ignored. 326 * 327 * Results: 328 * None. 329 * 330 * Side effects: 331 * None. 332 * 333 *------------------------------------------------------------------------------ 334 */ 335 336void 337vmci_qp_guest_endpoints_exit(void) 338{ 339 struct qp_guest_endpoint *entry; 340 341 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 342 343 while ((entry = 344 (struct qp_guest_endpoint *)queue_pair_list_get_head( 345 &qp_guest_endpoints)) != NULL) { 346 /* 347 * Don't make a hypercall for local QueuePairs. 348 */ 349 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL)) 350 vmci_queue_pair_detach_hypercall(entry->qp.handle); 351 /* 352 * We cannot fail the exit, so let's reset ref_count. 353 */ 354 entry->qp.ref_count = 0; 355 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 356 qp_guest_endpoint_destroy(entry); 357 } 358 359 atomic_store_int(&qp_guest_endpoints.hibernate, 0); 360 vmci_mutex_release(&qp_guest_endpoints.mutex); 361 queue_pair_list_destroy(&qp_guest_endpoints); 362} 363 364/* 365 *------------------------------------------------------------------------------ 366 * 367 * vmci_qp_guest_endpoints_sync -- 368 * 369 * Use this as a synchronization point when setting globals, for example, 370 * during device shutdown. 371 * 372 * Results: 373 * true. 374 * 375 * Side effects: 376 * None. 377 * 378 *------------------------------------------------------------------------------ 379 */ 380 381void 382vmci_qp_guest_endpoints_sync(void) 383{ 384 385 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 386 vmci_mutex_release(&qp_guest_endpoints.mutex); 387} 388 389/* 390 *------------------------------------------------------------------------------ 391 * 392 * qp_guest_endpoint_create -- 393 * 394 * Allocates and initializes a qp_guest_endpoint structure. Allocates a 395 * QueuePair rid (and handle) iff the given entry has an invalid handle. 396 * 0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved handles. Assumes 397 * that the QP list mutex is held by the caller. 398 * 399 * Results: 400 * Pointer to structure intialized. 401 * 402 * Side effects: 403 * None. 404 * 405 *------------------------------------------------------------------------------ 406 */ 407 408struct qp_guest_endpoint * 409qp_guest_endpoint_create(struct vmci_handle handle, vmci_id peer, 410 uint32_t flags, uint64_t produce_size, uint64_t consume_size, 411 void *produce_q, void *consume_q) 412{ 413 struct qp_guest_endpoint *entry; 414 static vmci_id queue_pair_rid; 415 const uint64_t num_ppns = CEILING(produce_size, PAGE_SIZE) + 416 CEILING(consume_size, PAGE_SIZE) + 417 2; /* One page each for the queue headers. */ 418 419 queue_pair_rid = VMCI_RESERVED_RESOURCE_ID_MAX + 1; 420 421 ASSERT((produce_size || consume_size) && produce_q && consume_q); 422 423 if (VMCI_HANDLE_INVALID(handle)) { 424 vmci_id context_id = vmci_get_context_id(); 425 vmci_id old_rid = queue_pair_rid; 426 427 /* 428 * Generate a unique QueuePair rid. Keep on trying until we 429 * wrap around in the RID space. 430 */ 431 ASSERT(old_rid > VMCI_RESERVED_RESOURCE_ID_MAX); 432 do { 433 handle = VMCI_MAKE_HANDLE(context_id, queue_pair_rid); 434 entry = 435 (struct qp_guest_endpoint *) 436 queue_pair_list_find_entry(&qp_guest_endpoints, 437 handle); 438 queue_pair_rid++; 439 if (UNLIKELY(!queue_pair_rid)) { 440 /* 441 * Skip the reserved rids. 442 */ 443 queue_pair_rid = 444 VMCI_RESERVED_RESOURCE_ID_MAX + 1; 445 } 446 } while (entry && queue_pair_rid != old_rid); 447 448 if (UNLIKELY(entry != NULL)) { 449 ASSERT(queue_pair_rid == old_rid); 450 /* 451 * We wrapped around --- no rids were free. 452 */ 453 return (NULL); 454 } 455 } 456 457 ASSERT(!VMCI_HANDLE_INVALID(handle) && 458 queue_pair_list_find_entry(&qp_guest_endpoints, handle) == NULL); 459 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL); 460 if (entry) { 461 entry->qp.handle = handle; 462 entry->qp.peer = peer; 463 entry->qp.flags = flags; 464 entry->qp.produce_size = produce_size; 465 entry->qp.consume_size = consume_size; 466 entry->qp.ref_count = 0; 467 entry->num_ppns = num_ppns; 468 memset(&entry->ppn_set, 0, sizeof(entry->ppn_set)); 469 entry->produce_q = produce_q; 470 entry->consume_q = consume_q; 471 } 472 return (entry); 473} 474 475/* 476 *------------------------------------------------------------------------------ 477 * 478 * qp_guest_endpoint_destroy -- 479 * 480 * Frees a qp_guest_endpoint structure. 481 * 482 * Results: 483 * None. 484 * 485 * Side effects: 486 * None. 487 * 488 *------------------------------------------------------------------------------ 489 */ 490 491void 492qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry) 493{ 494 495 ASSERT(entry); 496 ASSERT(entry->qp.ref_count == 0); 497 498 vmci_free_ppn_set(&entry->ppn_set); 499 vmci_free_queue(entry->produce_q, entry->qp.produce_size); 500 vmci_free_queue(entry->consume_q, entry->qp.consume_size); 501 vmci_free_kernel_mem(entry, sizeof(*entry)); 502} 503 504/* 505 *------------------------------------------------------------------------------ 506 * 507 * vmci_queue_pair_alloc_hypercall -- 508 * 509 * Helper to make a QueuePairAlloc hypercall when the driver is 510 * supporting a guest device. 511 * 512 * Results: 513 * Result of the hypercall. 514 * 515 * Side effects: 516 * Memory is allocated & freed. 517 * 518 *------------------------------------------------------------------------------ 519 */ 520static int 521vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint *entry) 522{ 523 struct vmci_queue_pair_alloc_msg *alloc_msg; 524 size_t msg_size; 525 int result; 526 527 if (!entry || entry->num_ppns <= 2) 528 return (VMCI_ERROR_INVALID_ARGS); 529 530 ASSERT(!(entry->qp.flags & VMCI_QPFLAG_LOCAL)); 531 532 msg_size = sizeof(*alloc_msg) + (size_t)entry->num_ppns * sizeof(PPN); 533 alloc_msg = vmci_alloc_kernel_mem(msg_size, VMCI_MEMORY_NORMAL); 534 if (!alloc_msg) 535 return (VMCI_ERROR_NO_MEM); 536 537 alloc_msg->hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 538 VMCI_QUEUEPAIR_ALLOC); 539 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE; 540 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE; 541 alloc_msg->handle = entry->qp.handle; 542 alloc_msg->peer = entry->qp.peer; 543 alloc_msg->flags = entry->qp.flags; 544 alloc_msg->produce_size = entry->qp.produce_size; 545 alloc_msg->consume_size = entry->qp.consume_size; 546 alloc_msg->num_ppns = entry->num_ppns; 547 result = vmci_populate_ppn_list((uint8_t *)alloc_msg + 548 sizeof(*alloc_msg), &entry->ppn_set); 549 if (result == VMCI_SUCCESS) 550 result = vmci_send_datagram((struct vmci_datagram *)alloc_msg); 551 vmci_free_kernel_mem(alloc_msg, msg_size); 552 553 return (result); 554} 555 556/* 557 *------------------------------------------------------------------------------ 558 * 559 * vmci_queue_pair_alloc_guest_work -- 560 * 561 * This functions handles the actual allocation of a VMCI queue pair guest 562 * endpoint. Allocates physical pages for the queue pair. It makes OS 563 * dependent calls through generic wrappers. 564 * 565 * Results: 566 * Success or failure. 567 * 568 * Side effects: 569 * Memory is allocated. 570 * 571 *------------------------------------------------------------------------------ 572 */ 573 574static int 575vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 576 struct vmci_queue **produce_q, uint64_t produce_size, 577 struct vmci_queue **consume_q, uint64_t consume_size, vmci_id peer, 578 uint32_t flags, vmci_privilege_flags priv_flags) 579{ 580 struct qp_guest_endpoint *queue_pair_entry = NULL; 581 void *my_consume_q = NULL; 582 void *my_produce_q = NULL; 583 const uint64_t num_consume_pages = CEILING(consume_size, PAGE_SIZE) + 1; 584 const uint64_t num_produce_pages = CEILING(produce_size, PAGE_SIZE) + 1; 585 int result; 586 587 ASSERT(handle && produce_q && consume_q && 588 (produce_size || consume_size)); 589 590 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS) 591 return (VMCI_ERROR_NO_ACCESS); 592 593 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 594 595 if ((atomic_load_int(&qp_guest_endpoints.hibernate) == 1) && 596 !(flags & VMCI_QPFLAG_LOCAL)) { 597 /* 598 * While guest OS is in hibernate state, creating non-local 599 * queue pairs is not allowed after the point where the VMCI 600 * guest driver converted the existing queue pairs to local 601 * ones. 602 */ 603 604 result = VMCI_ERROR_UNAVAILABLE; 605 goto error; 606 } 607 608 if ((queue_pair_entry = 609 (struct qp_guest_endpoint *)queue_pair_list_find_entry( 610 &qp_guest_endpoints, *handle)) != NULL) { 611 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 612 /* Local attach case. */ 613 if (queue_pair_entry->qp.ref_count > 1) { 614 VMCI_LOG_DEBUG(LGPFX"Error attempting to " 615 "attach more than once.\n"); 616 result = VMCI_ERROR_UNAVAILABLE; 617 goto error_keep_entry; 618 } 619 620 if (queue_pair_entry->qp.produce_size != consume_size || 621 queue_pair_entry->qp.consume_size != produce_size || 622 queue_pair_entry->qp.flags != 623 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) { 624 VMCI_LOG_DEBUG(LGPFX"Error mismatched " 625 "queue pair in local attach.\n"); 626 result = VMCI_ERROR_QUEUEPAIR_MISMATCH; 627 goto error_keep_entry; 628 } 629 630 /* 631 * Do a local attach. We swap the consume and produce 632 * queues for the attacher and deliver an attach event. 633 */ 634 result = queue_pair_notify_peer_local(true, *handle); 635 if (result < VMCI_SUCCESS) 636 goto error_keep_entry; 637 my_produce_q = queue_pair_entry->consume_q; 638 my_consume_q = queue_pair_entry->produce_q; 639 goto out; 640 } 641 result = VMCI_ERROR_ALREADY_EXISTS; 642 goto error_keep_entry; 643 } 644 645 my_produce_q = vmci_alloc_queue(produce_size, flags); 646 if (!my_produce_q) { 647 VMCI_LOG_WARNING(LGPFX"Error allocating pages for produce " 648 "queue.\n"); 649 result = VMCI_ERROR_NO_MEM; 650 goto error; 651 } 652 653 my_consume_q = vmci_alloc_queue(consume_size, flags); 654 if (!my_consume_q) { 655 VMCI_LOG_WARNING(LGPFX"Error allocating pages for consume " 656 "queue.\n"); 657 result = VMCI_ERROR_NO_MEM; 658 goto error; 659 } 660 661 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags, 662 produce_size, consume_size, my_produce_q, my_consume_q); 663 if (!queue_pair_entry) { 664 VMCI_LOG_WARNING(LGPFX"Error allocating memory in %s.\n", 665 __FUNCTION__); 666 result = VMCI_ERROR_NO_MEM; 667 goto error; 668 } 669 670 result = vmci_alloc_ppn_set(my_produce_q, num_produce_pages, 671 my_consume_q, num_consume_pages, &queue_pair_entry->ppn_set); 672 if (result < VMCI_SUCCESS) { 673 VMCI_LOG_WARNING(LGPFX"vmci_alloc_ppn_set failed.\n"); 674 goto error; 675 } 676 677 /* 678 * It's only necessary to notify the host if this queue pair will be 679 * attached to from another context. 680 */ 681 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 682 /* Local create case. */ 683 vmci_id context_id = vmci_get_context_id(); 684 685 /* 686 * Enforce similar checks on local queue pairs as we do for 687 * regular ones. The handle's context must match the creator 688 * or attacher context id (here they are both the current 689 * context id) and the attach-only flag cannot exist during 690 * create. We also ensure specified peer is this context or 691 * an invalid one. 692 */ 693 if (queue_pair_entry->qp.handle.context != context_id || 694 (queue_pair_entry->qp.peer != VMCI_INVALID_ID && 695 queue_pair_entry->qp.peer != context_id)) { 696 result = VMCI_ERROR_NO_ACCESS; 697 goto error; 698 } 699 700 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) { 701 result = VMCI_ERROR_NOT_FOUND; 702 goto error; 703 } 704 } else { 705 result = vmci_queue_pair_alloc_hypercall(queue_pair_entry); 706 if (result < VMCI_SUCCESS) { 707 VMCI_LOG_WARNING( 708 LGPFX"vmci_queue_pair_alloc_hypercall result = " 709 "%d.\n", result); 710 goto error; 711 } 712 } 713 714 queue_pair_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp); 715 716out: 717 queue_pair_entry->qp.ref_count++; 718 *handle = queue_pair_entry->qp.handle; 719 *produce_q = (struct vmci_queue *)my_produce_q; 720 *consume_q = (struct vmci_queue *)my_consume_q; 721 722 /* 723 * We should initialize the queue pair header pages on a local queue 724 * pair create. For non-local queue pairs, the hypervisor initializes 725 * the header pages in the create step. 726 */ 727 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) && 728 queue_pair_entry->qp.ref_count == 1) { 729 vmci_queue_header_init((*produce_q)->q_header, *handle); 730 vmci_queue_header_init((*consume_q)->q_header, *handle); 731 } 732 733 vmci_mutex_release(&qp_guest_endpoints.mutex); 734 735 return (VMCI_SUCCESS); 736 737error: 738 vmci_mutex_release(&qp_guest_endpoints.mutex); 739 if (queue_pair_entry) { 740 /* The queues will be freed inside the destroy routine. */ 741 qp_guest_endpoint_destroy(queue_pair_entry); 742 } else { 743 if (my_produce_q) 744 vmci_free_queue(my_produce_q, produce_size); 745 if (my_consume_q) 746 vmci_free_queue(my_consume_q, consume_size); 747 } 748 return (result); 749 750error_keep_entry: 751 /* This path should only be used when an existing entry was found. */ 752 ASSERT(queue_pair_entry->qp.ref_count > 0); 753 vmci_mutex_release(&qp_guest_endpoints.mutex); 754 return (result); 755} 756 757/* 758 *------------------------------------------------------------------------------ 759 * 760 * vmci_queue_pair_detach_hypercall -- 761 * 762 * Helper to make a QueuePairDetach hypercall when the driver is supporting 763 * a guest device. 764 * 765 * Results: 766 * Result of the hypercall. 767 * 768 * Side effects: 769 * None. 770 * 771 *------------------------------------------------------------------------------ 772 */ 773 774int 775vmci_queue_pair_detach_hypercall(struct vmci_handle handle) 776{ 777 struct vmci_queue_pair_detach_msg detach_msg; 778 779 detach_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 780 VMCI_QUEUEPAIR_DETACH); 781 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 782 detach_msg.hdr.payload_size = sizeof(handle); 783 detach_msg.handle = handle; 784 785 return (vmci_send_datagram((struct vmci_datagram *)&detach_msg)); 786} 787 788/* 789 *------------------------------------------------------------------------------ 790 * 791 * vmci_queue_pair_detach_guest_work -- 792 * 793 * Helper for VMCI QueuePair detach interface. Frees the physical pages for 794 * the queue pair. 795 * 796 * Results: 797 * Success or failure. 798 * 799 * Side effects: 800 * Memory may be freed. 801 * 802 *------------------------------------------------------------------------------ 803 */ 804 805static int 806vmci_queue_pair_detach_guest_work(struct vmci_handle handle) 807{ 808 struct qp_guest_endpoint *entry; 809 int result; 810 uint32_t ref_count; 811 812 ASSERT(!VMCI_HANDLE_INVALID(handle)); 813 814 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 815 816 entry = (struct qp_guest_endpoint *)queue_pair_list_find_entry( 817 &qp_guest_endpoints, handle); 818 if (!entry) { 819 vmci_mutex_release(&qp_guest_endpoints.mutex); 820 return (VMCI_ERROR_NOT_FOUND); 821 } 822 823 ASSERT(entry->qp.ref_count >= 1); 824 825 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) { 826 result = VMCI_SUCCESS; 827 828 if (entry->qp.ref_count > 1) { 829 result = queue_pair_notify_peer_local(false, handle); 830 831 /* 832 * We can fail to notify a local queuepair because we 833 * can't allocate. We still want to release the entry 834 * if that happens, so don't bail out yet. 835 */ 836 } 837 } else { 838 result = vmci_queue_pair_detach_hypercall(handle); 839 if (entry->hibernate_failure) { 840 if (result == VMCI_ERROR_NOT_FOUND) { 841 /* 842 * If a queue pair detach failed when entering 843 * hibernation, the guest driver and the device 844 * may disagree on its existence when coming 845 * out of hibernation. The guest driver will 846 * regard it as a non-local queue pair, but 847 * the device state is gone, since the device 848 * has been powered off. In this case, we 849 * treat the queue pair as a local queue pair 850 * with no peer. 851 */ 852 853 ASSERT(entry->qp.ref_count == 1); 854 result = VMCI_SUCCESS; 855 } 856 } 857 if (result < VMCI_SUCCESS) { 858 /* 859 * We failed to notify a non-local queuepair. That other 860 * queuepair might still be accessing the shared 861 * memory, so don't release the entry yet. It will get 862 * cleaned up by vmci_queue_pair_Exit() if necessary 863 * (assuming we are going away, otherwise why did this 864 * fail?). 865 */ 866 867 vmci_mutex_release(&qp_guest_endpoints.mutex); 868 return (result); 869 } 870 } 871 872 /* 873 * If we get here then we either failed to notify a local queuepair, or 874 * we succeeded in all cases. Release the entry if required. 875 */ 876 877 entry->qp.ref_count--; 878 if (entry->qp.ref_count == 0) 879 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 880 881 /* If we didn't remove the entry, this could change once we unlock. */ 882 ref_count = entry ? entry->qp.ref_count : 883 0xffffffff; /* 884 * Value does not matter, silence the 885 * compiler. 886 */ 887 888 vmci_mutex_release(&qp_guest_endpoints.mutex); 889 890 if (ref_count == 0) 891 qp_guest_endpoint_destroy(entry); 892 return (result); 893} 894 895/* 896 *------------------------------------------------------------------------------ 897 * 898 * queue_pair_notify_peer_local -- 899 * 900 * Dispatches a queue pair event message directly into the local event 901 * queue. 902 * 903 * Results: 904 * VMCI_SUCCESS on success, error code otherwise 905 * 906 * Side effects: 907 * None. 908 * 909 *------------------------------------------------------------------------------ 910 */ 911 912static int 913queue_pair_notify_peer_local(bool attach, struct vmci_handle handle) 914{ 915 struct vmci_event_msg *e_msg; 916 struct vmci_event_payload_qp *e_payload; 917 /* buf is only 48 bytes. */ 918 vmci_id context_id; 919 context_id = vmci_get_context_id(); 920 char buf[sizeof(*e_msg) + sizeof(*e_payload)]; 921 922 e_msg = (struct vmci_event_msg *)buf; 923 e_payload = vmci_event_msg_payload(e_msg); 924 925 e_msg->hdr.dst = VMCI_MAKE_HANDLE(context_id, VMCI_EVENT_HANDLER); 926 e_msg->hdr.src = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 927 VMCI_CONTEXT_RESOURCE_ID); 928 e_msg->hdr.payload_size = sizeof(*e_msg) + sizeof(*e_payload) - 929 sizeof(e_msg->hdr); 930 e_msg->event_data.event = attach ? VMCI_EVENT_QP_PEER_ATTACH : 931 VMCI_EVENT_QP_PEER_DETACH; 932 e_payload->peer_id = context_id; 933 e_payload->handle = handle; 934 935 return (vmci_event_dispatch((struct vmci_datagram *)e_msg)); 936} 937