misc/vmw_vmci/vmci_queue_pair.c

147 	struct vmci_queue *produce_q;
203 	struct vmci_queue *produce_q;
216 	void *produce_q;
434 	struct vmci_queue *produce_q = prod_q;
438 	if (!produce_q || !num_produce_pages || !consume_q ||
461 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
578 static void qp_init_queue_mutex(struct vmci_queue *produce_q,
586 	if (produce_q->kernel_if->host) {
587 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
588 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
589 		mutex_init(produce_q->kernel_if->mutex);
596 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
599 	if (produce_q->kernel_if->host) {
600 		produce_q->kernel_if->mutex = NULL;
606  * Acquire the mutex for the queue.  Note that the produce_q and
617  * Release the mutex for the queue.  Note that the produce_q and
652 				   struct vmci_queue *produce_q,
659 				     produce_q->kernel_if->num_pages,
661 				     produce_q->kernel_if->u.h.header_page);
662 	if (retval < (int)produce_q->kernel_if->num_pages) {
666 			qp_release_pages(produce_q->kernel_if->u.h.header_page,
682 		qp_release_pages(produce_q->kernel_if->u.h.header_page,
683 				 produce_q->kernel_if->num_pages, false);
697 					struct vmci_queue *produce_q,
710 	    produce_q->kernel_if->num_pages * PAGE_SIZE;
711 	return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
720 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
723 	qp_release_pages(produce_q->kernel_if->u.h.header_page,
724 			 produce_q->kernel_if->num_pages, true);
725 	memset(produce_q->kernel_if->u.h.header_page, 0,
726 	       sizeof(*produce_q->kernel_if->u.h.header_page) *
727 	       produce_q->kernel_if->num_pages);
743 static int qp_host_map_queues(struct vmci_queue *produce_q,
748 	if (!produce_q->q_header || !consume_q->q_header) {
751 		if (produce_q->q_header != consume_q->q_header)
754 		if (produce_q->kernel_if->u.h.header_page == NULL ||
755 		    *produce_q->kernel_if->u.h.header_page == NULL)
758 		headers[0] = *produce_q->kernel_if->u.h.header_page;
761 		produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
762 		if (produce_q->q_header != NULL) {
765 							 produce_q->q_header +
784 				struct vmci_queue *produce_q,
787 	if (produce_q->q_header) {
788 		if (produce_q->q_header < consume_q->q_header)
789 			vunmap(produce_q->q_header);
793 		produce_q->q_header = NULL;
883 			 void *produce_q,
906 		entry->produce_q = produce_q;
932 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
933 	qp_free_queue(entry->produce_q, entry->qp.produce_size);
1094 			       struct vmci_queue **produce_q,
1146 			my_consume_q = queue_pair_entry->produce_q;
1229 	*produce_q = (struct vmci_queue *)my_produce_q;
1239 		vmci_q_header_init((*produce_q)->q_header, *handle);
1357 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1358 	if (entry->produce_q == NULL) {
1368 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1382 		entry->produce_q->q_header = entry->local_mem;
1392 						      entry->produce_q,
1425 		vmci_q_header_init(entry->produce_q->q_header,
1437 		qp_host_free_queue(entry->produce_q, guest_produce_size);
1639 							      entry->produce_q,
1771 			      struct vmci_queue **produce_q,
1807 			*produce_q = entry->consume_q;
1808 			*consume_q = entry->produce_q;
1810 			*produce_q = entry->produce_q;
1830 		  struct vmci_queue **produce_q,
1841 	if (!handle || !produce_q || !consume_q ||
1846 		return qp_alloc_guest_work(handle, produce_q,
1851 		return qp_alloc_host_work(handle, produce_q,
2019 					 entry->produce_q, entry->consume_q);
2023 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2025 		qp_host_unregister_user_memory(entry->produce_q,
2060 	entry->produce_q->saved_header = NULL;
2139 		qp_acquire_queue_mutex(entry->produce_q);
2140 		headers_mapped = entry->produce_q->q_header ||
2145 						 entry->produce_q,
2152 			qp_host_unregister_user_memory(entry->produce_q,
2160 		qp_release_queue_mutex(entry->produce_q);
2178 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2179 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2255 		qp_acquire_queue_mutex(entry->produce_q);
2259 						 entry->produce_q,
2261 		qp_release_queue_mutex(entry->produce_q);
2288 	if (entry->produce_q->saved_header != NULL &&
2299 	if (NULL == entry->produce_q->q_header ||
2301 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2306 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2308 	entry->produce_q->saved_header = &entry->saved_produce_q;
2358 		qp_acquire_queue_mutex(entry->produce_q);
2364 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2373 		qp_host_unregister_user_memory(entry->produce_q,
2381 		qp_release_queue_mutex(entry->produce_q);
2431 	qp_acquire_queue_mutex(qpair->produce_q);
2440 	qp_release_queue_mutex(qpair->produce_q);
2447 static int qp_map_queue_headers(struct vmci_queue *produce_q,
2452 	if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2453 		result = qp_host_map_queues(produce_q, consume_q);
2455 			return (produce_q->saved_header &&
2476 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2478 		*produce_q_header = qpair->produce_q->q_header;
2480 	} else if (qpair->produce_q->saved_header &&
2482 		*produce_q_header = qpair->produce_q->saved_header;
2540 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2551 	result = qp_map_queue_headers(produce_q, consume_q);
2555 	free_space = vmci_q_header_free_space(produce_q->q_header,
2565 	tail = vmci_q_header_producer_tail(produce_q->q_header);
2567 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2573 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2575 			result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2588 	vmci_q_header_add_producer_tail(produce_q->q_header, written,
2606 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2618 	result = qp_map_queue_headers(produce_q, consume_q);
2623 					    produce_q->q_header,
2638 	head = vmci_q_header_consumer_head(produce_q->q_header);
2657 		vmci_q_header_add_consumer_head(produce_q->q_header,
2752 			       &my_qpair->produce_q,
3052 		result = qp_enqueue_locked(qpair->produce_q,
3096 		result = qp_dequeue_locked(qpair->produce_q,
3141 		result = qp_dequeue_locked(qpair->produce_q,
3182 		result = qp_enqueue_locked(qpair->produce_q,
3223 		result = qp_dequeue_locked(qpair->produce_q,
3265 		result = qp_dequeue_locked(qpair->produce_q,