172 	struct vmci_queue *produce_q;  member
228 	struct vmci_queue *produce_q;  member
241 	void *produce_q;  member
478 	struct vmci_queue *produce_q = prod_q;  in qp_alloc_ppn_set()  local
482 	if (!produce_q || !num_produce_pages || !consume_q ||  in qp_alloc_ppn_set()
505 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;  in qp_alloc_ppn_set()
669 static void qp_init_queue_mutex(struct vmci_queue *produce_q,  in qp_init_queue_mutex()  argument
677 	if (produce_q->kernel_if->host) {  in qp_init_queue_mutex()
678 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
679 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
680 		mutex_init(produce_q->kernel_if->mutex);  in qp_init_queue_mutex()
687 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,  in qp_cleanup_queue_mutex()  argument
690 	if (produce_q->kernel_if->host) {  in qp_cleanup_queue_mutex()
691 		produce_q->kernel_if->mutex = NULL;  in qp_cleanup_queue_mutex()
743 				   struct vmci_queue *produce_q,  in qp_host_get_user_memory()  argument
750 				     produce_q->kernel_if->num_pages, 1,  in qp_host_get_user_memory()
751 				     produce_q->kernel_if->u.h.header_page);  in qp_host_get_user_memory()
752 	if (retval < produce_q->kernel_if->num_pages) {  in qp_host_get_user_memory()
755 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
769 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
770 				 produce_q->kernel_if->num_pages, false);  in qp_host_get_user_memory()
784 					struct vmci_queue *produce_q,  in qp_host_register_user_memory()  argument
797 	    produce_q->kernel_if->num_pages * PAGE_SIZE;  in qp_host_register_user_memory()
798 	return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,  in qp_host_register_user_memory()
807 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,  in qp_host_unregister_user_memory()  argument
810 	qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_unregister_user_memory()
811 			 produce_q->kernel_if->num_pages, true);  in qp_host_unregister_user_memory()
812 	memset(produce_q->kernel_if->u.h.header_page, 0,  in qp_host_unregister_user_memory()
813 	       sizeof(*produce_q->kernel_if->u.h.header_page) *  in qp_host_unregister_user_memory()
814 	       produce_q->kernel_if->num_pages);  in qp_host_unregister_user_memory()
830 static int qp_host_map_queues(struct vmci_queue *produce_q,  in qp_host_map_queues()  argument
835 	if (!produce_q->q_header || !consume_q->q_header) {  in qp_host_map_queues()
838 		if (produce_q->q_header != consume_q->q_header)  in qp_host_map_queues()
841 		if (produce_q->kernel_if->u.h.header_page == NULL ||  in qp_host_map_queues()
842 		    *produce_q->kernel_if->u.h.header_page == NULL)  in qp_host_map_queues()
845 		headers[0] = *produce_q->kernel_if->u.h.header_page;  in qp_host_map_queues()
848 		produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);  in qp_host_map_queues()
849 		if (produce_q->q_header != NULL) {  in qp_host_map_queues()
852 							 produce_q->q_header +  in qp_host_map_queues()
871 				struct vmci_queue *produce_q,  in qp_host_unmap_queues()  argument
874 	if (produce_q->q_header) {  in qp_host_unmap_queues()
875 		if (produce_q->q_header < consume_q->q_header)  in qp_host_unmap_queues()
876 			vunmap(produce_q->q_header);  in qp_host_unmap_queues()
880 		produce_q->q_header = NULL;  in qp_host_unmap_queues()
969 			 void *produce_q,  in qp_guest_endpoint_create()  argument
992 		entry->produce_q = produce_q;  in qp_guest_endpoint_create()
1018 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in qp_guest_endpoint_destroy()
1019 	qp_free_queue(entry->produce_q, entry->qp.produce_size);  in qp_guest_endpoint_destroy()
1178 			       struct vmci_queue **produce_q,  in qp_alloc_guest_work()  argument
1230 			my_consume_q = queue_pair_entry->produce_q;  in qp_alloc_guest_work()
1313 	*produce_q = (struct vmci_queue *)my_produce_q;  in qp_alloc_guest_work()
1323 		vmci_q_header_init((*produce_q)->q_header, *handle);  in qp_alloc_guest_work()
1441 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);  in qp_broker_create()
1442 	if (entry->produce_q == NULL) {  in qp_broker_create()
1452 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);  in qp_broker_create()
1466 		entry->produce_q->q_header = entry->local_mem;  in qp_broker_create()
1476 						      entry->produce_q,  in qp_broker_create()
1509 		vmci_q_header_init(entry->produce_q->q_header,  in qp_broker_create()
1521 		qp_host_free_queue(entry->produce_q, guest_produce_size);  in qp_broker_create()
1722 							      entry->produce_q,  in qp_broker_attach()
1854 			      struct vmci_queue **produce_q,  in qp_alloc_host_work()  argument
1890 			*produce_q = entry->consume_q;  in qp_alloc_host_work()
1891 			*consume_q = entry->produce_q;  in qp_alloc_host_work()
1893 			*produce_q = entry->produce_q;  in qp_alloc_host_work()
1913 		  struct vmci_queue **produce_q,  in vmci_qp_alloc()  argument
1924 	if (!handle || !produce_q || !consume_q ||  in vmci_qp_alloc()
1929 		return qp_alloc_guest_work(handle, produce_q,  in vmci_qp_alloc()
1934 		return qp_alloc_host_work(handle, produce_q,  in vmci_qp_alloc()
2099 					 entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2103 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2105 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_set_page_store()
2140 	entry->produce_q->saved_header = NULL;  in qp_reset_saved_headers()
2219 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2220 		headers_mapped = entry->produce_q->q_header ||  in vmci_qp_broker_detach()
2225 						 entry->produce_q,  in vmci_qp_broker_detach()
2233 				qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2237 				qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2246 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2264 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in vmci_qp_broker_detach()
2265 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);  in vmci_qp_broker_detach()
2342 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2346 						 entry->produce_q,  in vmci_qp_broker_map()
2348 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2375 	if (entry->produce_q->saved_header != NULL &&  in qp_save_headers()
2386 	if (NULL == entry->produce_q->q_header ||  in qp_save_headers()
2388 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in qp_save_headers()
2393 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,  in qp_save_headers()
2395 	entry->produce_q->saved_header = &entry->saved_produce_q;  in qp_save_headers()
2447 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2453 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);  in vmci_qp_broker_unmap()
2462 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_unmap()
2470 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2520 	qp_acquire_queue_mutex(qpair->produce_q);  in qp_lock()
2529 	qp_release_queue_mutex(qpair->produce_q);  in qp_unlock()
2536 static int qp_map_queue_headers(struct vmci_queue *produce_q,  in qp_map_queue_headers()  argument
2541 	if (NULL == produce_q->q_header || NULL == consume_q->q_header) {  in qp_map_queue_headers()
2542 		result = qp_host_map_queues(produce_q, consume_q);  in qp_map_queue_headers()
2544 			return (produce_q->saved_header &&  in qp_map_queue_headers()
2565 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);  in qp_get_queue_headers()
2567 		*produce_q_header = qpair->produce_q->q_header;  in qp_get_queue_headers()
2569 	} else if (qpair->produce_q->saved_header &&  in qp_get_queue_headers()
2571 		*produce_q_header = qpair->produce_q->saved_header;  in qp_get_queue_headers()
2629 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,  in qp_enqueue_locked()  argument
2641 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_enqueue_locked()
2645 	free_space = vmci_q_header_free_space(produce_q->q_header,  in qp_enqueue_locked()
2655 	tail = vmci_q_header_producer_tail(produce_q->q_header);  in qp_enqueue_locked()
2657 		result = memcpy_to_queue(produce_q, tail, buf, 0, written);  in qp_enqueue_locked()
2663 		result = memcpy_to_queue(produce_q, tail, buf, 0, tmp);  in qp_enqueue_locked()
2665 			result = memcpy_to_queue(produce_q, 0, buf, tmp,  in qp_enqueue_locked()
2672 	vmci_q_header_add_producer_tail(produce_q->q_header, written,  in qp_enqueue_locked()
2690 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,  in qp_dequeue_locked()  argument
2703 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_dequeue_locked()
2708 					    produce_q->q_header,  in qp_dequeue_locked()
2717 	head = vmci_q_header_consumer_head(produce_q->q_header);  in qp_dequeue_locked()
2736 		vmci_q_header_add_consumer_head(produce_q->q_header,  in qp_dequeue_locked()
2832 			       &my_qpair->produce_q,  in vmci_qpair_alloc()
3128 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enqueue()
3169 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequeue()
3211 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peek()
3253 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enquev()
3295 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequev()
3339 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peekv()