894 	struct qp_entry *entry;  in qp_list_find()  local
899 	list_for_each_entry(entry, &qp_list->head, list_item) {  in qp_list_find()
900 		if (vmci_handle_is_equal(entry->handle, handle))  in qp_list_find()
901 			return entry;  in qp_list_find()
913 	struct qp_guest_endpoint *entry;  in qp_guest_handle_to_entry()  local
916 	entry = qp ? container_of(  in qp_guest_handle_to_entry()
918 	return entry;  in qp_guest_handle_to_entry()
927 	struct qp_broker_entry *entry;  in qp_broker_handle_to_entry()  local
930 	entry = qp ? container_of(  in qp_broker_handle_to_entry()
932 	return entry;  in qp_broker_handle_to_entry()
973 	struct qp_guest_endpoint *entry;  in qp_guest_endpoint_create()  local
984 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);  in qp_guest_endpoint_create()
985 	if (entry) {  in qp_guest_endpoint_create()
986 		entry->qp.peer = peer;  in qp_guest_endpoint_create()
987 		entry->qp.flags = flags;  in qp_guest_endpoint_create()
988 		entry->qp.produce_size = produce_size;  in qp_guest_endpoint_create()
989 		entry->qp.consume_size = consume_size;  in qp_guest_endpoint_create()
990 		entry->qp.ref_count = 0;  in qp_guest_endpoint_create()
991 		entry->num_ppns = num_ppns;  in qp_guest_endpoint_create()
992 		entry->produce_q = produce_q;  in qp_guest_endpoint_create()
993 		entry->consume_q = consume_q;  in qp_guest_endpoint_create()
994 		INIT_LIST_HEAD(&entry->qp.list_item);  in qp_guest_endpoint_create()
997 		result = vmci_resource_add(&entry->resource,  in qp_guest_endpoint_create()
1000 		entry->qp.handle = vmci_resource_handle(&entry->resource);  in qp_guest_endpoint_create()
1002 		    qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {  in qp_guest_endpoint_create()
1005 			kfree(entry);  in qp_guest_endpoint_create()
1006 			entry = NULL;  in qp_guest_endpoint_create()
1009 	return entry;  in qp_guest_endpoint_create()
1015 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)  in qp_guest_endpoint_destroy()  argument
1017 	qp_free_ppn_set(&entry->ppn_set);  in qp_guest_endpoint_destroy()
1018 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in qp_guest_endpoint_destroy()
1019 	qp_free_queue(entry->produce_q, entry->qp.produce_size);  in qp_guest_endpoint_destroy()
1020 	qp_free_queue(entry->consume_q, entry->qp.consume_size);  in qp_guest_endpoint_destroy()
1022 	vmci_resource_remove(&entry->resource);  in qp_guest_endpoint_destroy()
1024 	kfree(entry);  in qp_guest_endpoint_destroy()
1031 static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)  in qp_alloc_hypercall()  argument
1037 	if (!entry || entry->num_ppns <= 2)  in qp_alloc_hypercall()
1041 	    (size_t) entry->num_ppns * sizeof(u32);  in qp_alloc_hypercall()
1050 	alloc_msg->handle = entry->qp.handle;  in qp_alloc_hypercall()
1051 	alloc_msg->peer = entry->qp.peer;  in qp_alloc_hypercall()
1052 	alloc_msg->flags = entry->qp.flags;  in qp_alloc_hypercall()
1053 	alloc_msg->produce_size = entry->qp.produce_size;  in qp_alloc_hypercall()
1054 	alloc_msg->consume_size = entry->qp.consume_size;  in qp_alloc_hypercall()
1055 	alloc_msg->num_ppns = entry->num_ppns;  in qp_alloc_hypercall()
1058 				     &entry->ppn_set);  in qp_alloc_hypercall()
1087 static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)  in qp_list_add_entry()  argument
1089 	if (entry)  in qp_list_add_entry()
1090 		list_add(&entry->list_item, &qp_list->head);  in qp_list_add_entry()
1097 				 struct qp_entry *entry)  in qp_list_remove_entry()  argument
1099 	if (entry)  in qp_list_remove_entry()
1100 		list_del(&entry->list_item);  in qp_list_remove_entry()
1110 	struct qp_guest_endpoint *entry;  in qp_detatch_guest_work()  local
1115 	entry = qp_guest_handle_to_entry(handle);  in qp_detatch_guest_work()
1116 	if (!entry) {  in qp_detatch_guest_work()
1121 	if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {  in qp_detatch_guest_work()
1124 		if (entry->qp.ref_count > 1) {  in qp_detatch_guest_work()
1156 	entry->qp.ref_count--;  in qp_detatch_guest_work()
1157 	if (entry->qp.ref_count == 0)  in qp_detatch_guest_work()
1158 		qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);  in qp_detatch_guest_work()
1161 	if (entry)  in qp_detatch_guest_work()
1162 		ref_count = entry->qp.ref_count;  in qp_detatch_guest_work()
1167 		qp_guest_endpoint_destroy(entry);  in qp_detatch_guest_work()
1377 	struct qp_broker_entry *entry = NULL;  in qp_broker_create()  local
1405 	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);  in qp_broker_create()
1406 	if (!entry)  in qp_broker_create()
1425 	entry->qp.handle = handle;  in qp_broker_create()
1426 	entry->qp.peer = peer;  in qp_broker_create()
1427 	entry->qp.flags = flags;  in qp_broker_create()
1428 	entry->qp.produce_size = guest_produce_size;  in qp_broker_create()
1429 	entry->qp.consume_size = guest_consume_size;  in qp_broker_create()
1430 	entry->qp.ref_count = 1;  in qp_broker_create()
1431 	entry->create_id = context_id;  in qp_broker_create()
1432 	entry->attach_id = VMCI_INVALID_ID;  in qp_broker_create()
1433 	entry->state = VMCIQPB_NEW;  in qp_broker_create()
1434 	entry->require_trusted_attach =  in qp_broker_create()
1436 	entry->created_by_trusted =  in qp_broker_create()
1438 	entry->vmci_page_files = false;  in qp_broker_create()
1439 	entry->wakeup_cb = wakeup_cb;  in qp_broker_create()
1440 	entry->client_data = client_data;  in qp_broker_create()
1441 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);  in qp_broker_create()
1442 	if (entry->produce_q == NULL) {  in qp_broker_create()
1446 	entry->consume_q = qp_host_alloc_queue(guest_consume_size);  in qp_broker_create()
1447 	if (entry->consume_q == NULL) {  in qp_broker_create()
1452 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);  in qp_broker_create()
1454 	INIT_LIST_HEAD(&entry->qp.list_item);  in qp_broker_create()
1459 		entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),  in qp_broker_create()
1461 		if (entry->local_mem == NULL) {  in qp_broker_create()
1465 		entry->state = VMCIQPB_CREATED_MEM;  in qp_broker_create()
1466 		entry->produce_q->q_header = entry->local_mem;  in qp_broker_create()
1467 		tmp = (u8 *)entry->local_mem + PAGE_SIZE *  in qp_broker_create()
1468 		    (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);  in qp_broker_create()
1469 		entry->consume_q->q_header = (struct vmci_queue_header *)tmp;  in qp_broker_create()
1476 						      entry->produce_q,  in qp_broker_create()
1477 						      entry->consume_q);  in qp_broker_create()
1481 		entry->state = VMCIQPB_CREATED_MEM;  in qp_broker_create()
1490 		entry->state = VMCIQPB_CREATED_NO_MEM;  in qp_broker_create()
1493 	qp_list_add_entry(&qp_broker_list, &entry->qp);  in qp_broker_create()
1495 		*ent = entry;  in qp_broker_create()
1498 	result = vmci_resource_add(&entry->resource,  in qp_broker_create()
1507 	entry->qp.handle = vmci_resource_handle(&entry->resource);  in qp_broker_create()
1509 		vmci_q_header_init(entry->produce_q->q_header,  in qp_broker_create()
1510 				   entry->qp.handle);  in qp_broker_create()
1511 		vmci_q_header_init(entry->consume_q->q_header,  in qp_broker_create()
1512 				   entry->qp.handle);  in qp_broker_create()
1515 	vmci_ctx_qp_create(context, entry->qp.handle);  in qp_broker_create()
1520 	if (entry != NULL) {  in qp_broker_create()
1521 		qp_host_free_queue(entry->produce_q, guest_produce_size);  in qp_broker_create()
1522 		qp_host_free_queue(entry->consume_q, guest_consume_size);  in qp_broker_create()
1523 		kfree(entry);  in qp_broker_create()
1594 static int qp_broker_attach(struct qp_broker_entry *entry,  in qp_broker_attach()  argument
1610 	if (entry->state != VMCIQPB_CREATED_NO_MEM &&  in qp_broker_attach()
1611 	    entry->state != VMCIQPB_CREATED_MEM)  in qp_broker_attach()
1615 		if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||  in qp_broker_attach()
1616 		    context_id != entry->create_id) {  in qp_broker_attach()
1619 	} else if (context_id == entry->create_id ||  in qp_broker_attach()
1620 		   context_id == entry->attach_id) {  in qp_broker_attach()
1625 	    VMCI_CONTEXT_IS_VM(entry->create_id))  in qp_broker_attach()
1633 	    !entry->created_by_trusted)  in qp_broker_attach()
1640 	if (entry->require_trusted_attach &&  in qp_broker_attach()
1648 	if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)  in qp_broker_attach()
1651 	if (entry->create_id == VMCI_HOST_CONTEXT_ID) {  in qp_broker_attach()
1669 		create_context = vmci_ctx_get(entry->create_id);  in qp_broker_attach()
1677 	if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))  in qp_broker_attach()
1687 		if (entry->qp.produce_size != produce_size ||  in qp_broker_attach()
1688 		    entry->qp.consume_size != consume_size) {  in qp_broker_attach()
1691 	} else if (entry->qp.produce_size != consume_size ||  in qp_broker_attach()
1692 		   entry->qp.consume_size != produce_size) {  in qp_broker_attach()
1710 		if (entry->state != VMCIQPB_CREATED_NO_MEM)  in qp_broker_attach()
1722 							      entry->produce_q,  in qp_broker_attach()
1723 							      entry->consume_q);  in qp_broker_attach()
1727 			entry->state = VMCIQPB_ATTACHED_MEM;  in qp_broker_attach()
1729 			entry->state = VMCIQPB_ATTACHED_NO_MEM;  in qp_broker_attach()
1731 	} else if (entry->state == VMCIQPB_CREATED_NO_MEM) {  in qp_broker_attach()
1742 		entry->state = VMCIQPB_ATTACHED_MEM;  in qp_broker_attach()
1745 	if (entry->state == VMCIQPB_ATTACHED_MEM) {  in qp_broker_attach()
1747 		    qp_notify_peer(true, entry->qp.handle, context_id,  in qp_broker_attach()
1748 				   entry->create_id);  in qp_broker_attach()
1751 				entry->create_id, entry->qp.handle.context,  in qp_broker_attach()
1752 				entry->qp.handle.resource);  in qp_broker_attach()
1755 	entry->attach_id = context_id;  in qp_broker_attach()
1756 	entry->qp.ref_count++;  in qp_broker_attach()
1758 		entry->wakeup_cb = wakeup_cb;  in qp_broker_attach()
1759 		entry->client_data = client_data;  in qp_broker_attach()
1767 		vmci_ctx_qp_create(context, entry->qp.handle);  in qp_broker_attach()
1770 		*ent = entry;  in qp_broker_attach()
1794 	struct qp_broker_entry *entry = NULL;  in qp_broker_alloc()  local
1824 		entry = qp_broker_handle_to_entry(handle);  in qp_broker_alloc()
1826 	if (!entry) {  in qp_broker_alloc()
1835 		    qp_broker_attach(entry, peer, flags, priv_flags,  in qp_broker_alloc()
1866 	struct qp_broker_entry *entry;  in qp_alloc_host_work()  local
1877 	entry = NULL;  in qp_alloc_host_work()
1881 			    wakeup_cb, client_data, &entry, &swap);  in qp_alloc_host_work()
1890 			*produce_q = entry->consume_q;  in qp_alloc_host_work()
1891 			*consume_q = entry->produce_q;  in qp_alloc_host_work()
1893 			*produce_q = entry->produce_q;  in qp_alloc_host_work()
1894 			*consume_q = entry->consume_q;  in qp_alloc_host_work()
1897 		*handle = vmci_resource_handle(&entry->resource);  in qp_alloc_host_work()
1980 		struct qp_entry *entry =  in qp_list_get_head()  local
1983 		return entry;  in qp_list_get_head()
1991 	struct qp_entry *entry;  in vmci_qp_broker_exit()  local
1996 	while ((entry = qp_list_get_head(&qp_broker_list))) {  in vmci_qp_broker_exit()
1997 		be = (struct qp_broker_entry *)entry;  in vmci_qp_broker_exit()
1999 		qp_list_remove_entry(&qp_broker_list, entry);  in vmci_qp_broker_exit()
2048 	struct qp_broker_entry *entry;  in vmci_qp_broker_set_page_store()  local
2073 	entry = qp_broker_handle_to_entry(handle);  in vmci_qp_broker_set_page_store()
2074 	if (!entry) {  in vmci_qp_broker_set_page_store()
2085 	if (entry->create_id != context_id &&  in vmci_qp_broker_set_page_store()
2086 	    (entry->create_id != VMCI_HOST_CONTEXT_ID ||  in vmci_qp_broker_set_page_store()
2087 	     entry->attach_id != context_id)) {  in vmci_qp_broker_set_page_store()
2092 	if (entry->state != VMCIQPB_CREATED_NO_MEM &&  in vmci_qp_broker_set_page_store()
2093 	    entry->state != VMCIQPB_ATTACHED_NO_MEM) {  in vmci_qp_broker_set_page_store()
2099 					 entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2103 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2105 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_set_page_store()
2106 					       entry->consume_q);  in vmci_qp_broker_set_page_store()
2110 	if (entry->state == VMCIQPB_CREATED_NO_MEM)  in vmci_qp_broker_set_page_store()
2111 		entry->state = VMCIQPB_CREATED_MEM;  in vmci_qp_broker_set_page_store()
2113 		entry->state = VMCIQPB_ATTACHED_MEM;  in vmci_qp_broker_set_page_store()
2115 	entry->vmci_page_files = true;  in vmci_qp_broker_set_page_store()
2117 	if (entry->state == VMCIQPB_ATTACHED_MEM) {  in vmci_qp_broker_set_page_store()
2119 		    qp_notify_peer(true, handle, context_id, entry->create_id);  in vmci_qp_broker_set_page_store()
2122 				entry->create_id, entry->qp.handle.context,  in vmci_qp_broker_set_page_store()
2123 				entry->qp.handle.resource);  in vmci_qp_broker_set_page_store()
2138 static void qp_reset_saved_headers(struct qp_broker_entry *entry)  in qp_reset_saved_headers()  argument
2140 	entry->produce_q->saved_header = NULL;  in qp_reset_saved_headers()
2141 	entry->consume_q->saved_header = NULL;  in qp_reset_saved_headers()
2164 	struct qp_broker_entry *entry;  in vmci_qp_broker_detach()  local
2184 	entry = qp_broker_handle_to_entry(handle);  in vmci_qp_broker_detach()
2185 	if (!entry) {  in vmci_qp_broker_detach()
2192 	if (context_id != entry->create_id && context_id != entry->attach_id) {  in vmci_qp_broker_detach()
2197 	if (context_id == entry->create_id) {  in vmci_qp_broker_detach()
2198 		peer_id = entry->attach_id;  in vmci_qp_broker_detach()
2199 		entry->create_id = VMCI_INVALID_ID;  in vmci_qp_broker_detach()
2201 		peer_id = entry->create_id;  in vmci_qp_broker_detach()
2202 		entry->attach_id = VMCI_INVALID_ID;  in vmci_qp_broker_detach()
2204 	entry->qp.ref_count--;  in vmci_qp_broker_detach()
2206 	is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;  in vmci_qp_broker_detach()
2219 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2220 		headers_mapped = entry->produce_q->q_header ||  in vmci_qp_broker_detach()
2221 		    entry->consume_q->q_header;  in vmci_qp_broker_detach()
2222 		if (QPBROKERSTATE_HAS_MEM(entry)) {  in vmci_qp_broker_detach()
2225 						 entry->produce_q,  in vmci_qp_broker_detach()
2226 						 entry->consume_q);  in vmci_qp_broker_detach()
2232 			if (entry->vmci_page_files)  in vmci_qp_broker_detach()
2233 				qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2234 							       entry->  in vmci_qp_broker_detach()
2237 				qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2238 							       entry->  in vmci_qp_broker_detach()
2244 			qp_reset_saved_headers(entry);  in vmci_qp_broker_detach()
2246 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2248 		if (!headers_mapped && entry->wakeup_cb)  in vmci_qp_broker_detach()
2249 			entry->wakeup_cb(entry->client_data);  in vmci_qp_broker_detach()
2252 		if (entry->wakeup_cb) {  in vmci_qp_broker_detach()
2253 			entry->wakeup_cb = NULL;  in vmci_qp_broker_detach()
2254 			entry->client_data = NULL;  in vmci_qp_broker_detach()
2258 	if (entry->qp.ref_count == 0) {  in vmci_qp_broker_detach()
2259 		qp_list_remove_entry(&qp_broker_list, &entry->qp);  in vmci_qp_broker_detach()
2262 			kfree(entry->local_mem);  in vmci_qp_broker_detach()
2264 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in vmci_qp_broker_detach()
2265 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);  in vmci_qp_broker_detach()
2266 		qp_host_free_queue(entry->consume_q, entry->qp.consume_size);  in vmci_qp_broker_detach()
2268 		vmci_resource_remove(&entry->resource);  in vmci_qp_broker_detach()
2270 		kfree(entry);  in vmci_qp_broker_detach()
2276 		    QPBROKERSTATE_HAS_MEM(entry)) {  in vmci_qp_broker_detach()
2277 			entry->state = VMCIQPB_SHUTDOWN_MEM;  in vmci_qp_broker_detach()
2279 			entry->state = VMCIQPB_SHUTDOWN_NO_MEM;  in vmci_qp_broker_detach()
2302 	struct qp_broker_entry *entry;  in vmci_qp_broker_map()  local
2320 	entry = qp_broker_handle_to_entry(handle);  in vmci_qp_broker_map()
2321 	if (!entry) {  in vmci_qp_broker_map()
2328 	if (context_id != entry->create_id && context_id != entry->attach_id) {  in vmci_qp_broker_map()
2333 	is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;  in vmci_qp_broker_map()
2340 		page_store.len = QPE_NUM_PAGES(entry->qp);  in vmci_qp_broker_map()
2342 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2343 		qp_reset_saved_headers(entry);  in vmci_qp_broker_map()
2346 						 entry->produce_q,  in vmci_qp_broker_map()
2347 						 entry->consume_q);  in vmci_qp_broker_map()
2348 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2352 			entry->state++;  in vmci_qp_broker_map()
2354 			if (entry->wakeup_cb)  in vmci_qp_broker_map()
2355 				entry->wakeup_cb(entry->client_data);  in vmci_qp_broker_map()
2371 static int qp_save_headers(struct qp_broker_entry *entry)  in qp_save_headers()  argument
2375 	if (entry->produce_q->saved_header != NULL &&  in qp_save_headers()
2376 	    entry->consume_q->saved_header != NULL) {  in qp_save_headers()
2386 	if (NULL == entry->produce_q->q_header ||  in qp_save_headers()
2387 	    NULL == entry->consume_q->q_header) {  in qp_save_headers()
2388 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in qp_save_headers()
2393 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,  in qp_save_headers()
2394 	       sizeof(entry->saved_produce_q));  in qp_save_headers()
2395 	entry->produce_q->saved_header = &entry->saved_produce_q;  in qp_save_headers()
2396 	memcpy(&entry->saved_consume_q, entry->consume_q->q_header,  in qp_save_headers()
2397 	       sizeof(entry->saved_consume_q));  in qp_save_headers()
2398 	entry->consume_q->saved_header = &entry->saved_consume_q;  in qp_save_headers()
2413 	struct qp_broker_entry *entry;  in vmci_qp_broker_unmap()  local
2431 	entry = qp_broker_handle_to_entry(handle);  in vmci_qp_broker_unmap()
2432 	if (!entry) {  in vmci_qp_broker_unmap()
2439 	if (context_id != entry->create_id && context_id != entry->attach_id) {  in vmci_qp_broker_unmap()
2444 	is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;  in vmci_qp_broker_unmap()
2447 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2448 		result = qp_save_headers(entry);  in vmci_qp_broker_unmap()
2453 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);  in vmci_qp_broker_unmap()
2462 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_unmap()
2463 					       entry->consume_q);  in vmci_qp_broker_unmap()
2468 		entry->state--;  in vmci_qp_broker_unmap()
2470 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2488 	struct qp_entry *entry;  in vmci_qp_guest_endpoints_exit()  local
2493 	while ((entry = qp_list_get_head(&qp_guest_endpoints))) {  in vmci_qp_guest_endpoints_exit()
2494 		ep = (struct qp_guest_endpoint *)entry;  in vmci_qp_guest_endpoints_exit()
2497 		if (!(entry->flags & VMCI_QPFLAG_LOCAL))  in vmci_qp_guest_endpoints_exit()
2498 			qp_detatch_hypercall(entry->handle);  in vmci_qp_guest_endpoints_exit()
2501 		entry->ref_count = 0;  in vmci_qp_guest_endpoints_exit()
2502 		qp_list_remove_entry(&qp_guest_endpoints, entry);  in vmci_qp_guest_endpoints_exit()