258 	void (*add_page)(struct vmballoon *b, int idx, struct page *p);
259 	int (*lock)(struct vmballoon *b, unsigned int num_pages,
261 	int (*unlock)(struct vmballoon *b, unsigned int num_pages,
322 static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)  in vmballoon_send_start()  argument
327 	STATS_INC(b->stats.start);  in vmballoon_send_start()
333 		b->capabilities = capabilities;  in vmballoon_send_start()
337 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
344 	if (b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS)  in vmballoon_send_start()
345 		b->supported_page_sizes = 2;  in vmballoon_send_start()
347 		b->supported_page_sizes = 1;  in vmballoon_send_start()
351 		STATS_INC(b->stats.start_fail);  in vmballoon_send_start()
356 static bool vmballoon_check_status(struct vmballoon *b, unsigned long status)  in vmballoon_check_status()  argument
363 		b->reset_required = true;  in vmballoon_check_status()
377 static bool vmballoon_send_guest_id(struct vmballoon *b)  in vmballoon_send_guest_id()  argument
384 	STATS_INC(b->stats.guest_type);  in vmballoon_send_guest_id()
386 	if (vmballoon_check_status(b, status))  in vmballoon_send_guest_id()
390 	STATS_INC(b->stats.guest_type_fail);  in vmballoon_send_guest_id()
405 static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)  in vmballoon_send_get_target()  argument
418 	si_meminfo(&b->sysinfo);  in vmballoon_send_get_target()
419 	limit = b->sysinfo.totalram;  in vmballoon_send_get_target()
427 	STATS_INC(b->stats.target);  in vmballoon_send_get_target()
430 	if (vmballoon_check_status(b, status)) {  in vmballoon_send_get_target()
436 	STATS_INC(b->stats.target_fail);  in vmballoon_send_get_target()
445 static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,  in vmballoon_send_lock_page()  argument
455 	STATS_INC(b->stats.lock[false]);  in vmballoon_send_lock_page()
458 	if (vmballoon_check_status(b, status))  in vmballoon_send_lock_page()
462 	STATS_INC(b->stats.lock_fail[false]);  in vmballoon_send_lock_page()
466 static int vmballoon_send_batched_lock(struct vmballoon *b,  in vmballoon_send_batched_lock()  argument
470 	unsigned long pfn = page_to_pfn(b->page);  in vmballoon_send_batched_lock()
472 	STATS_INC(b->stats.lock[is_2m_pages]);  in vmballoon_send_batched_lock()
481 	if (vmballoon_check_status(b, status))  in vmballoon_send_batched_lock()
485 	STATS_INC(b->stats.lock_fail[is_2m_pages]);  in vmballoon_send_batched_lock()
493 static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn,  in vmballoon_send_unlock_page()  argument
503 	STATS_INC(b->stats.unlock[false]);  in vmballoon_send_unlock_page()
506 	if (vmballoon_check_status(b, status))  in vmballoon_send_unlock_page()
510 	STATS_INC(b->stats.unlock_fail[false]);  in vmballoon_send_unlock_page()
514 static bool vmballoon_send_batched_unlock(struct vmballoon *b,  in vmballoon_send_batched_unlock()  argument
518 	unsigned long pfn = page_to_pfn(b->page);  in vmballoon_send_batched_unlock()
520 	STATS_INC(b->stats.unlock[is_2m_pages]);  in vmballoon_send_batched_unlock()
529 	if (vmballoon_check_status(b, status))  in vmballoon_send_batched_unlock()
533 	STATS_INC(b->stats.unlock_fail[is_2m_pages]);  in vmballoon_send_batched_unlock()
559 static void vmballoon_pop(struct vmballoon *b)  in vmballoon_pop()  argument
567 				&b->page_sizes[is_2m_pages];  in vmballoon_pop()
573 			STATS_INC(b->stats.free[is_2m_pages]);  in vmballoon_pop()
574 			b->size -= size_per_page;  in vmballoon_pop()
579 	if (b->batch_page) {  in vmballoon_pop()
580 		vunmap(b->batch_page);  in vmballoon_pop()
581 		b->batch_page = NULL;  in vmballoon_pop()
584 	if (b->page) {  in vmballoon_pop()
585 		__free_page(b->page);  in vmballoon_pop()
586 		b->page = NULL;  in vmballoon_pop()
595 static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,  in vmballoon_lock_page()  argument
599 	struct page *page = b->page;  in vmballoon_lock_page()
600 	struct vmballoon_page_size *page_size = &b->page_sizes[false];  in vmballoon_lock_page()
604 	locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,  in vmballoon_lock_page()
607 		STATS_INC(b->stats.refused_alloc[false]);  in vmballoon_lock_page()
633 	b->size++;  in vmballoon_lock_page()
638 static int vmballoon_lock_batched_page(struct vmballoon *b,  in vmballoon_lock_batched_page()  argument
644 	locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages,  in vmballoon_lock_batched_page()
648 			u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_lock_batched_page()
658 		u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_lock_batched_page()
661 				&b->page_sizes[is_2m_pages];  in vmballoon_lock_batched_page()
663 		locked = vmballoon_batch_get_status(b->batch_page, i);  in vmballoon_lock_batched_page()
668 			b->size += size_per_page;  in vmballoon_lock_batched_page()
697 static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,  in vmballoon_unlock_page()  argument
700 	struct page *page = b->page;  in vmballoon_unlock_page()
701 	struct vmballoon_page_size *page_size = &b->page_sizes[false];  in vmballoon_unlock_page()
705 	if (!vmballoon_send_unlock_page(b, page_to_pfn(page), target)) {  in vmballoon_unlock_page()
712 	STATS_INC(b->stats.free[false]);  in vmballoon_unlock_page()
715 	b->size--;  in vmballoon_unlock_page()
720 static int vmballoon_unlock_batched_page(struct vmballoon *b,  in vmballoon_unlock_batched_page()  argument
728 	hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages,  in vmballoon_unlock_batched_page()
734 		u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_unlock_batched_page()
737 				&b->page_sizes[is_2m_pages];  in vmballoon_unlock_batched_page()
739 		locked = vmballoon_batch_get_status(b->batch_page, i);  in vmballoon_unlock_batched_page()
750 			STATS_INC(b->stats.free[is_2m_pages]);  in vmballoon_unlock_batched_page()
753 			b->size -= size_per_page;  in vmballoon_unlock_batched_page()
764 static void vmballoon_release_refused_pages(struct vmballoon *b,  in vmballoon_release_refused_pages()  argument
769 			&b->page_sizes[is_2m_pages];  in vmballoon_release_refused_pages()
774 		STATS_INC(b->stats.refused_free[is_2m_pages]);  in vmballoon_release_refused_pages()
780 static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)  in vmballoon_add_page()  argument
782 	b->page = p;  in vmballoon_add_page()
785 static void vmballoon_add_batched_page(struct vmballoon *b, int idx,  in vmballoon_add_batched_page()  argument
788 	vmballoon_batch_set_pa(b->batch_page, idx,  in vmballoon_add_batched_page()
797 static void vmballoon_inflate(struct vmballoon *b)  in vmballoon_inflate()  argument
806 	pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);  in vmballoon_inflate()
827 	if (b->slow_allocation_cycles) {  in vmballoon_inflate()
828 		rate = b->rate_alloc;  in vmballoon_inflate()
833 			b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES;  in vmballoon_inflate()
837 		 __func__, b->target - b->size, rate, b->rate_alloc);  in vmballoon_inflate()
839 	while (!b->reset_required &&  in vmballoon_inflate()
840 		b->size + num_pages * vmballoon_page_size(is_2m_pages)  in vmballoon_inflate()
841 		< b->target) {  in vmballoon_inflate()
845 			STATS_INC(b->stats.alloc[is_2m_pages]);  in vmballoon_inflate()
847 			STATS_INC(b->stats.sleep_alloc);  in vmballoon_inflate()
851 			STATS_INC(b->stats.alloc_fail[is_2m_pages]);  in vmballoon_inflate()
854 				b->ops->lock(b, num_pages, true, &b->target);  in vmballoon_inflate()
873 				b->rate_alloc = max(b->rate_alloc / 2,  in vmballoon_inflate()
875 				STATS_INC(b->stats.sleep_alloc_fail);  in vmballoon_inflate()
887 			b->slow_allocation_cycles = VMW_BALLOON_SLOW_CYCLES;  in vmballoon_inflate()
889 			if (allocations >= b->rate_alloc)  in vmballoon_inflate()
894 			rate = b->rate_alloc;  in vmballoon_inflate()
898 		b->ops->add_page(b, num_pages++, page);  in vmballoon_inflate()
899 		if (num_pages == b->batch_max_pages) {  in vmballoon_inflate()
900 			error = b->ops->lock(b, num_pages, is_2m_pages,  in vmballoon_inflate()
901 					&b->target);  in vmballoon_inflate()
916 		b->ops->lock(b, num_pages, is_2m_pages, &b->target);  in vmballoon_inflate()
922 	if (error == 0 && allocations >= b->rate_alloc) {  in vmballoon_inflate()
923 		unsigned int mult = allocations / b->rate_alloc;  in vmballoon_inflate()
925 		b->rate_alloc =  in vmballoon_inflate()
926 			min(b->rate_alloc + mult * VMW_BALLOON_RATE_ALLOC_INC,  in vmballoon_inflate()
930 	vmballoon_release_refused_pages(b, true);  in vmballoon_inflate()
931 	vmballoon_release_refused_pages(b, false);  in vmballoon_inflate()
937 static void vmballoon_deflate(struct vmballoon *b)  in vmballoon_deflate()  argument
941 	pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);  in vmballoon_deflate()
944 	for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes;  in vmballoon_deflate()
949 				&b->page_sizes[is_2m_pages];  in vmballoon_deflate()
952 			if (b->reset_required ||  in vmballoon_deflate()
953 				(b->target > 0 &&  in vmballoon_deflate()
954 					b->size - num_pages  in vmballoon_deflate()
956 				< b->target + vmballoon_page_size(true)))  in vmballoon_deflate()
960 			b->ops->add_page(b, num_pages++, page);  in vmballoon_deflate()
962 			if (num_pages == b->batch_max_pages) {  in vmballoon_deflate()
965 				error = b->ops->unlock(b, num_pages,  in vmballoon_deflate()
966 						is_2m_pages, &b->target);  in vmballoon_deflate()
976 			b->ops->unlock(b, num_pages, is_2m_pages, &b->target);  in vmballoon_deflate()
992 static bool vmballoon_init_batching(struct vmballoon *b)  in vmballoon_init_batching()  argument
994 	b->page = alloc_page(VMW_PAGE_ALLOC_NOSLEEP);  in vmballoon_init_batching()
995 	if (!b->page)  in vmballoon_init_batching()
998 	b->batch_page = vmap(&b->page, 1, VM_MAP, PAGE_KERNEL);  in vmballoon_init_batching()
999 	if (!b->batch_page) {  in vmballoon_init_batching()
1000 		__free_page(b->page);  in vmballoon_init_batching()
1012 	struct vmballoon *b = client_data;  in vmballoon_doorbell()  local
1014 	STATS_INC(b->stats.doorbell);  in vmballoon_doorbell()
1016 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
1022 static void vmballoon_vmci_cleanup(struct vmballoon *b)  in vmballoon_vmci_cleanup()  argument
1028 	STATS_INC(b->stats.doorbell_unset);  in vmballoon_vmci_cleanup()
1030 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
1031 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
1032 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
1039 static int vmballoon_vmci_init(struct vmballoon *b)  in vmballoon_vmci_init()  argument
1043 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) != 0) {  in vmballoon_vmci_init()
1044 		error = vmci_doorbell_create(&b->vmci_doorbell,  in vmballoon_vmci_init()
1047 				vmballoon_doorbell, b);  in vmballoon_vmci_init()
1051 					b->vmci_doorbell.context,  in vmballoon_vmci_init()
1052 					b->vmci_doorbell.resource, error);  in vmballoon_vmci_init()
1053 			STATS_INC(b->stats.doorbell_set);  in vmballoon_vmci_init()
1058 		vmballoon_vmci_cleanup(b);  in vmballoon_vmci_init()
1071 static void vmballoon_reset(struct vmballoon *b)  in vmballoon_reset()  argument
1075 	vmballoon_vmci_cleanup(b);  in vmballoon_reset()
1078 	vmballoon_pop(b);  in vmballoon_reset()
1080 	if (!vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))  in vmballoon_reset()
1083 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1084 		b->ops = &vmballoon_batched_ops;  in vmballoon_reset()
1085 		b->batch_max_pages = VMW_BALLOON_BATCH_MAX_PAGES;  in vmballoon_reset()
1086 		if (!vmballoon_init_batching(b)) {  in vmballoon_reset()
1093 			vmballoon_send_start(b, 0);  in vmballoon_reset()
1096 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1097 		b->ops = &vmballoon_basic_ops;  in vmballoon_reset()
1098 		b->batch_max_pages = 1;  in vmballoon_reset()
1101 	b->reset_required = false;  in vmballoon_reset()
1103 	error = vmballoon_vmci_init(b);  in vmballoon_reset()
1107 	if (!vmballoon_send_guest_id(b))  in vmballoon_reset()
1118 	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);  in vmballoon_work()  local
1121 	STATS_INC(b->stats.timer);  in vmballoon_work()
1123 	if (b->reset_required)  in vmballoon_work()
1124 		vmballoon_reset(b);  in vmballoon_work()
1126 	if (b->slow_allocation_cycles > 0)  in vmballoon_work()
1127 		b->slow_allocation_cycles--;  in vmballoon_work()
1129 	if (!b->reset_required && vmballoon_send_get_target(b, &target)) {  in vmballoon_work()
1131 		b->target = target;  in vmballoon_work()
1133 		if (b->size < target)  in vmballoon_work()
1134 			vmballoon_inflate(b);  in vmballoon_work()
1136 				b->size > target + vmballoon_page_size(true))  in vmballoon_work()
1137 			vmballoon_deflate(b);  in vmballoon_work()
1155 	struct vmballoon *b = f->private;  in vmballoon_debug_show()  local
1156 	struct vmballoon_stats *stats = &b->stats;  in vmballoon_debug_show()
1163 		   VMW_BALLOON_CAPABILITIES, b->capabilities,  in vmballoon_debug_show()
1164 		   b->reset_required ? 'y' : 'n');  in vmballoon_debug_show()
1170 		   b->target, b->size);  in vmballoon_debug_show()
1175 		   b->rate_alloc);  in vmballoon_debug_show()
1233 static int __init vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1237 	b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,  in vmballoon_debugfs_init()
1239 	if (IS_ERR(b->dbg_entry)) {  in vmballoon_debugfs_init()
1240 		error = PTR_ERR(b->dbg_entry);  in vmballoon_debugfs_init()
1248 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1250 	debugfs_remove(b->dbg_entry);  in vmballoon_debugfs_exit()
1255 static inline int vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1260 static inline void vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument