111 	struct kmmio_fault_page *f;  in get_kmmio_fault_page()  local
119 	list_for_each_entry_rcu(f, head, list) {  in get_kmmio_fault_page()
120 		if (f->addr == addr)  in get_kmmio_fault_page()
121 			return f;  in get_kmmio_fault_page()
148 static int clear_page_presence(struct kmmio_fault_page *f, bool clear)  in clear_page_presence()  argument
151 	pte_t *pte = lookup_address(f->addr, &level);  in clear_page_presence()
154 		pr_err("no pte for addr 0x%08lx\n", f->addr);  in clear_page_presence()
160 		clear_pmd_presence((pmd_t *)pte, clear, &f->old_presence);  in clear_page_presence()
163 		clear_pte_presence(pte, clear, &f->old_presence);  in clear_page_presence()
170 	__flush_tlb_one(f->addr);  in clear_page_presence()
185 static int arm_kmmio_fault_page(struct kmmio_fault_page *f)  in arm_kmmio_fault_page()  argument
188 	WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n"));  in arm_kmmio_fault_page()
189 	if (f->armed) {  in arm_kmmio_fault_page()
191 			   f->addr, f->count, !!f->old_presence);  in arm_kmmio_fault_page()
193 	ret = clear_page_presence(f, true);  in arm_kmmio_fault_page()
195 		  f->addr);  in arm_kmmio_fault_page()
196 	f->armed = true;  in arm_kmmio_fault_page()
201 static void disarm_kmmio_fault_page(struct kmmio_fault_page *f)  in disarm_kmmio_fault_page()  argument
203 	int ret = clear_page_presence(f, false);  in disarm_kmmio_fault_page()
205 			KERN_ERR "kmmio disarming at 0x%08lx failed.\n", f->addr);  in disarm_kmmio_fault_page()
206 	f->armed = false;  in disarm_kmmio_fault_page()
376 	struct kmmio_fault_page *f;  in add_kmmio_fault_page()  local
378 	f = get_kmmio_fault_page(addr);  in add_kmmio_fault_page()
379 	if (f) {  in add_kmmio_fault_page()
380 		if (!f->count)  in add_kmmio_fault_page()
381 			arm_kmmio_fault_page(f);  in add_kmmio_fault_page()
382 		f->count++;  in add_kmmio_fault_page()
386 	f = kzalloc(sizeof(*f), GFP_ATOMIC);  in add_kmmio_fault_page()
387 	if (!f)  in add_kmmio_fault_page()
390 	f->count = 1;  in add_kmmio_fault_page()
391 	f->addr = addr;  in add_kmmio_fault_page()
393 	if (arm_kmmio_fault_page(f)) {  in add_kmmio_fault_page()
394 		kfree(f);  in add_kmmio_fault_page()
398 	list_add_rcu(&f->list, kmmio_page_list(f->addr));  in add_kmmio_fault_page()
407 	struct kmmio_fault_page *f;  in release_kmmio_fault_page()  local
409 	f = get_kmmio_fault_page(addr);  in release_kmmio_fault_page()
410 	if (!f)  in release_kmmio_fault_page()
413 	f->count--;  in release_kmmio_fault_page()
414 	BUG_ON(f->count < 0);  in release_kmmio_fault_page()
415 	if (!f->count) {  in release_kmmio_fault_page()
416 		disarm_kmmio_fault_page(f);  in release_kmmio_fault_page()
417 		if (!f->scheduled_for_release) {  in release_kmmio_fault_page()
418 			f->release_next = *release_list;  in release_kmmio_fault_page()
419 			*release_list = f;  in release_kmmio_fault_page()
420 			f->scheduled_for_release = true;  in release_kmmio_fault_page()
477 	struct kmmio_fault_page *f = dr->release_list;  in rcu_free_kmmio_fault_pages()  local
478 	while (f) {  in rcu_free_kmmio_fault_pages()
479 		struct kmmio_fault_page *next = f->release_next;  in rcu_free_kmmio_fault_pages()
480 		BUG_ON(f->count);  in rcu_free_kmmio_fault_pages()
481 		kfree(f);  in rcu_free_kmmio_fault_pages()
482 		f = next;  in rcu_free_kmmio_fault_pages()
491 	struct kmmio_fault_page *f = dr->release_list;  in remove_kmmio_fault_pages()  local
496 	while (f) {  in remove_kmmio_fault_pages()
497 		if (!f->count) {  in remove_kmmio_fault_pages()
498 			list_del_rcu(&f->list);  in remove_kmmio_fault_pages()
499 			prevp = &f->release_next;  in remove_kmmio_fault_pages()
501 			*prevp = f->release_next;  in remove_kmmio_fault_pages()
502 			f->release_next = NULL;  in remove_kmmio_fault_pages()
503 			f->scheduled_for_release = false;  in remove_kmmio_fault_pages()
505 		f = *prevp;  in remove_kmmio_fault_pages()