58 	struct mmu_notifier *mn;  in __mmu_notifier_release()  local
66 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
73 		if (mn->ops->release)  in __mmu_notifier_release()
74 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
78 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
87 		hlist_del_init_rcu(&mn->hlist);  in __mmu_notifier_release()
113 	struct mmu_notifier *mn;  in __mmu_notifier_clear_flush_young()  local
117 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
118 		if (mn->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
119 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
130 	struct mmu_notifier *mn;  in __mmu_notifier_clear_young()  local
134 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_young()
135 		if (mn->ops->clear_young)  in __mmu_notifier_clear_young()
136 			young |= mn->ops->clear_young(mn, mm, start, end);  in __mmu_notifier_clear_young()
146 	struct mmu_notifier *mn;  in __mmu_notifier_test_young()  local
150 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
151 		if (mn->ops->test_young) {  in __mmu_notifier_test_young()
152 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
165 	struct mmu_notifier *mn;  in __mmu_notifier_change_pte()  local
169 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
170 		if (mn->ops->change_pte)  in __mmu_notifier_change_pte()
171 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
179 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_page()  local
183 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_page()
184 		if (mn->ops->invalidate_page)  in __mmu_notifier_invalidate_page()
185 			mn->ops->invalidate_page(mn, mm, address);  in __mmu_notifier_invalidate_page()
193 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_start()  local
197 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
198 		if (mn->ops->invalidate_range_start)  in __mmu_notifier_invalidate_range_start()
199 			mn->ops->invalidate_range_start(mn, mm, start, end);  in __mmu_notifier_invalidate_range_start()
208 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_end()  local
212 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
221 		if (mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range_end()
222 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
223 		if (mn->ops->invalidate_range_end)  in __mmu_notifier_invalidate_range_end()
224 			mn->ops->invalidate_range_end(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
233 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range()  local
237 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range()
238 		if (mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
239 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range()
245 static int do_mmu_notifier_register(struct mmu_notifier *mn,  in do_mmu_notifier_register()  argument
289 	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);  in do_mmu_notifier_register()
315 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
317 	return do_mmu_notifier_register(mn, mm, 1);  in mmu_notifier_register()
325 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
327 	return do_mmu_notifier_register(mn, mm, 0);  in __mmu_notifier_register()
349 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
353 	if (!hlist_unhashed(&mn->hlist)) {  in mmu_notifier_unregister()
365 		if (mn->ops->release)  in mmu_notifier_unregister()
366 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
374 		hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_unregister()
393 void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,  in mmu_notifier_unregister_no_release()  argument
401 	hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_unregister_no_release()