82 static void ib_ucontext_notifier_start_account(struct ib_ucontext *context)  in ib_ucontext_notifier_start_account()  argument
84 	atomic_inc(&context->notifier_count);  in ib_ucontext_notifier_start_account()
91 static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)  in ib_ucontext_notifier_end_account()  argument
93 	int zero_notifiers = atomic_dec_and_test(&context->notifier_count);  in ib_ucontext_notifier_end_account()
96 	    !list_empty(&context->no_private_counters)) {  in ib_ucontext_notifier_end_account()
103 		down_write(&context->umem_rwsem);  in ib_ucontext_notifier_end_account()
107 		if (!atomic_read(&context->notifier_count)) {  in ib_ucontext_notifier_end_account()
109 						 &context->no_private_counters,  in ib_ucontext_notifier_end_account()
119 		up_write(&context->umem_rwsem);  in ib_ucontext_notifier_end_account()
135 	item->context->invalidate_range(item, ib_umem_start(item),  in ib_umem_notifier_release_trampoline()
143 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_release()  local
145 	if (!context->invalidate_range)  in ib_umem_notifier_release()
148 	ib_ucontext_notifier_start_account(context);  in ib_umem_notifier_release()
149 	down_read(&context->umem_rwsem);  in ib_umem_notifier_release()
150 	rbt_ib_umem_for_each_in_range(&context->umem_tree, 0,  in ib_umem_notifier_release()
154 	up_read(&context->umem_rwsem);  in ib_umem_notifier_release()
161 	item->context->invalidate_range(item, start, start + PAGE_SIZE);  in invalidate_page_trampoline()
170 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_invalidate_page()  local
172 	if (!context->invalidate_range)  in ib_umem_notifier_invalidate_page()
175 	ib_ucontext_notifier_start_account(context);  in ib_umem_notifier_invalidate_page()
176 	down_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_page()
177 	rbt_ib_umem_for_each_in_range(&context->umem_tree, address,  in ib_umem_notifier_invalidate_page()
180 	up_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_page()
181 	ib_ucontext_notifier_end_account(context);  in ib_umem_notifier_invalidate_page()
188 	item->context->invalidate_range(item, start, end);  in invalidate_range_start_trampoline()
197 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_invalidate_range_start()  local
199 	if (!context->invalidate_range)  in ib_umem_notifier_invalidate_range_start()
202 	ib_ucontext_notifier_start_account(context);  in ib_umem_notifier_invalidate_range_start()
203 	down_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_start()
204 	rbt_ib_umem_for_each_in_range(&context->umem_tree, start,  in ib_umem_notifier_invalidate_range_start()
207 	up_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_start()
222 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_invalidate_range_end()  local
224 	if (!context->invalidate_range)  in ib_umem_notifier_invalidate_range_end()
227 	down_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_end()
228 	rbt_ib_umem_for_each_in_range(&context->umem_tree, start,  in ib_umem_notifier_invalidate_range_end()
231 	up_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_end()
232 	ib_ucontext_notifier_end_account(context);  in ib_umem_notifier_invalidate_range_end()
242 int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem)  in ib_umem_odp_get()  argument
256 	if (context->tgid != our_pid) {  in ib_umem_odp_get()
292 	down_write(&context->umem_rwsem);  in ib_umem_odp_get()
293 	context->odp_mrs_count++;  in ib_umem_odp_get()
296 				   &context->umem_tree);  in ib_umem_odp_get()
297 	if (likely(!atomic_read(&context->notifier_count)) ||  in ib_umem_odp_get()
298 	    context->odp_mrs_count == 1)  in ib_umem_odp_get()
302 			 &context->no_private_counters);  in ib_umem_odp_get()
303 	downgrade_write(&context->umem_rwsem);  in ib_umem_odp_get()
305 	if (context->odp_mrs_count == 1) {  in ib_umem_odp_get()
310 		atomic_set(&context->notifier_count, 0);  in ib_umem_odp_get()
311 		INIT_HLIST_NODE(&context->mn.hlist);  in ib_umem_odp_get()
312 		context->mn.ops = &ib_umem_notifiers;  in ib_umem_odp_get()
318 		ret_val = mmu_notifier_register(&context->mn, mm);  in ib_umem_odp_get()
327 	up_read(&context->umem_rwsem);  in ib_umem_odp_get()
339 	up_read(&context->umem_rwsem);  in ib_umem_odp_get()
352 	struct ib_ucontext *context = umem->context;  in ib_umem_odp_release()  local
363 	down_write(&context->umem_rwsem);  in ib_umem_odp_release()
366 				   &context->umem_tree);  in ib_umem_odp_release()
367 	context->odp_mrs_count--;  in ib_umem_odp_release()
380 	downgrade_write(&context->umem_rwsem);  in ib_umem_odp_release()
381 	if (!context->odp_mrs_count) {  in ib_umem_odp_release()
385 		owning_process = get_pid_task(context->tgid,  in ib_umem_odp_release()
401 		mmu_notifier_unregister(&context->mn, owning_mm);  in ib_umem_odp_release()
409 	up_read(&context->umem_rwsem);  in ib_umem_odp_release()
443 	struct ib_device *dev = umem->context->device;  in ib_umem_odp_map_dma_single_page()
482 	if (umem->context->invalidate_range || !stored_page)  in ib_umem_odp_map_dma_single_page()
485 	if (remove_existing_mapping && umem->context->invalidate_range) {  in ib_umem_odp_map_dma_single_page()
547 	owning_process = get_pid_task(umem->context->tgid, PIDTYPE_PID);  in ib_umem_odp_map_dma_pages()
626 	struct ib_device *dev = umem->context->device;  in ib_umem_odp_unmap_dma_pages()
661 			if (!umem->context->invalidate_range)  in ib_umem_odp_unmap_dma_pages()