38 	struct i915_mmu_notifier *mn;  member
50 	struct mmu_notifier mn;  member
58 	struct i915_mmu_notifier *mn;  member
99 static void *invalidate_range__linear(struct i915_mmu_notifier *mn,  in invalidate_range__linear()  argument
108 	serial = mn->serial;  in invalidate_range__linear()
109 	list_for_each_entry(mo, &mn->linear, link) {  in invalidate_range__linear()
120 		spin_unlock(&mn->lock);  in invalidate_range__linear()
124 		spin_lock(&mn->lock);  in invalidate_range__linear()
125 		if (serial != mn->serial)  in invalidate_range__linear()
137 	struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn);  in i915_gem_userptr_mn_invalidate_range_start()  local
146 		spin_lock(&mn->lock);  in i915_gem_userptr_mn_invalidate_range_start()
147 		if (mn->has_linear)  in i915_gem_userptr_mn_invalidate_range_start()
148 			it = invalidate_range__linear(mn, mm, start, end);  in i915_gem_userptr_mn_invalidate_range_start()
149 		else if (serial == mn->serial)  in i915_gem_userptr_mn_invalidate_range_start()
152 			it = interval_tree_iter_first(&mn->objects, start, end);  in i915_gem_userptr_mn_invalidate_range_start()
164 				spin_unlock(&mn->lock);  in i915_gem_userptr_mn_invalidate_range_start()
169 			serial = mn->serial;  in i915_gem_userptr_mn_invalidate_range_start()
171 		spin_unlock(&mn->lock);  in i915_gem_userptr_mn_invalidate_range_start()
186 	struct i915_mmu_notifier *mn;  in i915_mmu_notifier_create()  local
189 	mn = kmalloc(sizeof(*mn), GFP_KERNEL);  in i915_mmu_notifier_create()
190 	if (mn == NULL)  in i915_mmu_notifier_create()
193 	spin_lock_init(&mn->lock);  in i915_mmu_notifier_create()
194 	mn->mn.ops = &i915_gem_userptr_notifier;  in i915_mmu_notifier_create()
195 	mn->objects = RB_ROOT;  in i915_mmu_notifier_create()
196 	mn->serial = 1;  in i915_mmu_notifier_create()
197 	INIT_LIST_HEAD(&mn->linear);  in i915_mmu_notifier_create()
198 	mn->has_linear = false;  in i915_mmu_notifier_create()
201 	ret = __mmu_notifier_register(&mn->mn, mm);  in i915_mmu_notifier_create()
203 		kfree(mn);  in i915_mmu_notifier_create()
207 	return mn;  in i915_mmu_notifier_create()
210 static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mn)  in __i915_mmu_notifier_update_serial()  argument
212 	if (++mn->serial == 0)  in __i915_mmu_notifier_update_serial()
213 		mn->serial = 1;  in __i915_mmu_notifier_update_serial()
218 		      struct i915_mmu_notifier *mn,  in i915_mmu_notifier_add()  argument
234 	spin_lock(&mn->lock);  in i915_mmu_notifier_add()
235 	it = interval_tree_iter_first(&mn->objects,  in i915_mmu_notifier_add()
253 			mn->has_linear = mo->is_linear = true;  in i915_mmu_notifier_add()
257 		interval_tree_insert(&mo->it, &mn->objects);  in i915_mmu_notifier_add()
260 		list_add(&mo->link, &mn->linear);  in i915_mmu_notifier_add()
261 		__i915_mmu_notifier_update_serial(mn);  in i915_mmu_notifier_add()
263 	spin_unlock(&mn->lock);  in i915_mmu_notifier_add()
269 static bool i915_mmu_notifier_has_linear(struct i915_mmu_notifier *mn)  in i915_mmu_notifier_has_linear()  argument
273 	list_for_each_entry(mo, &mn->linear, link)  in i915_mmu_notifier_has_linear()
281 i915_mmu_notifier_del(struct i915_mmu_notifier *mn,  in i915_mmu_notifier_del()  argument
284 	spin_lock(&mn->lock);  in i915_mmu_notifier_del()
287 		mn->has_linear = i915_mmu_notifier_has_linear(mn);  in i915_mmu_notifier_del()
289 		interval_tree_remove(&mo->it, &mn->objects);  in i915_mmu_notifier_del()
290 	__i915_mmu_notifier_update_serial(mn);  in i915_mmu_notifier_del()
291 	spin_unlock(&mn->lock);  in i915_mmu_notifier_del()
303 	i915_mmu_notifier_del(mo->mn, mo);  in i915_gem_userptr_release__mmu_notifier()
312 	struct i915_mmu_notifier *mn = mm->mn;  in i915_mmu_notifier_find()  local
314 	mn = mm->mn;  in i915_mmu_notifier_find()
315 	if (mn)  in i915_mmu_notifier_find()
316 		return mn;  in i915_mmu_notifier_find()
320 	if ((mn = mm->mn) == NULL) {  in i915_mmu_notifier_find()
321 		mn = i915_mmu_notifier_create(mm->mm);  in i915_mmu_notifier_find()
322 		if (!IS_ERR(mn))  in i915_mmu_notifier_find()
323 			mm->mn = mn;  in i915_mmu_notifier_find()
328 	return mn;  in i915_mmu_notifier_find()
335 	struct i915_mmu_notifier *mn;  in i915_gem_userptr_init__mmu_notifier()  local
345 	mn = i915_mmu_notifier_find(obj->userptr.mm);  in i915_gem_userptr_init__mmu_notifier()
346 	if (IS_ERR(mn))  in i915_gem_userptr_init__mmu_notifier()
347 		return PTR_ERR(mn);  in i915_gem_userptr_init__mmu_notifier()
353 	mo->mn = mn;  in i915_gem_userptr_init__mmu_notifier()
358 	ret = i915_mmu_notifier_add(obj->base.dev, mn, mo);  in i915_gem_userptr_init__mmu_notifier()
369 i915_mmu_notifier_free(struct i915_mmu_notifier *mn,  in i915_mmu_notifier_free()  argument
372 	if (mn == NULL)  in i915_mmu_notifier_free()
375 	mmu_notifier_unregister(&mn->mn, mm);  in i915_mmu_notifier_free()
376 	kfree(mn);  in i915_mmu_notifier_free()
400 i915_mmu_notifier_free(struct i915_mmu_notifier *mn,  in i915_mmu_notifier_free()  argument
452 		mm->mn = NULL;  in i915_gem_userptr_init__mm_struct()
470 	i915_mmu_notifier_free(mm->mn, mm->mm);  in __i915_mm_struct_free__worker()