60 	struct drm_i915_gem_object *obj;  member
69 	struct drm_i915_gem_object *obj = mo->obj;  in __cancel_userptr__worker()  local
70 	struct drm_device *dev = obj->base.dev;  in __cancel_userptr__worker()
74 	obj->userptr.work = NULL;  in __cancel_userptr__worker()
76 	if (obj->pages != NULL) {  in __cancel_userptr__worker()
84 		list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) {  in __cancel_userptr__worker()
88 		WARN_ON(i915_gem_object_put_pages(obj));  in __cancel_userptr__worker()
93 	drm_gem_object_unreference(&obj->base);  in __cancel_userptr__worker()
99 	unsigned long end = mo->obj->userptr.ptr + mo->obj->base.size;  in cancel_userptr()
108 	if (mo->active && kref_get_unless_zero(&mo->obj->base.refcount)) {  in cancel_userptr()
205 		struct drm_i915_gem_object *obj;  in i915_mmu_notifier_add()  local
218 		obj = container_of(it, struct i915_mmu_object, it)->obj;  in i915_mmu_notifier_add()
219 		if (!obj->userptr.workers)  in i915_mmu_notifier_add()
260 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)  in i915_gem_userptr_release__mmu_notifier()  argument
264 	mo = obj->userptr.mmu_object;  in i915_gem_userptr_release__mmu_notifier()
271 	obj->userptr.mmu_object = NULL;  in i915_gem_userptr_release__mmu_notifier()
297 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,  in i915_gem_userptr_init__mmu_notifier()  argument
307 	if (WARN_ON(obj->userptr.mm == NULL))  in i915_gem_userptr_init__mmu_notifier()
310 	mn = i915_mmu_notifier_find(obj->userptr.mm);  in i915_gem_userptr_init__mmu_notifier()
319 	mo->it.start = obj->userptr.ptr;  in i915_gem_userptr_init__mmu_notifier()
320 	mo->it.last = mo->it.start + obj->base.size - 1;  in i915_gem_userptr_init__mmu_notifier()
321 	mo->obj = obj;  in i915_gem_userptr_init__mmu_notifier()
324 	ret = i915_mmu_notifier_add(obj->base.dev, mn, mo);  in i915_gem_userptr_init__mmu_notifier()
330 	obj->userptr.mmu_object = mo;  in i915_gem_userptr_init__mmu_notifier()
348 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)  in i915_gem_userptr_release__mmu_notifier()  argument
353 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,  in i915_gem_userptr_init__mmu_notifier()  argument
387 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)  in i915_gem_userptr_init__mm_struct()  argument
389 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);  in i915_gem_userptr_init__mm_struct()
413 		mm->dev = obj->base.dev;  in i915_gem_userptr_init__mm_struct()
426 	obj->userptr.mm = mm;  in i915_gem_userptr_init__mm_struct()
455 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)  in i915_gem_userptr_release__mm_struct()  argument
457 	if (obj->userptr.mm == NULL)  in i915_gem_userptr_release__mm_struct()
460 	kref_put_mutex(&obj->userptr.mm->kref,  in i915_gem_userptr_release__mm_struct()
462 		       &to_i915(obj->base.dev)->mm_lock);  in i915_gem_userptr_release__mm_struct()
463 	obj->userptr.mm = NULL;  in i915_gem_userptr_release__mm_struct()
468 	struct drm_i915_gem_object *obj;  member
512 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,  in __i915_gem_userptr_set_pages()  argument
517 	ret = st_set_pages(&obj->pages, pvec, num_pages);  in __i915_gem_userptr_set_pages()
521 	ret = i915_gem_gtt_prepare_object(obj);  in __i915_gem_userptr_set_pages()
523 		sg_free_table(obj->pages);  in __i915_gem_userptr_set_pages()
524 		kfree(obj->pages);  in __i915_gem_userptr_set_pages()
525 		obj->pages = NULL;  in __i915_gem_userptr_set_pages()
532 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj,  in __i915_gem_userptr_set_active()  argument
548 	if (obj->userptr.mmu_object == NULL)  in __i915_gem_userptr_set_active()
551 	spin_lock(&obj->userptr.mmu_object->mn->lock);  in __i915_gem_userptr_set_active()
555 	if (!value || !work_pending(&obj->userptr.mmu_object->work))  in __i915_gem_userptr_set_active()
556 		obj->userptr.mmu_object->active = value;  in __i915_gem_userptr_set_active()
559 	spin_unlock(&obj->userptr.mmu_object->mn->lock);  in __i915_gem_userptr_set_active()
569 	struct drm_i915_gem_object *obj = work->obj;  in __i915_gem_userptr_get_pages_worker()  local
570 	struct drm_device *dev = obj->base.dev;  in __i915_gem_userptr_get_pages_worker()
571 	const int npages = obj->base.size >> PAGE_SHIFT;  in __i915_gem_userptr_get_pages_worker()
583 		struct mm_struct *mm = obj->userptr.mm->mm;  in __i915_gem_userptr_get_pages_worker()
588 					     obj->userptr.ptr + pinned * PAGE_SIZE,  in __i915_gem_userptr_get_pages_worker()
590 					     !obj->userptr.read_only, 0,  in __i915_gem_userptr_get_pages_worker()
601 	if (obj->userptr.work == &work->work) {  in __i915_gem_userptr_get_pages_worker()
603 			ret = __i915_gem_userptr_set_pages(obj, pvec, npages);  in __i915_gem_userptr_get_pages_worker()
605 				list_add_tail(&obj->global_list,  in __i915_gem_userptr_get_pages_worker()
607 				obj->get_page.sg = obj->pages->sgl;  in __i915_gem_userptr_get_pages_worker()
608 				obj->get_page.last = 0;  in __i915_gem_userptr_get_pages_worker()
612 		obj->userptr.work = ERR_PTR(ret);  in __i915_gem_userptr_get_pages_worker()
614 			__i915_gem_userptr_set_active(obj, false);  in __i915_gem_userptr_get_pages_worker()
617 	obj->userptr.workers--;  in __i915_gem_userptr_get_pages_worker()
618 	drm_gem_object_unreference(&obj->base);  in __i915_gem_userptr_get_pages_worker()
629 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,  in __i915_gem_userptr_get_pages_schedule()  argument
653 	if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS)  in __i915_gem_userptr_get_pages_schedule()
660 	obj->userptr.work = &work->work;  in __i915_gem_userptr_get_pages_schedule()
661 	obj->userptr.workers++;  in __i915_gem_userptr_get_pages_schedule()
663 	work->obj = obj;  in __i915_gem_userptr_get_pages_schedule()
664 	drm_gem_object_reference(&obj->base);  in __i915_gem_userptr_get_pages_schedule()
677 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)  in i915_gem_userptr_get_pages()  argument
679 	const int num_pages = obj->base.size >> PAGE_SHIFT;  in i915_gem_userptr_get_pages()
700 	if (IS_ERR(obj->userptr.work)) {  in i915_gem_userptr_get_pages()
702 		ret = PTR_ERR(obj->userptr.work);  in i915_gem_userptr_get_pages()
703 		obj->userptr.work = NULL;  in i915_gem_userptr_get_pages()
706 	if (obj->userptr.work)  in i915_gem_userptr_get_pages()
711 	ret = __i915_gem_userptr_set_active(obj, true);  in i915_gem_userptr_get_pages()
717 	if (obj->userptr.mm->mm == current->mm) {  in i915_gem_userptr_get_pages()
723 				__i915_gem_userptr_set_active(obj, false);  in i915_gem_userptr_get_pages()
728 		pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,  in i915_gem_userptr_get_pages()
729 					       !obj->userptr.read_only, pvec);  in i915_gem_userptr_get_pages()
736 		ret = __i915_gem_userptr_get_pages_schedule(obj, &active);  in i915_gem_userptr_get_pages()
738 		ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);  in i915_gem_userptr_get_pages()
740 		__i915_gem_userptr_set_active(obj, active);  in i915_gem_userptr_get_pages()
748 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)  in i915_gem_userptr_put_pages()  argument
752 	BUG_ON(obj->userptr.work != NULL);  in i915_gem_userptr_put_pages()
753 	__i915_gem_userptr_set_active(obj, false);  in i915_gem_userptr_put_pages()
755 	if (obj->madv != I915_MADV_WILLNEED)  in i915_gem_userptr_put_pages()
756 		obj->dirty = 0;  in i915_gem_userptr_put_pages()
758 	i915_gem_gtt_finish_object(obj);  in i915_gem_userptr_put_pages()
760 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {  in i915_gem_userptr_put_pages()
763 		if (obj->dirty)  in i915_gem_userptr_put_pages()
769 	obj->dirty = 0;  in i915_gem_userptr_put_pages()
771 	sg_free_table(obj->pages);  in i915_gem_userptr_put_pages()
772 	kfree(obj->pages);  in i915_gem_userptr_put_pages()
776 i915_gem_userptr_release(struct drm_i915_gem_object *obj)  in i915_gem_userptr_release()  argument
778 	i915_gem_userptr_release__mmu_notifier(obj);  in i915_gem_userptr_release()
779 	i915_gem_userptr_release__mm_struct(obj);  in i915_gem_userptr_release()
783 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)  in i915_gem_userptr_dmabuf_export()  argument
785 	if (obj->userptr.mmu_object)  in i915_gem_userptr_dmabuf_export()
788 	return i915_gem_userptr_init__mmu_notifier(obj, 0);  in i915_gem_userptr_dmabuf_export()
837 	struct drm_i915_gem_object *obj;  in i915_gem_userptr_ioctl()  local
859 	obj = i915_gem_object_alloc(dev);  in i915_gem_userptr_ioctl()
860 	if (obj == NULL)  in i915_gem_userptr_ioctl()
863 	drm_gem_private_object_init(dev, &obj->base, args->user_size);  in i915_gem_userptr_ioctl()
864 	i915_gem_object_init(obj, &i915_gem_userptr_ops);  in i915_gem_userptr_ioctl()
865 	obj->cache_level = I915_CACHE_LLC;  in i915_gem_userptr_ioctl()
866 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;  in i915_gem_userptr_ioctl()
867 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;  in i915_gem_userptr_ioctl()
869 	obj->userptr.ptr = args->user_ptr;  in i915_gem_userptr_ioctl()
870 	obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);  in i915_gem_userptr_ioctl()
876 	ret = i915_gem_userptr_init__mm_struct(obj);  in i915_gem_userptr_ioctl()
878 		ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);  in i915_gem_userptr_ioctl()
880 		ret = drm_gem_handle_create(file, &obj->base, &handle);  in i915_gem_userptr_ioctl()
883 	drm_gem_object_unreference_unlocked(&obj->base);  in i915_gem_userptr_ioctl()