287 	struct i915_vma *vma, *next;  in drop_pages()  local
291 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)  in drop_pages()
292 		if (i915_vma_unbind(vma))  in drop_pages()
1578 		struct vm_area_struct *vma;  in i915_gem_mmap_ioctl()  local
1581 		vma = find_vma(mm, addr);  in i915_gem_mmap_ioctl()
1582 		if (vma)  in i915_gem_mmap_ioctl()
1583 			vma->vm_page_prot =  in i915_gem_mmap_ioctl()
1584 				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));  in i915_gem_mmap_ioctl()
1614 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)  in i915_gem_fault()  argument
1616 	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);  in i915_gem_fault()
1627 	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>  in i915_gem_fault()
1670 					   vma->vm_end - vma->vm_start,  in i915_gem_fault()
1675 			ret = vm_insert_pfn(vma,  in i915_gem_fault()
1676 					    (unsigned long)vma->vm_start + i * PAGE_SIZE,  in i915_gem_fault()
1684 		ret = vm_insert_pfn(vma,  in i915_gem_fault()
2212 void i915_vma_move_to_active(struct i915_vma *vma,  in i915_vma_move_to_active()  argument
2215 	list_move_tail(&vma->mm_list, &vma->vm->active_list);  in i915_vma_move_to_active()
2216 	return i915_gem_object_move_to_active(vma->obj, ring);  in i915_vma_move_to_active()
2222 	struct i915_vma *vma;  in i915_gem_object_move_to_inactive()  local
2227 	list_for_each_entry(vma, &obj->vma_list, vma_link) {  in i915_gem_object_move_to_inactive()
2228 		if (!list_empty(&vma->mm_list))  in i915_gem_object_move_to_inactive()
2229 			list_move_tail(&vma->mm_list, &vma->vm->inactive_list);  in i915_gem_object_move_to_inactive()
2959 int i915_vma_unbind(struct i915_vma *vma)  in i915_vma_unbind()  argument
2961 	struct drm_i915_gem_object *obj = vma->obj;  in i915_vma_unbind()
2965 	if (list_empty(&vma->vma_link))  in i915_vma_unbind()
2968 	if (!drm_mm_node_allocated(&vma->node)) {  in i915_vma_unbind()
2969 		i915_gem_vma_destroy(vma);  in i915_vma_unbind()
2973 	if (vma->pin_count)  in i915_vma_unbind()
2986 	if (i915_is_ggtt(vma->vm) &&  in i915_vma_unbind()
2987 	    vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {  in i915_vma_unbind()
2996 	trace_i915_vma_unbind(vma);  in i915_vma_unbind()
2998 	vma->unbind_vma(vma);  in i915_vma_unbind()
3000 	list_del_init(&vma->mm_list);  in i915_vma_unbind()
3001 	if (i915_is_ggtt(vma->vm)) {  in i915_vma_unbind()
3002 		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {  in i915_vma_unbind()
3004 		} else if (vma->ggtt_view.pages) {  in i915_vma_unbind()
3005 			sg_free_table(vma->ggtt_view.pages);  in i915_vma_unbind()
3006 			kfree(vma->ggtt_view.pages);  in i915_vma_unbind()
3008 		vma->ggtt_view.pages = NULL;  in i915_vma_unbind()
3011 	drm_mm_remove_node(&vma->node);  in i915_vma_unbind()
3012 	i915_gem_vma_destroy(vma);  in i915_vma_unbind()
3393 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,  in i915_gem_valid_gtt_space()  argument
3396 	struct drm_mm_node *gtt_space = &vma->node;  in i915_gem_valid_gtt_space()
3406 	if (vma->vm->mm.color_adjust == NULL)  in i915_gem_valid_gtt_space()
3443 	struct i915_vma *vma;  in i915_gem_object_bind_to_vm()  local
3487 	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :  in i915_gem_object_bind_to_vm()
3490 	if (IS_ERR(vma))  in i915_gem_object_bind_to_vm()
3494 	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,  in i915_gem_object_bind_to_vm()
3510 	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {  in i915_gem_object_bind_to_vm()
3520 	if (vma->vm->allocate_va_range) {  in i915_gem_object_bind_to_vm()
3521 		trace_i915_va_alloc(vma->vm, vma->node.start, vma->node.size,  in i915_gem_object_bind_to_vm()
3522 				VM_TO_TRACE_NAME(vma->vm));  in i915_gem_object_bind_to_vm()
3523 		ret = vma->vm->allocate_va_range(vma->vm,  in i915_gem_object_bind_to_vm()
3524 						vma->node.start,  in i915_gem_object_bind_to_vm()
3525 						vma->node.size);  in i915_gem_object_bind_to_vm()
3530 	trace_i915_vma_bind(vma, flags);  in i915_gem_object_bind_to_vm()
3531 	ret = i915_vma_bind(vma, obj->cache_level,  in i915_gem_object_bind_to_vm()
3537 	list_add_tail(&vma->mm_list, &vm->inactive_list);  in i915_gem_object_bind_to_vm()
3539 	return vma;  in i915_gem_object_bind_to_vm()
3544 	drm_mm_remove_node(&vma->node);  in i915_gem_object_bind_to_vm()
3546 	i915_gem_vma_destroy(vma);  in i915_gem_object_bind_to_vm()
3547 	vma = ERR_PTR(ret);  in i915_gem_object_bind_to_vm()
3550 	return vma;  in i915_gem_object_bind_to_vm()
3652 	struct i915_vma *vma;  in i915_gem_object_set_to_gtt_domain()  local
3707 	vma = i915_gem_obj_to_ggtt(obj);  in i915_gem_object_set_to_gtt_domain()
3708 	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)  in i915_gem_object_set_to_gtt_domain()
3709 		list_move_tail(&vma->mm_list,  in i915_gem_object_set_to_gtt_domain()
3719 	struct i915_vma *vma, *next;  in i915_gem_object_set_cache_level()  local
3730 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {  in i915_gem_object_set_cache_level()
3731 		if (!i915_gem_valid_gtt_space(vma, cache_level)) {  in i915_gem_object_set_cache_level()
3732 			ret = i915_vma_unbind(vma);  in i915_gem_object_set_cache_level()
3755 		list_for_each_entry(vma, &obj->vma_list, vma_link)  in i915_gem_object_set_cache_level()
3756 			if (drm_mm_node_allocated(&vma->node)) {  in i915_gem_object_set_cache_level()
3757 				ret = i915_vma_bind(vma, cache_level,  in i915_gem_object_set_cache_level()
3758 						    vma->bound & GLOBAL_BIND);  in i915_gem_object_set_cache_level()
3764 	list_for_each_entry(vma, &obj->vma_list, vma_link)  in i915_gem_object_set_cache_level()
3765 		vma->node.color = cache_level;  in i915_gem_object_set_cache_level()
3858 	struct i915_vma *vma;  in is_pin_display()  local
3860 	vma = i915_gem_obj_to_ggtt(obj);  in is_pin_display()
3861 	if (!vma)  in is_pin_display()
3871 	return vma->pin_count;  in is_pin_display()
4084 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)  in i915_vma_misplaced()  argument
4086 	struct drm_i915_gem_object *obj = vma->obj;  in i915_vma_misplaced()
4089 	    vma->node.start & (alignment - 1))  in i915_vma_misplaced()
4096 	    vma->node.start < (flags & PIN_OFFSET_MASK))  in i915_vma_misplaced()
4110 	struct i915_vma *vma;  in i915_gem_object_do_pin()  local
4126 	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :  in i915_gem_object_do_pin()
4129 	if (IS_ERR(vma))  in i915_gem_object_do_pin()
4130 		return PTR_ERR(vma);  in i915_gem_object_do_pin()
4132 	if (vma) {  in i915_gem_object_do_pin()
4133 		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))  in i915_gem_object_do_pin()
4136 		if (i915_vma_misplaced(vma, alignment, flags)) {  in i915_gem_object_do_pin()
4140 			WARN(vma->pin_count,  in i915_gem_object_do_pin()
4149 			ret = i915_vma_unbind(vma);  in i915_gem_object_do_pin()
4153 			vma = NULL;  in i915_gem_object_do_pin()
4157 	bound = vma ? vma->bound : 0;  in i915_gem_object_do_pin()
4158 	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {  in i915_gem_object_do_pin()
4163 		vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,  in i915_gem_object_do_pin()
4165 		if (IS_ERR(vma))  in i915_gem_object_do_pin()
4166 			return PTR_ERR(vma);  in i915_gem_object_do_pin()
4169 	if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND)) {  in i915_gem_object_do_pin()
4170 		ret = i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND);  in i915_gem_object_do_pin()
4175 	if ((bound ^ vma->bound) & GLOBAL_BIND) {  in i915_gem_object_do_pin()
4187 		fenceable = (vma->node.size == fence_size &&  in i915_gem_object_do_pin()
4188 			     (vma->node.start & (fence_alignment - 1)) == 0);  in i915_gem_object_do_pin()
4190 		mappable = (vma->node.start + fence_size <=  in i915_gem_object_do_pin()
4198 	vma->pin_count++;  in i915_gem_object_do_pin()
4233 	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);  in i915_gem_object_ggtt_unpin_view()  local
4235 	BUG_ON(!vma);  in i915_gem_object_ggtt_unpin_view()
4236 	WARN_ON(vma->pin_count == 0);  in i915_gem_object_ggtt_unpin_view()
4239 	if (--vma->pin_count == 0 && view->type == I915_GGTT_VIEW_NORMAL)  in i915_gem_object_ggtt_unpin_view()
4476 	struct i915_vma *vma, *next;  in i915_gem_free_object()  local
4482 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {  in i915_gem_free_object()
4485 		vma->pin_count = 0;  in i915_gem_free_object()
4486 		ret = i915_vma_unbind(vma);  in i915_gem_free_object()
4493 			WARN_ON(i915_vma_unbind(vma));  in i915_gem_free_object()
4538 	struct i915_vma *vma;  in i915_gem_obj_to_vma()  local
4539 	list_for_each_entry(vma, &obj->vma_list, vma_link) {  in i915_gem_obj_to_vma()
4540 		if (i915_is_ggtt(vma->vm) &&  in i915_gem_obj_to_vma()
4541 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)  in i915_gem_obj_to_vma()
4543 		if (vma->vm == vm)  in i915_gem_obj_to_vma()
4544 			return vma;  in i915_gem_obj_to_vma()
4553 	struct i915_vma *vma;  in i915_gem_obj_to_ggtt_view()  local
4558 	list_for_each_entry(vma, &obj->vma_list, vma_link)  in i915_gem_obj_to_ggtt_view()
4559 		if (vma->vm == ggtt &&  in i915_gem_obj_to_ggtt_view()
4560 		    i915_ggtt_view_equal(&vma->ggtt_view, view))  in i915_gem_obj_to_ggtt_view()
4561 			return vma;  in i915_gem_obj_to_ggtt_view()
4565 void i915_gem_vma_destroy(struct i915_vma *vma)  in i915_gem_vma_destroy()  argument
4568 	WARN_ON(vma->node.allocated);  in i915_gem_vma_destroy()
4571 	if (!list_empty(&vma->exec_list))  in i915_gem_vma_destroy()
4574 	vm = vma->vm;  in i915_gem_vma_destroy()
4579 	list_del(&vma->vma_link);  in i915_gem_vma_destroy()
4581 	kfree(vma);  in i915_gem_vma_destroy()
5098 	struct i915_vma *vma;  in i915_gem_obj_offset()  local
5102 	list_for_each_entry(vma, &o->vma_list, vma_link) {  in i915_gem_obj_offset()
5103 		if (i915_is_ggtt(vma->vm) &&  in i915_gem_obj_offset()
5104 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)  in i915_gem_obj_offset()
5106 		if (vma->vm == vm)  in i915_gem_obj_offset()
5107 			return vma->node.start;  in i915_gem_obj_offset()
5120 	struct i915_vma *vma;  in i915_gem_obj_ggtt_offset_view()  local
5122 	list_for_each_entry(vma, &o->vma_list, vma_link)  in i915_gem_obj_ggtt_offset_view()
5123 		if (vma->vm == ggtt &&  in i915_gem_obj_ggtt_offset_view()
5124 		    i915_ggtt_view_equal(&vma->ggtt_view, view))  in i915_gem_obj_ggtt_offset_view()
5125 			return vma->node.start;  in i915_gem_obj_ggtt_offset_view()
5134 	struct i915_vma *vma;  in i915_gem_obj_bound()  local
5136 	list_for_each_entry(vma, &o->vma_list, vma_link) {  in i915_gem_obj_bound()
5137 		if (i915_is_ggtt(vma->vm) &&  in i915_gem_obj_bound()
5138 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)  in i915_gem_obj_bound()
5140 		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))  in i915_gem_obj_bound()
5151 	struct i915_vma *vma;  in i915_gem_obj_ggtt_bound_view()  local
5153 	list_for_each_entry(vma, &o->vma_list, vma_link)  in i915_gem_obj_ggtt_bound_view()
5154 		if (vma->vm == ggtt &&  in i915_gem_obj_ggtt_bound_view()
5155 		    i915_ggtt_view_equal(&vma->ggtt_view, view) &&  in i915_gem_obj_ggtt_bound_view()
5156 		    drm_mm_node_allocated(&vma->node))  in i915_gem_obj_ggtt_bound_view()
5164 	struct i915_vma *vma;  in i915_gem_obj_bound_any()  local
5166 	list_for_each_entry(vma, &o->vma_list, vma_link)  in i915_gem_obj_bound_any()
5167 		if (drm_mm_node_allocated(&vma->node))  in i915_gem_obj_bound_any()
5177 	struct i915_vma *vma;  in i915_gem_obj_size()  local
5183 	list_for_each_entry(vma, &o->vma_list, vma_link) {  in i915_gem_obj_size()
5184 		if (i915_is_ggtt(vma->vm) &&  in i915_gem_obj_size()
5185 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)  in i915_gem_obj_size()
5187 		if (vma->vm == vm)  in i915_gem_obj_size()
5188 			return vma->node.size;  in i915_gem_obj_size()
5195 	struct i915_vma *vma;  in i915_gem_obj_is_pinned()  local
5196 	list_for_each_entry(vma, &obj->vma_list, vma_link) {  in i915_gem_obj_is_pinned()
5197 		if (i915_is_ggtt(vma->vm) &&  in i915_gem_obj_is_pinned()
5198 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)  in i915_gem_obj_is_pinned()
5200 		if (vma->pin_count > 0)  in i915_gem_obj_is_pinned()