Lines Matching refs:vm
159 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, in ppgtt_bind_vma()
167 vma->vm->clear_range(vma->vm, in ppgtt_unbind_vma()
457 static void gen8_initialize_pt(struct i915_address_space *vm, in gen8_initialize_pt() argument
462 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), in gen8_initialize_pt()
465 fill_px(vm->dev, pt, scratch_pte); in gen8_initialize_pt()
468 static void gen6_initialize_pt(struct i915_address_space *vm, in gen6_initialize_pt() argument
473 WARN_ON(px_dma(vm->scratch_page) == 0); in gen6_initialize_pt()
475 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), in gen6_initialize_pt()
478 fill32_px(vm->dev, pt, scratch_pte); in gen6_initialize_pt()
518 static void gen8_initialize_pd(struct i915_address_space *vm, in gen8_initialize_pd() argument
523 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); in gen8_initialize_pd()
525 fill_px(vm->dev, pd, scratch_pde); in gen8_initialize_pd()
599 static void gen8_initialize_pdp(struct i915_address_space *vm, in gen8_initialize_pdp() argument
604 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); in gen8_initialize_pdp()
606 fill_px(vm->dev, pdp, scratch_pdpe); in gen8_initialize_pdp()
609 static void gen8_initialize_pml4(struct i915_address_space *vm, in gen8_initialize_pml4() argument
614 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), in gen8_initialize_pml4()
617 fill_px(vm->dev, pml4, scratch_pml4e); in gen8_initialize_pml4()
696 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, in gen8_ppgtt_clear_pte_range() argument
703 container_of(vm, struct i915_hw_ppgtt, base); in gen8_ppgtt_clear_pte_range()
746 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) in gen8_ppgtt_clear_pte_range()
753 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, in gen8_ppgtt_clear_range() argument
759 container_of(vm, struct i915_hw_ppgtt, base); in gen8_ppgtt_clear_range()
760 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), in gen8_ppgtt_clear_range()
763 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { in gen8_ppgtt_clear_range()
764 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, in gen8_ppgtt_clear_range()
771 gen8_ppgtt_clear_pte_range(vm, pdp, start, length, in gen8_ppgtt_clear_range()
778 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, in gen8_ppgtt_insert_pte_entries() argument
785 container_of(vm, struct i915_hw_ppgtt, base); in gen8_ppgtt_insert_pte_entries()
807 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) in gen8_ppgtt_insert_pte_entries()
819 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, in gen8_ppgtt_insert_entries() argument
826 container_of(vm, struct i915_hw_ppgtt, base); in gen8_ppgtt_insert_entries()
831 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { in gen8_ppgtt_insert_entries()
832 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, in gen8_ppgtt_insert_entries()
840 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, in gen8_ppgtt_insert_entries()
863 static int gen8_init_scratch(struct i915_address_space *vm) in gen8_init_scratch() argument
865 struct drm_device *dev = vm->dev; in gen8_init_scratch()
867 vm->scratch_page = alloc_scratch_page(dev); in gen8_init_scratch()
868 if (IS_ERR(vm->scratch_page)) in gen8_init_scratch()
869 return PTR_ERR(vm->scratch_page); in gen8_init_scratch()
871 vm->scratch_pt = alloc_pt(dev); in gen8_init_scratch()
872 if (IS_ERR(vm->scratch_pt)) { in gen8_init_scratch()
873 free_scratch_page(dev, vm->scratch_page); in gen8_init_scratch()
874 return PTR_ERR(vm->scratch_pt); in gen8_init_scratch()
877 vm->scratch_pd = alloc_pd(dev); in gen8_init_scratch()
878 if (IS_ERR(vm->scratch_pd)) { in gen8_init_scratch()
879 free_pt(dev, vm->scratch_pt); in gen8_init_scratch()
880 free_scratch_page(dev, vm->scratch_page); in gen8_init_scratch()
881 return PTR_ERR(vm->scratch_pd); in gen8_init_scratch()
885 vm->scratch_pdp = alloc_pdp(dev); in gen8_init_scratch()
886 if (IS_ERR(vm->scratch_pdp)) { in gen8_init_scratch()
887 free_pd(dev, vm->scratch_pd); in gen8_init_scratch()
888 free_pt(dev, vm->scratch_pt); in gen8_init_scratch()
889 free_scratch_page(dev, vm->scratch_page); in gen8_init_scratch()
890 return PTR_ERR(vm->scratch_pdp); in gen8_init_scratch()
894 gen8_initialize_pt(vm, vm->scratch_pt); in gen8_init_scratch()
895 gen8_initialize_pd(vm, vm->scratch_pd); in gen8_init_scratch()
897 gen8_initialize_pdp(vm, vm->scratch_pdp); in gen8_init_scratch()
937 static void gen8_free_scratch(struct i915_address_space *vm) in gen8_free_scratch() argument
939 struct drm_device *dev = vm->dev; in gen8_free_scratch()
942 free_pdp(dev, vm->scratch_pdp); in gen8_free_scratch()
943 free_pd(dev, vm->scratch_pd); in gen8_free_scratch()
944 free_pt(dev, vm->scratch_pt); in gen8_free_scratch()
945 free_scratch_page(dev, vm->scratch_page); in gen8_free_scratch()
978 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) in gen8_ppgtt_cleanup() argument
981 container_of(vm, struct i915_hw_ppgtt, base); in gen8_ppgtt_cleanup()
983 if (intel_vgpu_active(vm->dev)) in gen8_ppgtt_cleanup()
991 gen8_free_scratch(vm); in gen8_ppgtt_cleanup()
1012 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, in gen8_ppgtt_alloc_pagetabs() argument
1018 struct drm_device *dev = vm->dev; in gen8_ppgtt_alloc_pagetabs()
1027 WARN_ON(pt == vm->scratch_pt); in gen8_ppgtt_alloc_pagetabs()
1035 gen8_initialize_pt(vm, pt); in gen8_ppgtt_alloc_pagetabs()
1038 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); in gen8_ppgtt_alloc_pagetabs()
1074 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, in gen8_ppgtt_alloc_page_directories() argument
1080 struct drm_device *dev = vm->dev; in gen8_ppgtt_alloc_page_directories()
1096 gen8_initialize_pd(vm, pd); in gen8_ppgtt_alloc_page_directories()
1099 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); in gen8_ppgtt_alloc_page_directories()
1128 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, in gen8_ppgtt_alloc_page_dirpointers() argument
1134 struct drm_device *dev = vm->dev; in gen8_ppgtt_alloc_page_dirpointers()
1147 gen8_initialize_pdp(vm, pdp); in gen8_ppgtt_alloc_page_dirpointers()
1150 trace_i915_page_directory_pointer_entry_alloc(vm, in gen8_ppgtt_alloc_page_dirpointers()
1213 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, in gen8_alloc_va_range_3lvl() argument
1219 container_of(vm, struct i915_hw_ppgtt, base); in gen8_alloc_va_range_3lvl()
1221 struct drm_device *dev = vm->dev; in gen8_alloc_va_range_3lvl()
1236 if (WARN_ON(start + length > vm->total)) in gen8_alloc_va_range_3lvl()
1244 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, in gen8_alloc_va_range_3lvl()
1253 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, in gen8_alloc_va_range_3lvl()
1324 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, in gen8_alloc_va_range_4lvl() argument
1331 container_of(vm, struct i915_hw_ppgtt, base); in gen8_alloc_va_range_4lvl()
1343 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, in gen8_alloc_va_range_4lvl()
1355 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); in gen8_alloc_va_range_4lvl()
1369 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); in gen8_alloc_va_range_4lvl()
1374 static int gen8_alloc_va_range(struct i915_address_space *vm, in gen8_alloc_va_range() argument
1378 container_of(vm, struct i915_hw_ppgtt, base); in gen8_alloc_va_range()
1380 if (USES_FULL_48BIT_PPGTT(vm->dev)) in gen8_alloc_va_range()
1381 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); in gen8_alloc_va_range()
1383 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); in gen8_alloc_va_range()
1446 struct i915_address_space *vm = &ppgtt->base; in gen8_dump_ppgtt() local
1449 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), in gen8_dump_ppgtt()
1452 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { in gen8_dump_ppgtt()
1559 struct i915_address_space *vm = &ppgtt->base; in gen6_dump_ppgtt() local
1566 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), in gen6_dump_ppgtt()
1793 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, in gen6_ppgtt_clear_range() argument
1799 container_of(vm, struct i915_hw_ppgtt, base); in gen6_ppgtt_clear_range()
1807 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), in gen6_ppgtt_clear_range()
1828 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, in gen6_ppgtt_insert_entries() argument
1834 container_of(vm, struct i915_hw_ppgtt, base); in gen6_ppgtt_insert_entries()
1847 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), in gen6_ppgtt_insert_entries()
1861 static int gen6_alloc_va_range(struct i915_address_space *vm, in gen6_alloc_va_range() argument
1865 struct drm_device *dev = vm->dev; in gen6_alloc_va_range()
1868 container_of(vm, struct i915_hw_ppgtt, base); in gen6_alloc_va_range()
1888 if (pt != vm->scratch_pt) { in gen6_alloc_va_range()
1902 gen6_initialize_pt(vm, pt); in gen6_alloc_va_range()
1906 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); in gen6_alloc_va_range()
1922 trace_i915_page_table_entry_map(vm, pde, pt, in gen6_alloc_va_range()
1943 ppgtt->pd.page_table[pde] = vm->scratch_pt; in gen6_alloc_va_range()
1944 free_pt(vm->dev, pt); in gen6_alloc_va_range()
1951 static int gen6_init_scratch(struct i915_address_space *vm) in gen6_init_scratch() argument
1953 struct drm_device *dev = vm->dev; in gen6_init_scratch()
1955 vm->scratch_page = alloc_scratch_page(dev); in gen6_init_scratch()
1956 if (IS_ERR(vm->scratch_page)) in gen6_init_scratch()
1957 return PTR_ERR(vm->scratch_page); in gen6_init_scratch()
1959 vm->scratch_pt = alloc_pt(dev); in gen6_init_scratch()
1960 if (IS_ERR(vm->scratch_pt)) { in gen6_init_scratch()
1961 free_scratch_page(dev, vm->scratch_page); in gen6_init_scratch()
1962 return PTR_ERR(vm->scratch_pt); in gen6_init_scratch()
1965 gen6_initialize_pt(vm, vm->scratch_pt); in gen6_init_scratch()
1970 static void gen6_free_scratch(struct i915_address_space *vm) in gen6_free_scratch() argument
1972 struct drm_device *dev = vm->dev; in gen6_free_scratch()
1974 free_pt(dev, vm->scratch_pt); in gen6_free_scratch()
1975 free_scratch_page(dev, vm->scratch_page); in gen6_free_scratch()
1978 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) in gen6_ppgtt_cleanup() argument
1981 container_of(vm, struct i915_hw_ppgtt, base); in gen6_ppgtt_cleanup()
1988 if (pt != vm->scratch_pt) in gen6_ppgtt_cleanup()
1992 gen6_free_scratch(vm); in gen6_ppgtt_cleanup()
1997 struct i915_address_space *vm = &ppgtt->base; in gen6_ppgtt_allocate_page_directories() local
2009 ret = gen6_init_scratch(vm); in gen6_ppgtt_allocate_page_directories()
2042 gen6_free_scratch(vm); in gen6_ppgtt_allocate_page_directories()
2124 static void i915_address_space_init(struct i915_address_space *vm, in i915_address_space_init() argument
2127 drm_mm_init(&vm->mm, vm->start, vm->total); in i915_address_space_init()
2128 vm->dev = dev_priv->dev; in i915_address_space_init()
2129 INIT_LIST_HEAD(&vm->active_list); in i915_address_space_init()
2130 INIT_LIST_HEAD(&vm->inactive_list); in i915_address_space_init()
2131 list_add_tail(&vm->global_link, &dev_priv->vm_list); in i915_address_space_init()
2343 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, in gen8_ggtt_insert_entries() argument
2348 struct drm_i915_private *dev_priv = vm->dev->dev_private; in gen8_ggtt_insert_entries()
2389 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, in gen6_ggtt_insert_entries() argument
2394 struct drm_i915_private *dev_priv = vm->dev->dev_private; in gen6_ggtt_insert_entries()
2404 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); in gen6_ggtt_insert_entries()
2416 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); in gen6_ggtt_insert_entries()
2427 static void gen8_ggtt_clear_range(struct i915_address_space *vm, in gen8_ggtt_clear_range() argument
2432 struct drm_i915_private *dev_priv = vm->dev->dev_private; in gen8_ggtt_clear_range()
2445 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), in gen8_ggtt_clear_range()
2453 static void gen6_ggtt_clear_range(struct i915_address_space *vm, in gen6_ggtt_clear_range() argument
2458 struct drm_i915_private *dev_priv = vm->dev->dev_private; in gen6_ggtt_clear_range()
2471 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), in gen6_ggtt_clear_range()
2479 static void i915_ggtt_insert_entries(struct i915_address_space *vm, in i915_ggtt_insert_entries() argument
2491 static void i915_ggtt_clear_range(struct i915_address_space *vm, in i915_ggtt_clear_range() argument
2517 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, in ggtt_bind_vma()
2535 struct drm_device *dev = vma->vm->dev; in aliasing_gtt_bind_vma()
2553 vma->vm->insert_entries(vma->vm, pages, in aliasing_gtt_bind_vma()
2570 struct drm_device *dev = vma->vm->dev; in ggtt_unbind_vma()
2578 vma->vm->clear_range(vma->vm, in ggtt_unbind_vma()
2744 struct i915_address_space *vm = &dev_priv->gtt.base; in i915_global_gtt_cleanup() local
2752 if (drm_mm_initialized(&vm->mm)) { in i915_global_gtt_cleanup()
2756 drm_mm_takedown(&vm->mm); in i915_global_gtt_cleanup()
2757 list_del(&vm->global_link); in i915_global_gtt_cleanup()
2760 vm->cleanup(vm); in i915_global_gtt_cleanup()
3044 static void gen6_gmch_remove(struct i915_address_space *vm) in gen6_gmch_remove() argument
3047 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); in gen6_gmch_remove()
3050 free_scratch_page(vm->dev, vm->scratch_page); in gen6_gmch_remove()
3082 static void i915_gmch_remove(struct i915_address_space *vm) in i915_gmch_remove() argument
3146 struct i915_address_space *vm; in i915_gem_restore_gtt_mappings() local
3159 vm = &dev_priv->gtt.base; in i915_gem_restore_gtt_mappings()
3163 if (vma->vm != vm) in i915_gem_restore_gtt_mappings()
3186 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { in i915_gem_restore_gtt_mappings()
3190 container_of(vm, struct i915_hw_ppgtt, in i915_gem_restore_gtt_mappings()
3193 if (i915_is_ggtt(vm)) in i915_gem_restore_gtt_mappings()
3206 struct i915_address_space *vm, in __i915_gem_vma_create() argument
3211 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) in __i915_gem_vma_create()
3221 vma->vm = vm; in __i915_gem_vma_create()
3224 if (i915_is_ggtt(vm)) in __i915_gem_vma_create()
3228 if (!i915_is_ggtt(vm)) in __i915_gem_vma_create()
3229 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); in __i915_gem_vma_create()
3236 struct i915_address_space *vm) in i915_gem_obj_lookup_or_create_vma() argument
3240 vma = i915_gem_obj_to_vma(obj, vm); in i915_gem_obj_lookup_or_create_vma()
3242 vma = __i915_gem_vma_create(obj, vm, in i915_gem_obj_lookup_or_create_vma()
3243 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); in i915_gem_obj_lookup_or_create_vma()
3499 if (vma->bound == 0 && vma->vm->allocate_va_range) { in i915_vma_bind()
3500 trace_i915_va_alloc(vma->vm, in i915_vma_bind()
3503 VM_TO_TRACE_NAME(vma->vm)); in i915_vma_bind()
3507 ret = vma->vm->allocate_va_range(vma->vm, in i915_vma_bind()
3515 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); in i915_vma_bind()