Lines Matching refs:obj
41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
44 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
47 i915_gem_object_retire(struct drm_i915_gem_object *obj);
50 struct drm_i915_gem_object *obj);
51 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
61 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) in cpu_write_needs_clflush() argument
63 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) in cpu_write_needs_clflush()
66 return obj->pin_display; in cpu_write_needs_clflush()
69 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) in i915_gem_object_fence_lost() argument
71 if (obj->tiling_mode) in i915_gem_object_fence_lost()
72 i915_gem_release_mmap(obj); in i915_gem_object_fence_lost()
77 obj->fence_dirty = false; in i915_gem_object_fence_lost()
78 obj->fence_reg = I915_FENCE_REG_NONE; in i915_gem_object_fence_lost()
152 struct drm_i915_gem_object *obj; in i915_gem_get_aperture_ioctl() local
157 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) in i915_gem_get_aperture_ioctl()
158 if (i915_gem_obj_is_pinned(obj)) in i915_gem_get_aperture_ioctl()
159 pinned += i915_gem_obj_ggtt_size(obj); in i915_gem_get_aperture_ioctl()
169 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages_phys() argument
171 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_get_pages_phys()
172 char *vaddr = obj->phys_handle->vaddr; in i915_gem_object_get_pages_phys()
177 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) in i915_gem_object_get_pages_phys()
180 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { in i915_gem_object_get_pages_phys()
197 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_get_pages_phys()
210 sg->length = obj->base.size; in i915_gem_object_get_pages_phys()
212 sg_dma_address(sg) = obj->phys_handle->busaddr; in i915_gem_object_get_pages_phys()
213 sg_dma_len(sg) = obj->base.size; in i915_gem_object_get_pages_phys()
215 obj->pages = st; in i915_gem_object_get_pages_phys()
216 obj->has_dma_mapping = true; in i915_gem_object_get_pages_phys()
221 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages_phys() argument
225 BUG_ON(obj->madv == __I915_MADV_PURGED); in i915_gem_object_put_pages_phys()
227 ret = i915_gem_object_set_to_cpu_domain(obj, true); in i915_gem_object_put_pages_phys()
233 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_put_pages_phys()
236 if (obj->madv == I915_MADV_DONTNEED) in i915_gem_object_put_pages_phys()
237 obj->dirty = 0; in i915_gem_object_put_pages_phys()
239 if (obj->dirty) { in i915_gem_object_put_pages_phys()
240 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_put_pages_phys()
241 char *vaddr = obj->phys_handle->vaddr; in i915_gem_object_put_pages_phys()
244 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { in i915_gem_object_put_pages_phys()
258 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_object_put_pages_phys()
263 obj->dirty = 0; in i915_gem_object_put_pages_phys()
266 sg_free_table(obj->pages); in i915_gem_object_put_pages_phys()
267 kfree(obj->pages); in i915_gem_object_put_pages_phys()
269 obj->has_dma_mapping = false; in i915_gem_object_put_pages_phys()
273 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) in i915_gem_object_release_phys() argument
275 drm_pci_free(obj->base.dev, obj->phys_handle); in i915_gem_object_release_phys()
285 drop_pages(struct drm_i915_gem_object *obj) in drop_pages() argument
290 drm_gem_object_reference(&obj->base); in drop_pages()
291 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) in drop_pages()
295 ret = i915_gem_object_put_pages(obj); in drop_pages()
296 drm_gem_object_unreference(&obj->base); in drop_pages()
302 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, in i915_gem_object_attach_phys() argument
308 if (obj->phys_handle) { in i915_gem_object_attach_phys()
309 if ((unsigned long)obj->phys_handle->vaddr & (align -1)) in i915_gem_object_attach_phys()
315 if (obj->madv != I915_MADV_WILLNEED) in i915_gem_object_attach_phys()
318 if (obj->base.filp == NULL) in i915_gem_object_attach_phys()
321 ret = drop_pages(obj); in i915_gem_object_attach_phys()
326 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); in i915_gem_object_attach_phys()
330 obj->phys_handle = phys; in i915_gem_object_attach_phys()
331 obj->ops = &i915_gem_phys_ops; in i915_gem_object_attach_phys()
333 return i915_gem_object_get_pages(obj); in i915_gem_object_attach_phys()
337 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, in i915_gem_phys_pwrite() argument
341 struct drm_device *dev = obj->base.dev; in i915_gem_phys_pwrite()
342 void *vaddr = obj->phys_handle->vaddr + args->offset; in i915_gem_phys_pwrite()
349 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_phys_pwrite()
353 intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); in i915_gem_phys_pwrite()
374 intel_fb_obj_flush(obj, false); in i915_gem_phys_pwrite()
384 void i915_gem_object_free(struct drm_i915_gem_object *obj) in i915_gem_object_free() argument
386 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_free()
387 kmem_cache_free(dev_priv->slab, obj); in i915_gem_object_free()
396 struct drm_i915_gem_object *obj; in i915_gem_create() local
405 obj = i915_gem_alloc_object(dev, size); in i915_gem_create()
406 if (obj == NULL) in i915_gem_create()
409 ret = drm_gem_handle_create(file, &obj->base, &handle); in i915_gem_create()
411 drm_gem_object_unreference_unlocked(&obj->base); in i915_gem_create()
501 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, in i915_gem_obj_prepare_shmem_read() argument
508 if (!obj->base.filp) in i915_gem_obj_prepare_shmem_read()
511 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { in i915_gem_obj_prepare_shmem_read()
516 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev, in i915_gem_obj_prepare_shmem_read()
517 obj->cache_level); in i915_gem_obj_prepare_shmem_read()
518 ret = i915_gem_object_wait_rendering(obj, true); in i915_gem_obj_prepare_shmem_read()
522 i915_gem_object_retire(obj); in i915_gem_obj_prepare_shmem_read()
525 ret = i915_gem_object_get_pages(obj); in i915_gem_obj_prepare_shmem_read()
529 i915_gem_object_pin_pages(obj); in i915_gem_obj_prepare_shmem_read()
613 struct drm_i915_gem_object *obj, in i915_gem_shmem_pread() argument
629 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); in i915_gem_shmem_pread()
631 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); in i915_gem_shmem_pread()
637 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, in i915_gem_shmem_pread()
691 i915_gem_object_unpin_pages(obj); in i915_gem_shmem_pread()
706 struct drm_i915_gem_object *obj; in i915_gem_pread_ioctl() local
721 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_pread_ioctl()
722 if (&obj->base == NULL) { in i915_gem_pread_ioctl()
728 if (args->offset > obj->base.size || in i915_gem_pread_ioctl()
729 args->size > obj->base.size - args->offset) { in i915_gem_pread_ioctl()
737 if (!obj->base.filp) { in i915_gem_pread_ioctl()
742 trace_i915_gem_object_pread(obj, args->offset, args->size); in i915_gem_pread_ioctl()
744 ret = i915_gem_shmem_pread(dev, obj, args, file); in i915_gem_pread_ioctl()
747 drm_gem_object_unreference(&obj->base); in i915_gem_pread_ioctl()
782 struct drm_i915_gem_object *obj, in i915_gem_gtt_pwrite_fast() argument
792 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); in i915_gem_gtt_pwrite_fast()
796 ret = i915_gem_object_set_to_gtt_domain(obj, true); in i915_gem_gtt_pwrite_fast()
800 ret = i915_gem_object_put_fence(obj); in i915_gem_gtt_pwrite_fast()
807 offset = i915_gem_obj_ggtt_offset(obj) + args->offset; in i915_gem_gtt_pwrite_fast()
809 intel_fb_obj_invalidate(obj, NULL, ORIGIN_GTT); in i915_gem_gtt_pwrite_fast()
840 intel_fb_obj_flush(obj, false); in i915_gem_gtt_pwrite_fast()
842 i915_gem_object_ggtt_unpin(obj); in i915_gem_gtt_pwrite_fast()
914 struct drm_i915_gem_object *obj, in i915_gem_shmem_pwrite() argument
931 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); in i915_gem_shmem_pwrite()
933 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { in i915_gem_shmem_pwrite()
938 needs_clflush_after = cpu_write_needs_clflush(obj); in i915_gem_shmem_pwrite()
939 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_shmem_pwrite()
943 i915_gem_object_retire(obj); in i915_gem_shmem_pwrite()
947 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) in i915_gem_shmem_pwrite()
949 !cpu_cache_is_coherent(dev, obj->cache_level); in i915_gem_shmem_pwrite()
951 ret = i915_gem_object_get_pages(obj); in i915_gem_shmem_pwrite()
955 intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); in i915_gem_shmem_pwrite()
957 i915_gem_object_pin_pages(obj); in i915_gem_shmem_pwrite()
960 obj->dirty = 1; in i915_gem_shmem_pwrite()
962 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, in i915_gem_shmem_pwrite()
1017 i915_gem_object_unpin_pages(obj); in i915_gem_shmem_pwrite()
1026 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { in i915_gem_shmem_pwrite()
1027 if (i915_gem_clflush_object(obj, obj->pin_display)) in i915_gem_shmem_pwrite()
1035 intel_fb_obj_flush(obj, false); in i915_gem_shmem_pwrite()
1050 struct drm_i915_gem_object *obj; in i915_gem_pwrite_ioctl() local
1074 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_pwrite_ioctl()
1075 if (&obj->base == NULL) { in i915_gem_pwrite_ioctl()
1081 if (args->offset > obj->base.size || in i915_gem_pwrite_ioctl()
1082 args->size > obj->base.size - args->offset) { in i915_gem_pwrite_ioctl()
1090 if (!obj->base.filp) { in i915_gem_pwrite_ioctl()
1095 trace_i915_gem_object_pwrite(obj, args->offset, args->size); in i915_gem_pwrite_ioctl()
1104 if (obj->tiling_mode == I915_TILING_NONE && in i915_gem_pwrite_ioctl()
1105 obj->base.write_domain != I915_GEM_DOMAIN_CPU && in i915_gem_pwrite_ioctl()
1106 cpu_write_needs_clflush(obj)) { in i915_gem_pwrite_ioctl()
1107 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); in i915_gem_pwrite_ioctl()
1114 if (obj->phys_handle) in i915_gem_pwrite_ioctl()
1115 ret = i915_gem_phys_pwrite(obj, args, file); in i915_gem_pwrite_ioctl()
1117 ret = i915_gem_shmem_pwrite(dev, obj, args, file); in i915_gem_pwrite_ioctl()
1121 drm_gem_object_unreference(&obj->base); in i915_gem_pwrite_ioctl()
1360 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj) in i915_gem_object_wait_rendering__tail() argument
1362 if (!obj->active) in i915_gem_object_wait_rendering__tail()
1372 i915_gem_request_assign(&obj->last_write_req, NULL); in i915_gem_object_wait_rendering__tail()
1382 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, in i915_gem_object_wait_rendering() argument
1388 req = readonly ? obj->last_write_req : obj->last_read_req; in i915_gem_object_wait_rendering()
1396 return i915_gem_object_wait_rendering__tail(obj); in i915_gem_object_wait_rendering()
1403 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, in i915_gem_object_wait_rendering__nonblocking() argument
1408 struct drm_device *dev = obj->base.dev; in i915_gem_object_wait_rendering__nonblocking()
1416 req = readonly ? obj->last_write_req : obj->last_read_req; in i915_gem_object_wait_rendering__nonblocking()
1437 return i915_gem_object_wait_rendering__tail(obj); in i915_gem_object_wait_rendering__nonblocking()
1449 struct drm_i915_gem_object *obj; in i915_gem_set_domain_ioctl() local
1471 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_set_domain_ioctl()
1472 if (&obj->base == NULL) { in i915_gem_set_domain_ioctl()
1481 ret = i915_gem_object_wait_rendering__nonblocking(obj, in i915_gem_set_domain_ioctl()
1488 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); in i915_gem_set_domain_ioctl()
1490 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); in i915_gem_set_domain_ioctl()
1493 drm_gem_object_unreference(&obj->base); in i915_gem_set_domain_ioctl()
1507 struct drm_i915_gem_object *obj; in i915_gem_sw_finish_ioctl() local
1514 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_sw_finish_ioctl()
1515 if (&obj->base == NULL) { in i915_gem_sw_finish_ioctl()
1521 if (obj->pin_display) in i915_gem_sw_finish_ioctl()
1522 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_sw_finish_ioctl()
1524 drm_gem_object_unreference(&obj->base); in i915_gem_sw_finish_ioctl()
1552 struct drm_gem_object *obj; in i915_gem_mmap_ioctl() local
1561 obj = drm_gem_object_lookup(dev, file, args->handle); in i915_gem_mmap_ioctl()
1562 if (obj == NULL) in i915_gem_mmap_ioctl()
1568 if (!obj->filp) { in i915_gem_mmap_ioctl()
1569 drm_gem_object_unreference_unlocked(obj); in i915_gem_mmap_ioctl()
1573 addr = vm_mmap(obj->filp, 0, args->size, in i915_gem_mmap_ioctl()
1589 drm_gem_object_unreference_unlocked(obj); in i915_gem_mmap_ioctl()
1616 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data); in i915_gem_fault() local
1617 struct drm_device *dev = obj->base.dev; in i915_gem_fault()
1634 trace_i915_gem_object_fault(obj, page_offset, true, write); in i915_gem_fault()
1641 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); in i915_gem_fault()
1646 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { in i915_gem_fault()
1652 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE); in i915_gem_fault()
1656 ret = i915_gem_object_set_to_gtt_domain(obj, write); in i915_gem_fault()
1660 ret = i915_gem_object_get_fence(obj); in i915_gem_fault()
1665 pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj); in i915_gem_fault()
1668 if (!obj->fault_mappable) { in i915_gem_fault()
1671 obj->base.size); in i915_gem_fault()
1682 obj->fault_mappable = true; in i915_gem_fault()
1688 i915_gem_object_ggtt_unpin(obj); in i915_gem_fault()
1752 i915_gem_release_mmap(struct drm_i915_gem_object *obj) in i915_gem_release_mmap() argument
1754 if (!obj->fault_mappable) in i915_gem_release_mmap()
1757 drm_vma_node_unmap(&obj->base.vma_node, in i915_gem_release_mmap()
1758 obj->base.dev->anon_inode->i_mapping); in i915_gem_release_mmap()
1759 obj->fault_mappable = false; in i915_gem_release_mmap()
1765 struct drm_i915_gem_object *obj; in i915_gem_release_all_mmaps() local
1767 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) in i915_gem_release_all_mmaps()
1768 i915_gem_release_mmap(obj); in i915_gem_release_all_mmaps()
1818 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) in i915_gem_object_create_mmap_offset() argument
1820 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_create_mmap_offset()
1823 if (drm_vma_node_has_offset(&obj->base.vma_node)) in i915_gem_object_create_mmap_offset()
1828 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
1840 obj->base.size >> PAGE_SHIFT, in i915_gem_object_create_mmap_offset()
1844 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
1849 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
1856 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) in i915_gem_object_free_mmap_offset() argument
1858 drm_gem_free_mmap_offset(&obj->base); in i915_gem_object_free_mmap_offset()
1868 struct drm_i915_gem_object *obj; in i915_gem_mmap_gtt() local
1875 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); in i915_gem_mmap_gtt()
1876 if (&obj->base == NULL) { in i915_gem_mmap_gtt()
1881 if (obj->base.size > dev_priv->gtt.mappable_end) { in i915_gem_mmap_gtt()
1886 if (obj->madv != I915_MADV_WILLNEED) { in i915_gem_mmap_gtt()
1892 ret = i915_gem_object_create_mmap_offset(obj); in i915_gem_mmap_gtt()
1896 *offset = drm_vma_node_offset_addr(&obj->base.vma_node); in i915_gem_mmap_gtt()
1899 drm_gem_object_unreference(&obj->base); in i915_gem_mmap_gtt()
1931 i915_gem_object_truncate(struct drm_i915_gem_object *obj) in i915_gem_object_truncate() argument
1933 i915_gem_object_free_mmap_offset(obj); in i915_gem_object_truncate()
1935 if (obj->base.filp == NULL) in i915_gem_object_truncate()
1943 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); in i915_gem_object_truncate()
1944 obj->madv = __I915_MADV_PURGED; in i915_gem_object_truncate()
1949 i915_gem_object_invalidate(struct drm_i915_gem_object *obj) in i915_gem_object_invalidate() argument
1953 switch (obj->madv) { in i915_gem_object_invalidate()
1955 i915_gem_object_truncate(obj); in i915_gem_object_invalidate()
1960 if (obj->base.filp == NULL) in i915_gem_object_invalidate()
1963 mapping = file_inode(obj->base.filp)->i_mapping, in i915_gem_object_invalidate()
1968 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages_gtt() argument
1973 BUG_ON(obj->madv == __I915_MADV_PURGED); in i915_gem_object_put_pages_gtt()
1975 ret = i915_gem_object_set_to_cpu_domain(obj, true); in i915_gem_object_put_pages_gtt()
1981 i915_gem_clflush_object(obj, true); in i915_gem_object_put_pages_gtt()
1982 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_put_pages_gtt()
1985 if (i915_gem_object_needs_bit17_swizzle(obj)) in i915_gem_object_put_pages_gtt()
1986 i915_gem_object_save_bit_17_swizzle(obj); in i915_gem_object_put_pages_gtt()
1988 if (obj->madv == I915_MADV_DONTNEED) in i915_gem_object_put_pages_gtt()
1989 obj->dirty = 0; in i915_gem_object_put_pages_gtt()
1991 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { in i915_gem_object_put_pages_gtt()
1994 if (obj->dirty) in i915_gem_object_put_pages_gtt()
1997 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_object_put_pages_gtt()
2002 obj->dirty = 0; in i915_gem_object_put_pages_gtt()
2004 sg_free_table(obj->pages); in i915_gem_object_put_pages_gtt()
2005 kfree(obj->pages); in i915_gem_object_put_pages_gtt()
2009 i915_gem_object_put_pages(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages() argument
2011 const struct drm_i915_gem_object_ops *ops = obj->ops; in i915_gem_object_put_pages()
2013 if (obj->pages == NULL) in i915_gem_object_put_pages()
2016 if (obj->pages_pin_count) in i915_gem_object_put_pages()
2019 BUG_ON(i915_gem_obj_bound_any(obj)); in i915_gem_object_put_pages()
2024 list_del(&obj->global_list); in i915_gem_object_put_pages()
2026 ops->put_pages(obj); in i915_gem_object_put_pages()
2027 obj->pages = NULL; in i915_gem_object_put_pages()
2029 i915_gem_object_invalidate(obj); in i915_gem_object_put_pages()
2035 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages_gtt() argument
2037 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_get_pages_gtt()
2051 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); in i915_gem_object_get_pages_gtt()
2052 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); in i915_gem_object_get_pages_gtt()
2058 page_count = obj->base.size / PAGE_SIZE; in i915_gem_object_get_pages_gtt()
2069 mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_get_pages_gtt()
2120 obj->pages = st; in i915_gem_object_get_pages_gtt()
2122 if (i915_gem_object_needs_bit17_swizzle(obj)) in i915_gem_object_get_pages_gtt()
2123 i915_gem_object_do_bit_17_swizzle(obj); in i915_gem_object_get_pages_gtt()
2125 if (obj->tiling_mode != I915_TILING_NONE && in i915_gem_object_get_pages_gtt()
2127 i915_gem_object_pin_pages(obj); in i915_gem_object_get_pages_gtt()
2160 i915_gem_object_get_pages(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages() argument
2162 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_get_pages()
2163 const struct drm_i915_gem_object_ops *ops = obj->ops; in i915_gem_object_get_pages()
2166 if (obj->pages) in i915_gem_object_get_pages()
2169 if (obj->madv != I915_MADV_WILLNEED) { in i915_gem_object_get_pages()
2174 BUG_ON(obj->pages_pin_count); in i915_gem_object_get_pages()
2176 ret = ops->get_pages(obj); in i915_gem_object_get_pages()
2180 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); in i915_gem_object_get_pages()
2185 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, in i915_gem_object_move_to_active() argument
2194 old_ring = i915_gem_request_get_ring(obj->last_read_req); in i915_gem_object_move_to_active()
2196 if (old_ring != ring && obj->last_write_req) { in i915_gem_object_move_to_active()
2198 i915_gem_request_assign(&obj->last_write_req, req); in i915_gem_object_move_to_active()
2202 if (!obj->active) { in i915_gem_object_move_to_active()
2203 drm_gem_object_reference(&obj->base); in i915_gem_object_move_to_active()
2204 obj->active = 1; in i915_gem_object_move_to_active()
2207 list_move_tail(&obj->ring_list, &ring->active_list); in i915_gem_object_move_to_active()
2209 i915_gem_request_assign(&obj->last_read_req, req); in i915_gem_object_move_to_active()
2216 return i915_gem_object_move_to_active(vma->obj, ring); in i915_vma_move_to_active()
2220 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) in i915_gem_object_move_to_inactive() argument
2224 BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); in i915_gem_object_move_to_inactive()
2225 BUG_ON(!obj->active); in i915_gem_object_move_to_inactive()
2227 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_object_move_to_inactive()
2232 intel_fb_obj_flush(obj, true); in i915_gem_object_move_to_inactive()
2234 list_del_init(&obj->ring_list); in i915_gem_object_move_to_inactive()
2236 i915_gem_request_assign(&obj->last_read_req, NULL); in i915_gem_object_move_to_inactive()
2237 i915_gem_request_assign(&obj->last_write_req, NULL); in i915_gem_object_move_to_inactive()
2238 obj->base.write_domain = 0; in i915_gem_object_move_to_inactive()
2240 i915_gem_request_assign(&obj->last_fenced_req, NULL); in i915_gem_object_move_to_inactive()
2242 obj->active = 0; in i915_gem_object_move_to_inactive()
2243 drm_gem_object_unreference(&obj->base); in i915_gem_object_move_to_inactive()
2249 i915_gem_object_retire(struct drm_i915_gem_object *obj) in i915_gem_object_retire() argument
2251 if (obj->last_read_req == NULL) in i915_gem_object_retire()
2254 if (i915_gem_request_completed(obj->last_read_req, true)) in i915_gem_object_retire()
2255 i915_gem_object_move_to_inactive(obj); in i915_gem_object_retire()
2330 struct drm_i915_gem_object *obj) in __i915_add_request() argument
2392 request->batch_obj = obj; in __i915_add_request()
2561 struct drm_i915_gem_object *obj; in i915_gem_reset_ring_cleanup() local
2563 obj = list_first_entry(&ring->active_list, in i915_gem_reset_ring_cleanup()
2567 i915_gem_object_move_to_inactive(obj); in i915_gem_reset_ring_cleanup()
2623 if (reg->obj) { in i915_gem_restore_fences()
2624 i915_gem_object_update_fence(reg->obj, reg, in i915_gem_restore_fences()
2625 reg->obj->tiling_mode); in i915_gem_restore_fences()
2697 struct drm_i915_gem_object *obj; in i915_gem_retire_requests_ring() local
2699 obj = list_first_entry(&ring->active_list, in i915_gem_retire_requests_ring()
2703 if (!i915_gem_request_completed(obj->last_read_req, true)) in i915_gem_retire_requests_ring()
2706 i915_gem_object_move_to_inactive(obj); in i915_gem_retire_requests_ring()
2782 i915_gem_object_flush_active(struct drm_i915_gem_object *obj) in i915_gem_object_flush_active() argument
2787 if (obj->active) { in i915_gem_object_flush_active()
2788 ring = i915_gem_request_get_ring(obj->last_read_req); in i915_gem_object_flush_active()
2790 ret = i915_gem_check_olr(obj->last_read_req); in i915_gem_object_flush_active()
2827 struct drm_i915_gem_object *obj; in i915_gem_wait_ioctl() local
2839 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); in i915_gem_wait_ioctl()
2840 if (&obj->base == NULL) { in i915_gem_wait_ioctl()
2846 ret = i915_gem_object_flush_active(obj); in i915_gem_wait_ioctl()
2850 if (!obj->active || !obj->last_read_req) in i915_gem_wait_ioctl()
2853 req = obj->last_read_req; in i915_gem_wait_ioctl()
2863 drm_gem_object_unreference(&obj->base); in i915_gem_wait_ioctl()
2877 drm_gem_object_unreference(&obj->base); in i915_gem_wait_ioctl()
2895 i915_gem_object_sync(struct drm_i915_gem_object *obj, in i915_gem_object_sync() argument
2902 from = i915_gem_request_get_ring(obj->last_read_req); in i915_gem_object_sync()
2907 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) in i915_gem_object_sync()
2908 return i915_gem_object_wait_rendering(obj, false); in i915_gem_object_sync()
2912 seqno = i915_gem_request_get_seqno(obj->last_read_req); in i915_gem_object_sync()
2918 ret = i915_gem_check_olr(obj->last_read_req); in i915_gem_object_sync()
2922 trace_i915_gem_ring_sync_to(from, to, obj->last_read_req); in i915_gem_object_sync()
2930 i915_gem_request_get_seqno(obj->last_read_req); in i915_gem_object_sync()
2935 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_finish_gtt() argument
2940 i915_gem_release_mmap(obj); in i915_gem_object_finish_gtt()
2942 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) in i915_gem_object_finish_gtt()
2948 old_read_domains = obj->base.read_domains; in i915_gem_object_finish_gtt()
2949 old_write_domain = obj->base.write_domain; in i915_gem_object_finish_gtt()
2951 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; in i915_gem_object_finish_gtt()
2952 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; in i915_gem_object_finish_gtt()
2954 trace_i915_gem_object_change_domain(obj, in i915_gem_object_finish_gtt()
2961 struct drm_i915_gem_object *obj = vma->obj; in i915_vma_unbind() local
2962 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_vma_unbind()
2976 BUG_ON(obj->pages == NULL); in i915_vma_unbind()
2978 ret = i915_gem_object_finish_gpu(obj); in i915_vma_unbind()
2988 i915_gem_object_finish_gtt(obj); in i915_vma_unbind()
2991 ret = i915_gem_object_put_fence(obj); in i915_vma_unbind()
3003 obj->map_and_fenceable = false; in i915_vma_unbind()
3016 if (list_empty(&obj->vma_list)) { in i915_vma_unbind()
3019 i915_gem_object_retire(obj); in i915_vma_unbind()
3021 i915_gem_gtt_finish_object(obj); in i915_vma_unbind()
3022 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); in i915_vma_unbind()
3029 i915_gem_object_unpin_pages(obj); in i915_vma_unbind()
3057 struct drm_i915_gem_object *obj) in i965_write_fence_reg() argument
3085 if (obj) { in i965_write_fence_reg()
3086 u32 size = i915_gem_obj_ggtt_size(obj); in i965_write_fence_reg()
3090 if (obj->tiling_mode != I915_TILING_NONE) { in i965_write_fence_reg()
3091 uint32_t row_size = obj->stride * in i965_write_fence_reg()
3092 (obj->tiling_mode == I915_TILING_Y ? 32 : 8); in i965_write_fence_reg()
3096 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & in i965_write_fence_reg()
3098 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; in i965_write_fence_reg()
3099 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; in i965_write_fence_reg()
3100 if (obj->tiling_mode == I915_TILING_Y) in i965_write_fence_reg()
3116 struct drm_i915_gem_object *obj) in i915_write_fence_reg() argument
3121 if (obj) { in i915_write_fence_reg()
3122 u32 size = i915_gem_obj_ggtt_size(obj); in i915_write_fence_reg()
3126 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || in i915_write_fence_reg()
3128 (i915_gem_obj_ggtt_offset(obj) & (size - 1)), in i915_write_fence_reg()
3130 i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); in i915_write_fence_reg()
3132 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) in i915_write_fence_reg()
3138 pitch_val = obj->stride / tile_width; in i915_write_fence_reg()
3141 val = i915_gem_obj_ggtt_offset(obj); in i915_write_fence_reg()
3142 if (obj->tiling_mode == I915_TILING_Y) in i915_write_fence_reg()
3160 struct drm_i915_gem_object *obj) in i830_write_fence_reg() argument
3165 if (obj) { in i830_write_fence_reg()
3166 u32 size = i915_gem_obj_ggtt_size(obj); in i830_write_fence_reg()
3169 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || in i830_write_fence_reg()
3171 (i915_gem_obj_ggtt_offset(obj) & (size - 1)), in i830_write_fence_reg()
3173 i915_gem_obj_ggtt_offset(obj), size); in i830_write_fence_reg()
3175 pitch_val = obj->stride / 128; in i830_write_fence_reg()
3178 val = i915_gem_obj_ggtt_offset(obj); in i830_write_fence_reg()
3179 if (obj->tiling_mode == I915_TILING_Y) in i830_write_fence_reg()
3191 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) in i915_gem_object_needs_mb() argument
3193 return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT; in i915_gem_object_needs_mb()
3197 struct drm_i915_gem_object *obj) in i915_gem_write_fence() argument
3204 if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) in i915_gem_write_fence()
3207 WARN(obj && (!obj->stride || !obj->tiling_mode), in i915_gem_write_fence()
3209 obj->stride, obj->tiling_mode); in i915_gem_write_fence()
3212 i830_write_fence_reg(dev, reg, obj); in i915_gem_write_fence()
3214 i915_write_fence_reg(dev, reg, obj); in i915_gem_write_fence()
3216 i965_write_fence_reg(dev, reg, obj); in i915_gem_write_fence()
3221 if (i915_gem_object_needs_mb(obj)) in i915_gem_write_fence()
3231 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, in i915_gem_object_update_fence() argument
3235 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_update_fence()
3238 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); in i915_gem_object_update_fence()
3241 obj->fence_reg = reg; in i915_gem_object_update_fence()
3242 fence->obj = obj; in i915_gem_object_update_fence()
3245 obj->fence_reg = I915_FENCE_REG_NONE; in i915_gem_object_update_fence()
3246 fence->obj = NULL; in i915_gem_object_update_fence()
3249 obj->fence_dirty = false; in i915_gem_object_update_fence()
3253 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) in i915_gem_object_wait_fence() argument
3255 if (obj->last_fenced_req) { in i915_gem_object_wait_fence()
3256 int ret = i915_wait_request(obj->last_fenced_req); in i915_gem_object_wait_fence()
3260 i915_gem_request_assign(&obj->last_fenced_req, NULL); in i915_gem_object_wait_fence()
3267 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) in i915_gem_object_put_fence() argument
3269 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_put_fence()
3273 ret = i915_gem_object_wait_fence(obj); in i915_gem_object_put_fence()
3277 if (obj->fence_reg == I915_FENCE_REG_NONE) in i915_gem_object_put_fence()
3280 fence = &dev_priv->fence_regs[obj->fence_reg]; in i915_gem_object_put_fence()
3285 i915_gem_object_fence_lost(obj); in i915_gem_object_put_fence()
3286 i915_gem_object_update_fence(obj, fence, false); in i915_gem_object_put_fence()
3302 if (!reg->obj) in i915_find_fence_reg()
3343 i915_gem_object_get_fence(struct drm_i915_gem_object *obj) in i915_gem_object_get_fence() argument
3345 struct drm_device *dev = obj->base.dev; in i915_gem_object_get_fence()
3347 bool enable = obj->tiling_mode != I915_TILING_NONE; in i915_gem_object_get_fence()
3354 if (obj->fence_dirty) { in i915_gem_object_get_fence()
3355 ret = i915_gem_object_wait_fence(obj); in i915_gem_object_get_fence()
3361 if (obj->fence_reg != I915_FENCE_REG_NONE) { in i915_gem_object_get_fence()
3362 reg = &dev_priv->fence_regs[obj->fence_reg]; in i915_gem_object_get_fence()
3363 if (!obj->fence_dirty) { in i915_gem_object_get_fence()
3369 if (WARN_ON(!obj->map_and_fenceable)) in i915_gem_object_get_fence()
3376 if (reg->obj) { in i915_gem_object_get_fence()
3377 struct drm_i915_gem_object *old = reg->obj; in i915_gem_object_get_fence()
3388 i915_gem_object_update_fence(obj, reg, enable); in i915_gem_object_get_fence()
3430 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, in i915_gem_object_bind_to_vm() argument
3436 struct drm_device *dev = obj->base.dev; in i915_gem_object_bind_to_vm()
3450 obj->base.size, in i915_gem_object_bind_to_vm()
3451 obj->tiling_mode); in i915_gem_object_bind_to_vm()
3453 obj->base.size, in i915_gem_object_bind_to_vm()
3454 obj->tiling_mode, true); in i915_gem_object_bind_to_vm()
3457 obj->base.size, in i915_gem_object_bind_to_vm()
3458 obj->tiling_mode, false); in i915_gem_object_bind_to_vm()
3468 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; in i915_gem_object_bind_to_vm()
3473 if (obj->base.size > end) { in i915_gem_object_bind_to_vm()
3475 obj->base.size, in i915_gem_object_bind_to_vm()
3481 ret = i915_gem_object_get_pages(obj); in i915_gem_object_bind_to_vm()
3485 i915_gem_object_pin_pages(obj); in i915_gem_object_bind_to_vm()
3487 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) : in i915_gem_object_bind_to_vm()
3488 i915_gem_obj_lookup_or_create_vma(obj, vm); in i915_gem_object_bind_to_vm()
3496 obj->cache_level, in i915_gem_object_bind_to_vm()
3502 obj->cache_level, in i915_gem_object_bind_to_vm()
3510 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { in i915_gem_object_bind_to_vm()
3515 ret = i915_gem_gtt_prepare_object(obj); in i915_gem_object_bind_to_vm()
3531 ret = i915_vma_bind(vma, obj->cache_level, in i915_gem_object_bind_to_vm()
3536 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); in i915_gem_object_bind_to_vm()
3542 i915_gem_gtt_finish_object(obj); in i915_gem_object_bind_to_vm()
3549 i915_gem_object_unpin_pages(obj); in i915_gem_object_bind_to_vm()
3554 i915_gem_clflush_object(struct drm_i915_gem_object *obj, in i915_gem_clflush_object() argument
3561 if (obj->pages == NULL) in i915_gem_clflush_object()
3568 if (obj->stolen || obj->phys_handle) in i915_gem_clflush_object()
3579 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { in i915_gem_clflush_object()
3580 obj->cache_dirty = true; in i915_gem_clflush_object()
3584 trace_i915_gem_object_clflush(obj); in i915_gem_clflush_object()
3585 drm_clflush_sg(obj->pages); in i915_gem_clflush_object()
3586 obj->cache_dirty = false; in i915_gem_clflush_object()
3593 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) in i915_gem_object_flush_gtt_write_domain() argument
3597 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) in i915_gem_object_flush_gtt_write_domain()
3610 old_write_domain = obj->base.write_domain; in i915_gem_object_flush_gtt_write_domain()
3611 obj->base.write_domain = 0; in i915_gem_object_flush_gtt_write_domain()
3613 intel_fb_obj_flush(obj, false); in i915_gem_object_flush_gtt_write_domain()
3615 trace_i915_gem_object_change_domain(obj, in i915_gem_object_flush_gtt_write_domain()
3616 obj->base.read_domains, in i915_gem_object_flush_gtt_write_domain()
3622 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) in i915_gem_object_flush_cpu_write_domain() argument
3626 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) in i915_gem_object_flush_cpu_write_domain()
3629 if (i915_gem_clflush_object(obj, obj->pin_display)) in i915_gem_object_flush_cpu_write_domain()
3630 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_flush_cpu_write_domain()
3632 old_write_domain = obj->base.write_domain; in i915_gem_object_flush_cpu_write_domain()
3633 obj->base.write_domain = 0; in i915_gem_object_flush_cpu_write_domain()
3635 intel_fb_obj_flush(obj, false); in i915_gem_object_flush_cpu_write_domain()
3637 trace_i915_gem_object_change_domain(obj, in i915_gem_object_flush_cpu_write_domain()
3638 obj->base.read_domains, in i915_gem_object_flush_cpu_write_domain()
3649 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) in i915_gem_object_set_to_gtt_domain() argument
3655 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) in i915_gem_object_set_to_gtt_domain()
3658 ret = i915_gem_object_wait_rendering(obj, !write); in i915_gem_object_set_to_gtt_domain()
3662 i915_gem_object_retire(obj); in i915_gem_object_set_to_gtt_domain()
3672 ret = i915_gem_object_get_pages(obj); in i915_gem_object_set_to_gtt_domain()
3676 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_object_set_to_gtt_domain()
3682 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) in i915_gem_object_set_to_gtt_domain()
3685 old_write_domain = obj->base.write_domain; in i915_gem_object_set_to_gtt_domain()
3686 old_read_domains = obj->base.read_domains; in i915_gem_object_set_to_gtt_domain()
3691 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); in i915_gem_object_set_to_gtt_domain()
3692 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3694 obj->base.read_domains = I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3695 obj->base.write_domain = I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3696 obj->dirty = 1; in i915_gem_object_set_to_gtt_domain()
3700 intel_fb_obj_invalidate(obj, NULL, ORIGIN_GTT); in i915_gem_object_set_to_gtt_domain()
3702 trace_i915_gem_object_change_domain(obj, in i915_gem_object_set_to_gtt_domain()
3707 vma = i915_gem_obj_to_ggtt(obj); in i915_gem_object_set_to_gtt_domain()
3708 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active) in i915_gem_object_set_to_gtt_domain()
3710 &to_i915(obj->base.dev)->gtt.base.inactive_list); in i915_gem_object_set_to_gtt_domain()
3715 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, in i915_gem_object_set_cache_level() argument
3718 struct drm_device *dev = obj->base.dev; in i915_gem_object_set_cache_level()
3722 if (obj->cache_level == cache_level) in i915_gem_object_set_cache_level()
3725 if (i915_gem_obj_is_pinned(obj)) { in i915_gem_object_set_cache_level()
3730 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { in i915_gem_object_set_cache_level()
3738 if (i915_gem_obj_bound_any(obj)) { in i915_gem_object_set_cache_level()
3739 ret = i915_gem_object_finish_gpu(obj); in i915_gem_object_set_cache_level()
3743 i915_gem_object_finish_gtt(obj); in i915_gem_object_set_cache_level()
3750 ret = i915_gem_object_put_fence(obj); in i915_gem_object_set_cache_level()
3755 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_object_set_cache_level()
3764 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_object_set_cache_level()
3766 obj->cache_level = cache_level; in i915_gem_object_set_cache_level()
3768 if (obj->cache_dirty && in i915_gem_object_set_cache_level()
3769 obj->base.write_domain != I915_GEM_DOMAIN_CPU && in i915_gem_object_set_cache_level()
3770 cpu_write_needs_clflush(obj)) { in i915_gem_object_set_cache_level()
3771 if (i915_gem_clflush_object(obj, true)) in i915_gem_object_set_cache_level()
3772 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_set_cache_level()
3782 struct drm_i915_gem_object *obj; in i915_gem_get_caching_ioctl() local
3789 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_get_caching_ioctl()
3790 if (&obj->base == NULL) { in i915_gem_get_caching_ioctl()
3795 switch (obj->cache_level) { in i915_gem_get_caching_ioctl()
3810 drm_gem_object_unreference(&obj->base); in i915_gem_get_caching_ioctl()
3820 struct drm_i915_gem_object *obj; in i915_gem_set_caching_ioctl() local
3842 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_set_caching_ioctl()
3843 if (&obj->base == NULL) { in i915_gem_set_caching_ioctl()
3848 ret = i915_gem_object_set_cache_level(obj, level); in i915_gem_set_caching_ioctl()
3850 drm_gem_object_unreference(&obj->base); in i915_gem_set_caching_ioctl()
3856 static bool is_pin_display(struct drm_i915_gem_object *obj) in is_pin_display() argument
3860 vma = i915_gem_obj_to_ggtt(obj); in is_pin_display()
3880 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, in i915_gem_object_pin_to_display_plane() argument
3889 if (pipelined != i915_gem_request_get_ring(obj->last_read_req)) { in i915_gem_object_pin_to_display_plane()
3890 ret = i915_gem_object_sync(obj, pipelined); in i915_gem_object_pin_to_display_plane()
3898 was_pin_display = obj->pin_display; in i915_gem_object_pin_to_display_plane()
3899 obj->pin_display = true; in i915_gem_object_pin_to_display_plane()
3910 ret = i915_gem_object_set_cache_level(obj, in i915_gem_object_pin_to_display_plane()
3911 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); in i915_gem_object_pin_to_display_plane()
3919 ret = i915_gem_object_ggtt_pin(obj, view, alignment, in i915_gem_object_pin_to_display_plane()
3925 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_object_pin_to_display_plane()
3927 old_write_domain = obj->base.write_domain; in i915_gem_object_pin_to_display_plane()
3928 old_read_domains = obj->base.read_domains; in i915_gem_object_pin_to_display_plane()
3933 obj->base.write_domain = 0; in i915_gem_object_pin_to_display_plane()
3934 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; in i915_gem_object_pin_to_display_plane()
3936 trace_i915_gem_object_change_domain(obj, in i915_gem_object_pin_to_display_plane()
3943 WARN_ON(was_pin_display != is_pin_display(obj)); in i915_gem_object_pin_to_display_plane()
3944 obj->pin_display = was_pin_display; in i915_gem_object_pin_to_display_plane()
3949 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, in i915_gem_object_unpin_from_display_plane() argument
3952 i915_gem_object_ggtt_unpin_view(obj, view); in i915_gem_object_unpin_from_display_plane()
3954 obj->pin_display = is_pin_display(obj); in i915_gem_object_unpin_from_display_plane()
3958 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) in i915_gem_object_finish_gpu() argument
3962 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) in i915_gem_object_finish_gpu()
3965 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_object_finish_gpu()
3970 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; in i915_gem_object_finish_gpu()
3981 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) in i915_gem_object_set_to_cpu_domain() argument
3986 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) in i915_gem_object_set_to_cpu_domain()
3989 ret = i915_gem_object_wait_rendering(obj, !write); in i915_gem_object_set_to_cpu_domain()
3993 i915_gem_object_retire(obj); in i915_gem_object_set_to_cpu_domain()
3994 i915_gem_object_flush_gtt_write_domain(obj); in i915_gem_object_set_to_cpu_domain()
3996 old_write_domain = obj->base.write_domain; in i915_gem_object_set_to_cpu_domain()
3997 old_read_domains = obj->base.read_domains; in i915_gem_object_set_to_cpu_domain()
4000 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { in i915_gem_object_set_to_cpu_domain()
4001 i915_gem_clflush_object(obj, false); in i915_gem_object_set_to_cpu_domain()
4003 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4009 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); in i915_gem_object_set_to_cpu_domain()
4015 obj->base.read_domains = I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4016 obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4020 intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); in i915_gem_object_set_to_cpu_domain()
4022 trace_i915_gem_object_change_domain(obj, in i915_gem_object_set_to_cpu_domain()
4086 struct drm_i915_gem_object *obj = vma->obj; in i915_vma_misplaced() local
4092 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable) in i915_vma_misplaced()
4103 i915_gem_object_do_pin(struct drm_i915_gem_object *obj, in i915_gem_object_do_pin() argument
4109 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_do_pin()
4126 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) : in i915_gem_object_do_pin()
4127 i915_gem_obj_to_vma(obj, vm); in i915_gem_object_do_pin()
4138 offset = ggtt_view ? i915_gem_obj_ggtt_offset_view(obj, ggtt_view) : in i915_gem_object_do_pin()
4139 i915_gem_obj_offset(obj, vm); in i915_gem_object_do_pin()
4148 obj->map_and_fenceable); in i915_gem_object_do_pin()
4163 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment, in i915_gem_object_do_pin()
4170 ret = i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND); in i915_gem_object_do_pin()
4179 fence_size = i915_gem_get_gtt_size(obj->base.dev, in i915_gem_object_do_pin()
4180 obj->base.size, in i915_gem_object_do_pin()
4181 obj->tiling_mode); in i915_gem_object_do_pin()
4182 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, in i915_gem_object_do_pin()
4183 obj->base.size, in i915_gem_object_do_pin()
4184 obj->tiling_mode, in i915_gem_object_do_pin()
4193 obj->map_and_fenceable = mappable && fenceable; in i915_gem_object_do_pin()
4196 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); in i915_gem_object_do_pin()
4200 obj->pin_mappable |= true; in i915_gem_object_do_pin()
4206 i915_gem_object_pin(struct drm_i915_gem_object *obj, in i915_gem_object_pin() argument
4211 return i915_gem_object_do_pin(obj, vm, in i915_gem_object_pin()
4217 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, in i915_gem_object_ggtt_pin() argument
4225 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view, in i915_gem_object_ggtt_pin()
4230 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, in i915_gem_object_ggtt_unpin_view() argument
4233 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view); in i915_gem_object_ggtt_unpin_view()
4237 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view)); in i915_gem_object_ggtt_unpin_view()
4240 obj->pin_mappable = false; in i915_gem_object_ggtt_unpin_view()
4244 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) in i915_gem_object_pin_fence() argument
4246 if (obj->fence_reg != I915_FENCE_REG_NONE) { in i915_gem_object_pin_fence()
4247 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_pin_fence()
4248 struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); in i915_gem_object_pin_fence()
4251 dev_priv->fence_regs[obj->fence_reg].pin_count > in i915_gem_object_pin_fence()
4253 dev_priv->fence_regs[obj->fence_reg].pin_count++; in i915_gem_object_pin_fence()
4260 i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) in i915_gem_object_unpin_fence() argument
4262 if (obj->fence_reg != I915_FENCE_REG_NONE) { in i915_gem_object_unpin_fence()
4263 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_unpin_fence()
4264 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); in i915_gem_object_unpin_fence()
4265 dev_priv->fence_regs[obj->fence_reg].pin_count--; in i915_gem_object_unpin_fence()
4274 struct drm_i915_gem_object *obj; in i915_gem_busy_ioctl() local
4281 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_busy_ioctl()
4282 if (&obj->base == NULL) { in i915_gem_busy_ioctl()
4292 ret = i915_gem_object_flush_active(obj); in i915_gem_busy_ioctl()
4294 args->busy = obj->active; in i915_gem_busy_ioctl()
4295 if (obj->last_read_req) { in i915_gem_busy_ioctl()
4298 ring = i915_gem_request_get_ring(obj->last_read_req); in i915_gem_busy_ioctl()
4302 drm_gem_object_unreference(&obj->base); in i915_gem_busy_ioctl()
4321 struct drm_i915_gem_object *obj; in i915_gem_madvise_ioctl() local
4336 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); in i915_gem_madvise_ioctl()
4337 if (&obj->base == NULL) { in i915_gem_madvise_ioctl()
4342 if (i915_gem_obj_is_pinned(obj)) { in i915_gem_madvise_ioctl()
4347 if (obj->pages && in i915_gem_madvise_ioctl()
4348 obj->tiling_mode != I915_TILING_NONE && in i915_gem_madvise_ioctl()
4350 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_madvise_ioctl()
4351 i915_gem_object_unpin_pages(obj); in i915_gem_madvise_ioctl()
4353 i915_gem_object_pin_pages(obj); in i915_gem_madvise_ioctl()
4356 if (obj->madv != __I915_MADV_PURGED) in i915_gem_madvise_ioctl()
4357 obj->madv = args->madv; in i915_gem_madvise_ioctl()
4360 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) in i915_gem_madvise_ioctl()
4361 i915_gem_object_truncate(obj); in i915_gem_madvise_ioctl()
4363 args->retained = obj->madv != __I915_MADV_PURGED; in i915_gem_madvise_ioctl()
4366 drm_gem_object_unreference(&obj->base); in i915_gem_madvise_ioctl()
4372 void i915_gem_object_init(struct drm_i915_gem_object *obj, in i915_gem_object_init() argument
4375 INIT_LIST_HEAD(&obj->global_list); in i915_gem_object_init()
4376 INIT_LIST_HEAD(&obj->ring_list); in i915_gem_object_init()
4377 INIT_LIST_HEAD(&obj->obj_exec_link); in i915_gem_object_init()
4378 INIT_LIST_HEAD(&obj->vma_list); in i915_gem_object_init()
4379 INIT_LIST_HEAD(&obj->batch_pool_list); in i915_gem_object_init()
4381 obj->ops = ops; in i915_gem_object_init()
4383 obj->fence_reg = I915_FENCE_REG_NONE; in i915_gem_object_init()
4384 obj->madv = I915_MADV_WILLNEED; in i915_gem_object_init()
4386 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); in i915_gem_object_init()
4397 struct drm_i915_gem_object *obj; in i915_gem_alloc_object() local
4401 obj = i915_gem_object_alloc(dev); in i915_gem_alloc_object()
4402 if (obj == NULL) in i915_gem_alloc_object()
4405 if (drm_gem_object_init(dev, &obj->base, size) != 0) { in i915_gem_alloc_object()
4406 i915_gem_object_free(obj); in i915_gem_alloc_object()
4417 mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_alloc_object()
4420 i915_gem_object_init(obj, &i915_gem_object_ops); in i915_gem_alloc_object()
4422 obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_alloc_object()
4423 obj->base.read_domains = I915_GEM_DOMAIN_CPU; in i915_gem_alloc_object()
4438 obj->cache_level = I915_CACHE_LLC; in i915_gem_alloc_object()
4440 obj->cache_level = I915_CACHE_NONE; in i915_gem_alloc_object()
4442 trace_i915_gem_object_create(obj); in i915_gem_alloc_object()
4444 return obj; in i915_gem_alloc_object()
4447 static bool discard_backing_storage(struct drm_i915_gem_object *obj) in discard_backing_storage() argument
4455 if (obj->madv != I915_MADV_WILLNEED) in discard_backing_storage()
4458 if (obj->base.filp == NULL) in discard_backing_storage()
4468 return atomic_long_read(&obj->base.filp->f_count) == 1; in discard_backing_storage()
4473 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); in i915_gem_free_object() local
4474 struct drm_device *dev = obj->base.dev; in i915_gem_free_object()
4480 trace_i915_gem_object_destroy(obj); in i915_gem_free_object()
4482 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { in i915_gem_free_object()
4501 if (obj->stolen) in i915_gem_free_object()
4502 i915_gem_object_unpin_pages(obj); in i915_gem_free_object()
4504 WARN_ON(obj->frontbuffer_bits); in i915_gem_free_object()
4506 if (obj->pages && obj->madv == I915_MADV_WILLNEED && in i915_gem_free_object()
4508 obj->tiling_mode != I915_TILING_NONE) in i915_gem_free_object()
4509 i915_gem_object_unpin_pages(obj); in i915_gem_free_object()
4511 if (WARN_ON(obj->pages_pin_count)) in i915_gem_free_object()
4512 obj->pages_pin_count = 0; in i915_gem_free_object()
4513 if (discard_backing_storage(obj)) in i915_gem_free_object()
4514 obj->madv = I915_MADV_DONTNEED; in i915_gem_free_object()
4515 i915_gem_object_put_pages(obj); in i915_gem_free_object()
4516 i915_gem_object_free_mmap_offset(obj); in i915_gem_free_object()
4518 BUG_ON(obj->pages); in i915_gem_free_object()
4520 if (obj->base.import_attach) in i915_gem_free_object()
4521 drm_prime_gem_destroy(&obj->base, NULL); in i915_gem_free_object()
4523 if (obj->ops->release) in i915_gem_free_object()
4524 obj->ops->release(obj); in i915_gem_free_object()
4526 drm_gem_object_release(&obj->base); in i915_gem_free_object()
4527 i915_gem_info_remove_obj(dev_priv, obj->base.size); in i915_gem_free_object()
4529 kfree(obj->bit_17); in i915_gem_free_object()
4530 i915_gem_object_free(obj); in i915_gem_free_object()
4535 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, in i915_gem_obj_to_vma() argument
4539 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_obj_to_vma()
4549 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, in i915_gem_obj_to_ggtt_view() argument
4552 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); in i915_gem_obj_to_ggtt_view()
4558 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_obj_to_ggtt_view()
5193 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) in i915_gem_obj_is_pinned() argument
5196 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_obj_is_pinned()