Lines Matching refs:obj
43 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
44 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
46 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
48 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
56 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) in cpu_write_needs_clflush() argument
58 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) in cpu_write_needs_clflush()
61 return obj->pin_display; in cpu_write_needs_clflush()
156 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages_phys() argument
158 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_get_pages_phys()
159 char *vaddr = obj->phys_handle->vaddr; in i915_gem_object_get_pages_phys()
164 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) in i915_gem_object_get_pages_phys()
167 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { in i915_gem_object_get_pages_phys()
184 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_get_pages_phys()
197 sg->length = obj->base.size; in i915_gem_object_get_pages_phys()
199 sg_dma_address(sg) = obj->phys_handle->busaddr; in i915_gem_object_get_pages_phys()
200 sg_dma_len(sg) = obj->base.size; in i915_gem_object_get_pages_phys()
202 obj->pages = st; in i915_gem_object_get_pages_phys()
207 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages_phys() argument
211 BUG_ON(obj->madv == __I915_MADV_PURGED); in i915_gem_object_put_pages_phys()
213 ret = i915_gem_object_set_to_cpu_domain(obj, true); in i915_gem_object_put_pages_phys()
219 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_put_pages_phys()
222 if (obj->madv == I915_MADV_DONTNEED) in i915_gem_object_put_pages_phys()
223 obj->dirty = 0; in i915_gem_object_put_pages_phys()
225 if (obj->dirty) { in i915_gem_object_put_pages_phys()
226 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_put_pages_phys()
227 char *vaddr = obj->phys_handle->vaddr; in i915_gem_object_put_pages_phys()
230 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { in i915_gem_object_put_pages_phys()
244 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_object_put_pages_phys()
249 obj->dirty = 0; in i915_gem_object_put_pages_phys()
252 sg_free_table(obj->pages); in i915_gem_object_put_pages_phys()
253 kfree(obj->pages); in i915_gem_object_put_pages_phys()
257 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) in i915_gem_object_release_phys() argument
259 drm_pci_free(obj->base.dev, obj->phys_handle); in i915_gem_object_release_phys()
269 drop_pages(struct drm_i915_gem_object *obj) in drop_pages() argument
274 drm_gem_object_reference(&obj->base); in drop_pages()
275 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) in drop_pages()
279 ret = i915_gem_object_put_pages(obj); in drop_pages()
280 drm_gem_object_unreference(&obj->base); in drop_pages()
286 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, in i915_gem_object_attach_phys() argument
292 if (obj->phys_handle) { in i915_gem_object_attach_phys()
293 if ((unsigned long)obj->phys_handle->vaddr & (align -1)) in i915_gem_object_attach_phys()
299 if (obj->madv != I915_MADV_WILLNEED) in i915_gem_object_attach_phys()
302 if (obj->base.filp == NULL) in i915_gem_object_attach_phys()
305 ret = drop_pages(obj); in i915_gem_object_attach_phys()
310 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); in i915_gem_object_attach_phys()
314 obj->phys_handle = phys; in i915_gem_object_attach_phys()
315 obj->ops = &i915_gem_phys_ops; in i915_gem_object_attach_phys()
317 return i915_gem_object_get_pages(obj); in i915_gem_object_attach_phys()
321 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, in i915_gem_phys_pwrite() argument
325 struct drm_device *dev = obj->base.dev; in i915_gem_phys_pwrite()
326 void *vaddr = obj->phys_handle->vaddr + args->offset; in i915_gem_phys_pwrite()
333 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_phys_pwrite()
337 intel_fb_obj_invalidate(obj, ORIGIN_CPU); in i915_gem_phys_pwrite()
358 intel_fb_obj_flush(obj, false, ORIGIN_CPU); in i915_gem_phys_pwrite()
368 void i915_gem_object_free(struct drm_i915_gem_object *obj) in i915_gem_object_free() argument
370 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_free()
371 kmem_cache_free(dev_priv->objects, obj); in i915_gem_object_free()
380 struct drm_i915_gem_object *obj; in i915_gem_create() local
389 obj = i915_gem_alloc_object(dev, size); in i915_gem_create()
390 if (obj == NULL) in i915_gem_create()
393 ret = drm_gem_handle_create(file, &obj->base, &handle); in i915_gem_create()
395 drm_gem_object_unreference_unlocked(&obj->base); in i915_gem_create()
485 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, in i915_gem_obj_prepare_shmem_read() argument
492 if (!obj->base.filp) in i915_gem_obj_prepare_shmem_read()
495 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { in i915_gem_obj_prepare_shmem_read()
500 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev, in i915_gem_obj_prepare_shmem_read()
501 obj->cache_level); in i915_gem_obj_prepare_shmem_read()
502 ret = i915_gem_object_wait_rendering(obj, true); in i915_gem_obj_prepare_shmem_read()
507 ret = i915_gem_object_get_pages(obj); in i915_gem_obj_prepare_shmem_read()
511 i915_gem_object_pin_pages(obj); in i915_gem_obj_prepare_shmem_read()
595 struct drm_i915_gem_object *obj, in i915_gem_shmem_pread() argument
611 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); in i915_gem_shmem_pread()
613 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); in i915_gem_shmem_pread()
619 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, in i915_gem_shmem_pread()
673 i915_gem_object_unpin_pages(obj); in i915_gem_shmem_pread()
688 struct drm_i915_gem_object *obj; in i915_gem_pread_ioctl() local
703 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_pread_ioctl()
704 if (&obj->base == NULL) { in i915_gem_pread_ioctl()
710 if (args->offset > obj->base.size || in i915_gem_pread_ioctl()
711 args->size > obj->base.size - args->offset) { in i915_gem_pread_ioctl()
719 if (!obj->base.filp) { in i915_gem_pread_ioctl()
724 trace_i915_gem_object_pread(obj, args->offset, args->size); in i915_gem_pread_ioctl()
726 ret = i915_gem_shmem_pread(dev, obj, args, file); in i915_gem_pread_ioctl()
729 drm_gem_object_unreference(&obj->base); in i915_gem_pread_ioctl()
764 struct drm_i915_gem_object *obj, in i915_gem_gtt_pwrite_fast() argument
774 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); in i915_gem_gtt_pwrite_fast()
778 ret = i915_gem_object_set_to_gtt_domain(obj, true); in i915_gem_gtt_pwrite_fast()
782 ret = i915_gem_object_put_fence(obj); in i915_gem_gtt_pwrite_fast()
789 offset = i915_gem_obj_ggtt_offset(obj) + args->offset; in i915_gem_gtt_pwrite_fast()
791 intel_fb_obj_invalidate(obj, ORIGIN_GTT); in i915_gem_gtt_pwrite_fast()
822 intel_fb_obj_flush(obj, false, ORIGIN_GTT); in i915_gem_gtt_pwrite_fast()
824 i915_gem_object_ggtt_unpin(obj); in i915_gem_gtt_pwrite_fast()
896 struct drm_i915_gem_object *obj, in i915_gem_shmem_pwrite() argument
913 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); in i915_gem_shmem_pwrite()
915 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { in i915_gem_shmem_pwrite()
920 needs_clflush_after = cpu_write_needs_clflush(obj); in i915_gem_shmem_pwrite()
921 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_shmem_pwrite()
927 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) in i915_gem_shmem_pwrite()
929 !cpu_cache_is_coherent(dev, obj->cache_level); in i915_gem_shmem_pwrite()
931 ret = i915_gem_object_get_pages(obj); in i915_gem_shmem_pwrite()
935 intel_fb_obj_invalidate(obj, ORIGIN_CPU); in i915_gem_shmem_pwrite()
937 i915_gem_object_pin_pages(obj); in i915_gem_shmem_pwrite()
940 obj->dirty = 1; in i915_gem_shmem_pwrite()
942 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, in i915_gem_shmem_pwrite()
997 i915_gem_object_unpin_pages(obj); in i915_gem_shmem_pwrite()
1006 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { in i915_gem_shmem_pwrite()
1007 if (i915_gem_clflush_object(obj, obj->pin_display)) in i915_gem_shmem_pwrite()
1015 obj->cache_dirty = true; in i915_gem_shmem_pwrite()
1017 intel_fb_obj_flush(obj, false, ORIGIN_CPU); in i915_gem_shmem_pwrite()
1032 struct drm_i915_gem_object *obj; in i915_gem_pwrite_ioctl() local
1056 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_pwrite_ioctl()
1057 if (&obj->base == NULL) { in i915_gem_pwrite_ioctl()
1063 if (args->offset > obj->base.size || in i915_gem_pwrite_ioctl()
1064 args->size > obj->base.size - args->offset) { in i915_gem_pwrite_ioctl()
1072 if (!obj->base.filp) { in i915_gem_pwrite_ioctl()
1077 trace_i915_gem_object_pwrite(obj, args->offset, args->size); in i915_gem_pwrite_ioctl()
1086 if (obj->tiling_mode == I915_TILING_NONE && in i915_gem_pwrite_ioctl()
1087 obj->base.write_domain != I915_GEM_DOMAIN_CPU && in i915_gem_pwrite_ioctl()
1088 cpu_write_needs_clflush(obj)) { in i915_gem_pwrite_ioctl()
1089 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); in i915_gem_pwrite_ioctl()
1096 if (obj->phys_handle) in i915_gem_pwrite_ioctl()
1097 ret = i915_gem_phys_pwrite(obj, args, file); in i915_gem_pwrite_ioctl()
1099 ret = i915_gem_shmem_pwrite(dev, obj, args, file); in i915_gem_pwrite_ioctl()
1103 drm_gem_object_unreference(&obj->base); in i915_gem_pwrite_ioctl()
1492 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, in i915_gem_object_wait_rendering() argument
1497 if (!obj->active) in i915_gem_object_wait_rendering()
1501 if (obj->last_write_req != NULL) { in i915_gem_object_wait_rendering()
1502 ret = i915_wait_request(obj->last_write_req); in i915_gem_object_wait_rendering()
1506 i = obj->last_write_req->ring->id; in i915_gem_object_wait_rendering()
1507 if (obj->last_read_req[i] == obj->last_write_req) in i915_gem_object_wait_rendering()
1508 i915_gem_object_retire__read(obj, i); in i915_gem_object_wait_rendering()
1510 i915_gem_object_retire__write(obj); in i915_gem_object_wait_rendering()
1514 if (obj->last_read_req[i] == NULL) in i915_gem_object_wait_rendering()
1517 ret = i915_wait_request(obj->last_read_req[i]); in i915_gem_object_wait_rendering()
1521 i915_gem_object_retire__read(obj, i); in i915_gem_object_wait_rendering()
1523 RQ_BUG_ON(obj->active); in i915_gem_object_wait_rendering()
1530 i915_gem_object_retire_request(struct drm_i915_gem_object *obj, in i915_gem_object_retire_request() argument
1535 if (obj->last_read_req[ring] == req) in i915_gem_object_retire_request()
1536 i915_gem_object_retire__read(obj, ring); in i915_gem_object_retire_request()
1537 else if (obj->last_write_req == req) in i915_gem_object_retire_request()
1538 i915_gem_object_retire__write(obj); in i915_gem_object_retire_request()
1547 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, in i915_gem_object_wait_rendering__nonblocking() argument
1551 struct drm_device *dev = obj->base.dev; in i915_gem_object_wait_rendering__nonblocking()
1560 if (!obj->active) in i915_gem_object_wait_rendering__nonblocking()
1572 req = obj->last_write_req; in i915_gem_object_wait_rendering__nonblocking()
1581 req = obj->last_read_req[i]; in i915_gem_object_wait_rendering__nonblocking()
1597 i915_gem_object_retire_request(obj, requests[i]); in i915_gem_object_wait_rendering__nonblocking()
1619 struct drm_i915_gem_object *obj; in i915_gem_set_domain_ioctl() local
1641 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_set_domain_ioctl()
1642 if (&obj->base == NULL) { in i915_gem_set_domain_ioctl()
1651 ret = i915_gem_object_wait_rendering__nonblocking(obj, in i915_gem_set_domain_ioctl()
1658 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); in i915_gem_set_domain_ioctl()
1660 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); in i915_gem_set_domain_ioctl()
1663 intel_fb_obj_invalidate(obj, in i915_gem_set_domain_ioctl()
1668 drm_gem_object_unreference(&obj->base); in i915_gem_set_domain_ioctl()
1682 struct drm_i915_gem_object *obj; in i915_gem_sw_finish_ioctl() local
1689 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_sw_finish_ioctl()
1690 if (&obj->base == NULL) { in i915_gem_sw_finish_ioctl()
1696 if (obj->pin_display) in i915_gem_sw_finish_ioctl()
1697 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_sw_finish_ioctl()
1699 drm_gem_object_unreference(&obj->base); in i915_gem_sw_finish_ioctl()
1727 struct drm_gem_object *obj; in i915_gem_mmap_ioctl() local
1736 obj = drm_gem_object_lookup(dev, file, args->handle); in i915_gem_mmap_ioctl()
1737 if (obj == NULL) in i915_gem_mmap_ioctl()
1743 if (!obj->filp) { in i915_gem_mmap_ioctl()
1744 drm_gem_object_unreference_unlocked(obj); in i915_gem_mmap_ioctl()
1748 addr = vm_mmap(obj->filp, 0, args->size, in i915_gem_mmap_ioctl()
1764 drm_gem_object_unreference_unlocked(obj); in i915_gem_mmap_ioctl()
1791 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data); in i915_gem_fault() local
1792 struct drm_device *dev = obj->base.dev; in i915_gem_fault()
1810 trace_i915_gem_object_fault(obj, page_offset, true, write); in i915_gem_fault()
1817 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); in i915_gem_fault()
1822 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { in i915_gem_fault()
1828 if (obj->base.size >= dev_priv->gtt.mappable_end && in i915_gem_fault()
1829 obj->tiling_mode == I915_TILING_NONE) { in i915_gem_fault()
1843 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE); in i915_gem_fault()
1847 ret = i915_gem_object_set_to_gtt_domain(obj, write); in i915_gem_fault()
1851 ret = i915_gem_object_get_fence(obj); in i915_gem_fault()
1857 i915_gem_obj_ggtt_offset_view(obj, &view); in i915_gem_fault()
1876 obj->fault_mappable = true; in i915_gem_fault()
1878 if (!obj->fault_mappable) { in i915_gem_fault()
1881 obj->base.size); in i915_gem_fault()
1892 obj->fault_mappable = true; in i915_gem_fault()
1899 i915_gem_object_ggtt_unpin_view(obj, &view); in i915_gem_fault()
1963 i915_gem_release_mmap(struct drm_i915_gem_object *obj) in i915_gem_release_mmap() argument
1965 if (!obj->fault_mappable) in i915_gem_release_mmap()
1968 drm_vma_node_unmap(&obj->base.vma_node, in i915_gem_release_mmap()
1969 obj->base.dev->anon_inode->i_mapping); in i915_gem_release_mmap()
1970 obj->fault_mappable = false; in i915_gem_release_mmap()
1976 struct drm_i915_gem_object *obj; in i915_gem_release_all_mmaps() local
1978 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) in i915_gem_release_all_mmaps()
1979 i915_gem_release_mmap(obj); in i915_gem_release_all_mmaps()
2029 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) in i915_gem_object_create_mmap_offset() argument
2031 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_create_mmap_offset()
2034 if (drm_vma_node_has_offset(&obj->base.vma_node)) in i915_gem_object_create_mmap_offset()
2039 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
2051 obj->base.size >> PAGE_SHIFT, in i915_gem_object_create_mmap_offset()
2055 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
2060 ret = drm_gem_create_mmap_offset(&obj->base); in i915_gem_object_create_mmap_offset()
2067 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) in i915_gem_object_free_mmap_offset() argument
2069 drm_gem_free_mmap_offset(&obj->base); in i915_gem_object_free_mmap_offset()
2078 struct drm_i915_gem_object *obj; in i915_gem_mmap_gtt() local
2085 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); in i915_gem_mmap_gtt()
2086 if (&obj->base == NULL) { in i915_gem_mmap_gtt()
2091 if (obj->madv != I915_MADV_WILLNEED) { in i915_gem_mmap_gtt()
2097 ret = i915_gem_object_create_mmap_offset(obj); in i915_gem_mmap_gtt()
2101 *offset = drm_vma_node_offset_addr(&obj->base.vma_node); in i915_gem_mmap_gtt()
2104 drm_gem_object_unreference(&obj->base); in i915_gem_mmap_gtt()
2136 i915_gem_object_truncate(struct drm_i915_gem_object *obj) in i915_gem_object_truncate() argument
2138 i915_gem_object_free_mmap_offset(obj); in i915_gem_object_truncate()
2140 if (obj->base.filp == NULL) in i915_gem_object_truncate()
2148 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); in i915_gem_object_truncate()
2149 obj->madv = __I915_MADV_PURGED; in i915_gem_object_truncate()
2154 i915_gem_object_invalidate(struct drm_i915_gem_object *obj) in i915_gem_object_invalidate() argument
2158 switch (obj->madv) { in i915_gem_object_invalidate()
2160 i915_gem_object_truncate(obj); in i915_gem_object_invalidate()
2165 if (obj->base.filp == NULL) in i915_gem_object_invalidate()
2168 mapping = file_inode(obj->base.filp)->i_mapping, in i915_gem_object_invalidate()
2173 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages_gtt() argument
2178 BUG_ON(obj->madv == __I915_MADV_PURGED); in i915_gem_object_put_pages_gtt()
2180 ret = i915_gem_object_set_to_cpu_domain(obj, true); in i915_gem_object_put_pages_gtt()
2186 i915_gem_clflush_object(obj, true); in i915_gem_object_put_pages_gtt()
2187 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_put_pages_gtt()
2190 i915_gem_gtt_finish_object(obj); in i915_gem_object_put_pages_gtt()
2192 if (i915_gem_object_needs_bit17_swizzle(obj)) in i915_gem_object_put_pages_gtt()
2193 i915_gem_object_save_bit_17_swizzle(obj); in i915_gem_object_put_pages_gtt()
2195 if (obj->madv == I915_MADV_DONTNEED) in i915_gem_object_put_pages_gtt()
2196 obj->dirty = 0; in i915_gem_object_put_pages_gtt()
2198 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { in i915_gem_object_put_pages_gtt()
2201 if (obj->dirty) in i915_gem_object_put_pages_gtt()
2204 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_object_put_pages_gtt()
2209 obj->dirty = 0; in i915_gem_object_put_pages_gtt()
2211 sg_free_table(obj->pages); in i915_gem_object_put_pages_gtt()
2212 kfree(obj->pages); in i915_gem_object_put_pages_gtt()
2216 i915_gem_object_put_pages(struct drm_i915_gem_object *obj) in i915_gem_object_put_pages() argument
2218 const struct drm_i915_gem_object_ops *ops = obj->ops; in i915_gem_object_put_pages()
2220 if (obj->pages == NULL) in i915_gem_object_put_pages()
2223 if (obj->pages_pin_count) in i915_gem_object_put_pages()
2226 BUG_ON(i915_gem_obj_bound_any(obj)); in i915_gem_object_put_pages()
2231 list_del(&obj->global_list); in i915_gem_object_put_pages()
2233 ops->put_pages(obj); in i915_gem_object_put_pages()
2234 obj->pages = NULL; in i915_gem_object_put_pages()
2236 i915_gem_object_invalidate(obj); in i915_gem_object_put_pages()
2242 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages_gtt() argument
2244 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_get_pages_gtt()
2259 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); in i915_gem_object_get_pages_gtt()
2260 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); in i915_gem_object_get_pages_gtt()
2266 page_count = obj->base.size / PAGE_SIZE; in i915_gem_object_get_pages_gtt()
2277 mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_object_get_pages_gtt()
2329 obj->pages = st; in i915_gem_object_get_pages_gtt()
2331 ret = i915_gem_gtt_prepare_object(obj); in i915_gem_object_get_pages_gtt()
2335 if (i915_gem_object_needs_bit17_swizzle(obj)) in i915_gem_object_get_pages_gtt()
2336 i915_gem_object_do_bit_17_swizzle(obj); in i915_gem_object_get_pages_gtt()
2338 if (obj->tiling_mode != I915_TILING_NONE && in i915_gem_object_get_pages_gtt()
2340 i915_gem_object_pin_pages(obj); in i915_gem_object_get_pages_gtt()
2373 i915_gem_object_get_pages(struct drm_i915_gem_object *obj) in i915_gem_object_get_pages() argument
2375 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_get_pages()
2376 const struct drm_i915_gem_object_ops *ops = obj->ops; in i915_gem_object_get_pages()
2379 if (obj->pages) in i915_gem_object_get_pages()
2382 if (obj->madv != I915_MADV_WILLNEED) { in i915_gem_object_get_pages()
2387 BUG_ON(obj->pages_pin_count); in i915_gem_object_get_pages()
2389 ret = ops->get_pages(obj); in i915_gem_object_get_pages()
2393 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); in i915_gem_object_get_pages()
2395 obj->get_page.sg = obj->pages->sgl; in i915_gem_object_get_pages()
2396 obj->get_page.last = 0; in i915_gem_object_get_pages()
2404 struct drm_i915_gem_object *obj = vma->obj; in i915_vma_move_to_active() local
2410 if (obj->active == 0) in i915_vma_move_to_active()
2411 drm_gem_object_reference(&obj->base); in i915_vma_move_to_active()
2412 obj->active |= intel_ring_flag(ring); in i915_vma_move_to_active()
2414 list_move_tail(&obj->ring_list[ring->id], &ring->active_list); in i915_vma_move_to_active()
2415 i915_gem_request_assign(&obj->last_read_req[ring->id], req); in i915_vma_move_to_active()
2421 i915_gem_object_retire__write(struct drm_i915_gem_object *obj) in i915_gem_object_retire__write() argument
2423 RQ_BUG_ON(obj->last_write_req == NULL); in i915_gem_object_retire__write()
2424 RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring))); in i915_gem_object_retire__write()
2426 i915_gem_request_assign(&obj->last_write_req, NULL); in i915_gem_object_retire__write()
2427 intel_fb_obj_flush(obj, true, ORIGIN_CS); in i915_gem_object_retire__write()
2431 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) in i915_gem_object_retire__read() argument
2435 RQ_BUG_ON(obj->last_read_req[ring] == NULL); in i915_gem_object_retire__read()
2436 RQ_BUG_ON(!(obj->active & (1 << ring))); in i915_gem_object_retire__read()
2438 list_del_init(&obj->ring_list[ring]); in i915_gem_object_retire__read()
2439 i915_gem_request_assign(&obj->last_read_req[ring], NULL); in i915_gem_object_retire__read()
2441 if (obj->last_write_req && obj->last_write_req->ring->id == ring) in i915_gem_object_retire__read()
2442 i915_gem_object_retire__write(obj); in i915_gem_object_retire__read()
2444 obj->active &= ~(1 << ring); in i915_gem_object_retire__read()
2445 if (obj->active) in i915_gem_object_retire__read()
2452 list_move_tail(&obj->global_list, in i915_gem_object_retire__read()
2453 &to_i915(obj->base.dev)->mm.bound_list); in i915_gem_object_retire__read()
2455 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_object_retire__read()
2460 i915_gem_request_assign(&obj->last_fenced_req, NULL); in i915_gem_object_retire__read()
2461 drm_gem_object_unreference(&obj->base); in i915_gem_object_retire__read()
2540 struct drm_i915_gem_object *obj, in __i915_add_request() argument
2605 request->batch_obj = obj; in __i915_add_request()
2801 struct drm_i915_gem_object *obj; in i915_gem_reset_ring_cleanup() local
2803 obj = list_first_entry(&ring->active_list, in i915_gem_reset_ring_cleanup()
2807 i915_gem_object_retire__read(obj, ring->id); in i915_gem_reset_ring_cleanup()
2902 struct drm_i915_gem_object *obj; in i915_gem_retire_requests_ring() local
2904 obj = list_first_entry(&ring->active_list, in i915_gem_retire_requests_ring()
2908 if (!list_empty(&obj->last_read_req[ring->id]->list)) in i915_gem_retire_requests_ring()
2911 i915_gem_object_retire__read(obj, ring->id); in i915_gem_retire_requests_ring()
3004 i915_gem_object_flush_active(struct drm_i915_gem_object *obj) in i915_gem_object_flush_active() argument
3008 if (!obj->active) in i915_gem_object_flush_active()
3014 req = obj->last_read_req[i]; in i915_gem_object_flush_active()
3024 i915_gem_object_retire__read(obj, i); in i915_gem_object_flush_active()
3058 struct drm_i915_gem_object *obj; in i915_gem_wait_ioctl() local
3071 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); in i915_gem_wait_ioctl()
3072 if (&obj->base == NULL) { in i915_gem_wait_ioctl()
3078 ret = i915_gem_object_flush_active(obj); in i915_gem_wait_ioctl()
3082 if (!obj->active) in i915_gem_wait_ioctl()
3093 drm_gem_object_unreference(&obj->base); in i915_gem_wait_ioctl()
3097 if (obj->last_read_req[i] == NULL) in i915_gem_wait_ioctl()
3100 req[n++] = i915_gem_request_reference(obj->last_read_req[i]); in i915_gem_wait_ioctl()
3115 drm_gem_object_unreference(&obj->base); in i915_gem_wait_ioctl()
3121 __i915_gem_object_sync(struct drm_i915_gem_object *obj, in __i915_gem_object_sync() argument
3136 if (!i915_semaphore_is_enabled(obj->base.dev)) { in __i915_gem_object_sync()
3137 struct drm_i915_private *i915 = to_i915(obj->base.dev); in __i915_gem_object_sync()
3146 i915_gem_object_retire_request(obj, from_req); in __i915_gem_object_sync()
3172 i915_gem_request_get_seqno(obj->last_read_req[from->id]); in __i915_gem_object_sync()
3214 i915_gem_object_sync(struct drm_i915_gem_object *obj, in i915_gem_object_sync() argument
3218 const bool readonly = obj->base.pending_write_domain == 0; in i915_gem_object_sync()
3222 if (!obj->active) in i915_gem_object_sync()
3226 return i915_gem_object_wait_rendering(obj, readonly); in i915_gem_object_sync()
3230 if (obj->last_write_req) in i915_gem_object_sync()
3231 req[n++] = obj->last_write_req; in i915_gem_object_sync()
3234 if (obj->last_read_req[i]) in i915_gem_object_sync()
3235 req[n++] = obj->last_read_req[i]; in i915_gem_object_sync()
3238 ret = __i915_gem_object_sync(obj, to, req[i], to_req); in i915_gem_object_sync()
3246 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) in i915_gem_object_finish_gtt() argument
3251 i915_gem_release_mmap(obj); in i915_gem_object_finish_gtt()
3253 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) in i915_gem_object_finish_gtt()
3259 old_read_domains = obj->base.read_domains; in i915_gem_object_finish_gtt()
3260 old_write_domain = obj->base.write_domain; in i915_gem_object_finish_gtt()
3262 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; in i915_gem_object_finish_gtt()
3263 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; in i915_gem_object_finish_gtt()
3265 trace_i915_gem_object_change_domain(obj, in i915_gem_object_finish_gtt()
3272 struct drm_i915_gem_object *obj = vma->obj; in __i915_vma_unbind() local
3273 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in __i915_vma_unbind()
3287 BUG_ON(obj->pages == NULL); in __i915_vma_unbind()
3290 ret = i915_gem_object_wait_rendering(obj, false); in __i915_vma_unbind()
3297 i915_gem_object_finish_gtt(obj); in __i915_vma_unbind()
3300 ret = i915_gem_object_put_fence(obj); in __i915_vma_unbind()
3313 obj->map_and_fenceable = false; in __i915_vma_unbind()
3326 if (list_empty(&obj->vma_list)) in __i915_vma_unbind()
3327 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); in __i915_vma_unbind()
3333 i915_gem_object_unpin_pages(obj); in __i915_vma_unbind()
3419 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, in i915_gem_object_bind_to_vm() argument
3425 struct drm_device *dev = obj->base.dev; in i915_gem_object_bind_to_vm()
3440 view_size = i915_ggtt_view_size(obj, ggtt_view); in i915_gem_object_bind_to_vm()
3444 obj->tiling_mode); in i915_gem_object_bind_to_vm()
3447 obj->tiling_mode, in i915_gem_object_bind_to_vm()
3451 obj->tiling_mode, in i915_gem_object_bind_to_vm()
3456 obj->base.size, in i915_gem_object_bind_to_vm()
3457 obj->tiling_mode); in i915_gem_object_bind_to_vm()
3459 obj->base.size, in i915_gem_object_bind_to_vm()
3460 obj->tiling_mode, in i915_gem_object_bind_to_vm()
3464 obj->base.size, in i915_gem_object_bind_to_vm()
3465 obj->tiling_mode, in i915_gem_object_bind_to_vm()
3467 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; in i915_gem_object_bind_to_vm()
3500 ret = i915_gem_object_get_pages(obj); in i915_gem_object_bind_to_vm()
3504 i915_gem_object_pin_pages(obj); in i915_gem_object_bind_to_vm()
3506 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) : in i915_gem_object_bind_to_vm()
3507 i915_gem_obj_lookup_or_create_vma(obj, vm); in i915_gem_object_bind_to_vm()
3523 obj->cache_level, in i915_gem_object_bind_to_vm()
3529 obj->cache_level, in i915_gem_object_bind_to_vm()
3537 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { in i915_gem_object_bind_to_vm()
3543 ret = i915_vma_bind(vma, obj->cache_level, flags); in i915_gem_object_bind_to_vm()
3547 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); in i915_gem_object_bind_to_vm()
3558 i915_gem_object_unpin_pages(obj); in i915_gem_object_bind_to_vm()
3563 i915_gem_clflush_object(struct drm_i915_gem_object *obj, in i915_gem_clflush_object() argument
3570 if (obj->pages == NULL) in i915_gem_clflush_object()
3577 if (obj->stolen || obj->phys_handle) in i915_gem_clflush_object()
3588 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { in i915_gem_clflush_object()
3589 obj->cache_dirty = true; in i915_gem_clflush_object()
3593 trace_i915_gem_object_clflush(obj); in i915_gem_clflush_object()
3594 drm_clflush_sg(obj->pages); in i915_gem_clflush_object()
3595 obj->cache_dirty = false; in i915_gem_clflush_object()
3602 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) in i915_gem_object_flush_gtt_write_domain() argument
3606 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) in i915_gem_object_flush_gtt_write_domain()
3619 old_write_domain = obj->base.write_domain; in i915_gem_object_flush_gtt_write_domain()
3620 obj->base.write_domain = 0; in i915_gem_object_flush_gtt_write_domain()
3622 intel_fb_obj_flush(obj, false, ORIGIN_GTT); in i915_gem_object_flush_gtt_write_domain()
3624 trace_i915_gem_object_change_domain(obj, in i915_gem_object_flush_gtt_write_domain()
3625 obj->base.read_domains, in i915_gem_object_flush_gtt_write_domain()
3631 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) in i915_gem_object_flush_cpu_write_domain() argument
3635 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) in i915_gem_object_flush_cpu_write_domain()
3638 if (i915_gem_clflush_object(obj, obj->pin_display)) in i915_gem_object_flush_cpu_write_domain()
3639 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_flush_cpu_write_domain()
3641 old_write_domain = obj->base.write_domain; in i915_gem_object_flush_cpu_write_domain()
3642 obj->base.write_domain = 0; in i915_gem_object_flush_cpu_write_domain()
3644 intel_fb_obj_flush(obj, false, ORIGIN_CPU); in i915_gem_object_flush_cpu_write_domain()
3646 trace_i915_gem_object_change_domain(obj, in i915_gem_object_flush_cpu_write_domain()
3647 obj->base.read_domains, in i915_gem_object_flush_cpu_write_domain()
3658 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) in i915_gem_object_set_to_gtt_domain() argument
3664 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) in i915_gem_object_set_to_gtt_domain()
3667 ret = i915_gem_object_wait_rendering(obj, !write); in i915_gem_object_set_to_gtt_domain()
3679 ret = i915_gem_object_get_pages(obj); in i915_gem_object_set_to_gtt_domain()
3683 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_object_set_to_gtt_domain()
3689 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) in i915_gem_object_set_to_gtt_domain()
3692 old_write_domain = obj->base.write_domain; in i915_gem_object_set_to_gtt_domain()
3693 old_read_domains = obj->base.read_domains; in i915_gem_object_set_to_gtt_domain()
3698 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); in i915_gem_object_set_to_gtt_domain()
3699 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3701 obj->base.read_domains = I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3702 obj->base.write_domain = I915_GEM_DOMAIN_GTT; in i915_gem_object_set_to_gtt_domain()
3703 obj->dirty = 1; in i915_gem_object_set_to_gtt_domain()
3706 trace_i915_gem_object_change_domain(obj, in i915_gem_object_set_to_gtt_domain()
3711 vma = i915_gem_obj_to_ggtt(obj); in i915_gem_object_set_to_gtt_domain()
3712 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active) in i915_gem_object_set_to_gtt_domain()
3714 &to_i915(obj->base.dev)->gtt.base.inactive_list); in i915_gem_object_set_to_gtt_domain()
3732 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, in i915_gem_object_set_cache_level() argument
3735 struct drm_device *dev = obj->base.dev; in i915_gem_object_set_cache_level()
3740 if (obj->cache_level == cache_level) in i915_gem_object_set_cache_level()
3748 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { in i915_gem_object_set_cache_level()
3777 ret = i915_gem_object_wait_rendering(obj, false); in i915_gem_object_set_cache_level()
3789 i915_gem_release_mmap(obj); in i915_gem_object_set_cache_level()
3798 ret = i915_gem_object_put_fence(obj); in i915_gem_object_set_cache_level()
3811 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_object_set_cache_level()
3821 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_object_set_cache_level()
3823 obj->cache_level = cache_level; in i915_gem_object_set_cache_level()
3830 if (obj->cache_dirty && in i915_gem_object_set_cache_level()
3831 obj->base.write_domain != I915_GEM_DOMAIN_CPU && in i915_gem_object_set_cache_level()
3832 cpu_write_needs_clflush(obj)) { in i915_gem_object_set_cache_level()
3833 if (i915_gem_clflush_object(obj, true)) in i915_gem_object_set_cache_level()
3834 i915_gem_chipset_flush(obj->base.dev); in i915_gem_object_set_cache_level()
3844 struct drm_i915_gem_object *obj; in i915_gem_get_caching_ioctl() local
3846 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_get_caching_ioctl()
3847 if (&obj->base == NULL) in i915_gem_get_caching_ioctl()
3850 switch (obj->cache_level) { in i915_gem_get_caching_ioctl()
3865 drm_gem_object_unreference_unlocked(&obj->base); in i915_gem_get_caching_ioctl()
3874 struct drm_i915_gem_object *obj; in i915_gem_set_caching_ioctl() local
3907 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_set_caching_ioctl()
3908 if (&obj->base == NULL) { in i915_gem_set_caching_ioctl()
3913 ret = i915_gem_object_set_cache_level(obj, level); in i915_gem_set_caching_ioctl()
3915 drm_gem_object_unreference(&obj->base); in i915_gem_set_caching_ioctl()
3930 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, in i915_gem_object_pin_to_display_plane() argument
3939 ret = i915_gem_object_sync(obj, pipelined, pipelined_request); in i915_gem_object_pin_to_display_plane()
3946 obj->pin_display++; in i915_gem_object_pin_to_display_plane()
3957 ret = i915_gem_object_set_cache_level(obj, in i915_gem_object_pin_to_display_plane()
3958 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); in i915_gem_object_pin_to_display_plane()
3966 ret = i915_gem_object_ggtt_pin(obj, view, alignment, in i915_gem_object_pin_to_display_plane()
3972 i915_gem_object_flush_cpu_write_domain(obj); in i915_gem_object_pin_to_display_plane()
3974 old_write_domain = obj->base.write_domain; in i915_gem_object_pin_to_display_plane()
3975 old_read_domains = obj->base.read_domains; in i915_gem_object_pin_to_display_plane()
3980 obj->base.write_domain = 0; in i915_gem_object_pin_to_display_plane()
3981 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; in i915_gem_object_pin_to_display_plane()
3983 trace_i915_gem_object_change_domain(obj, in i915_gem_object_pin_to_display_plane()
3990 obj->pin_display--; in i915_gem_object_pin_to_display_plane()
3995 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, in i915_gem_object_unpin_from_display_plane() argument
3998 if (WARN_ON(obj->pin_display == 0)) in i915_gem_object_unpin_from_display_plane()
4001 i915_gem_object_ggtt_unpin_view(obj, view); in i915_gem_object_unpin_from_display_plane()
4003 obj->pin_display--; in i915_gem_object_unpin_from_display_plane()
4013 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) in i915_gem_object_set_to_cpu_domain() argument
4018 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) in i915_gem_object_set_to_cpu_domain()
4021 ret = i915_gem_object_wait_rendering(obj, !write); in i915_gem_object_set_to_cpu_domain()
4025 i915_gem_object_flush_gtt_write_domain(obj); in i915_gem_object_set_to_cpu_domain()
4027 old_write_domain = obj->base.write_domain; in i915_gem_object_set_to_cpu_domain()
4028 old_read_domains = obj->base.read_domains; in i915_gem_object_set_to_cpu_domain()
4031 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { in i915_gem_object_set_to_cpu_domain()
4032 i915_gem_clflush_object(obj, false); in i915_gem_object_set_to_cpu_domain()
4034 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4040 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); in i915_gem_object_set_to_cpu_domain()
4046 obj->base.read_domains = I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4047 obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_object_set_to_cpu_domain()
4050 trace_i915_gem_object_change_domain(obj, in i915_gem_object_set_to_cpu_domain()
4119 struct drm_i915_gem_object *obj = vma->obj; in i915_vma_misplaced() local
4125 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable) in i915_vma_misplaced()
4137 struct drm_i915_gem_object *obj = vma->obj; in __i915_vma_set_map_and_fenceable() local
4141 fence_size = i915_gem_get_gtt_size(obj->base.dev, in __i915_vma_set_map_and_fenceable()
4142 obj->base.size, in __i915_vma_set_map_and_fenceable()
4143 obj->tiling_mode); in __i915_vma_set_map_and_fenceable()
4144 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, in __i915_vma_set_map_and_fenceable()
4145 obj->base.size, in __i915_vma_set_map_and_fenceable()
4146 obj->tiling_mode, in __i915_vma_set_map_and_fenceable()
4153 to_i915(obj->base.dev)->gtt.mappable_end); in __i915_vma_set_map_and_fenceable()
4155 obj->map_and_fenceable = mappable && fenceable; in __i915_vma_set_map_and_fenceable()
4159 i915_gem_object_do_pin(struct drm_i915_gem_object *obj, in i915_gem_object_do_pin() argument
4165 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; in i915_gem_object_do_pin()
4182 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) : in i915_gem_object_do_pin()
4183 i915_gem_obj_to_vma(obj, vm); in i915_gem_object_do_pin()
4202 obj->map_and_fenceable); in i915_gem_object_do_pin()
4213 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment, in i915_gem_object_do_pin()
4218 ret = i915_vma_bind(vma, obj->cache_level, flags); in i915_gem_object_do_pin()
4226 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); in i915_gem_object_do_pin()
4234 i915_gem_object_pin(struct drm_i915_gem_object *obj, in i915_gem_object_pin() argument
4239 return i915_gem_object_do_pin(obj, vm, in i915_gem_object_pin()
4245 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, in i915_gem_object_ggtt_pin() argument
4253 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view, in i915_gem_object_ggtt_pin()
4258 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, in i915_gem_object_ggtt_unpin_view() argument
4261 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view); in i915_gem_object_ggtt_unpin_view()
4265 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view)); in i915_gem_object_ggtt_unpin_view()
4275 struct drm_i915_gem_object *obj; in i915_gem_busy_ioctl() local
4282 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); in i915_gem_busy_ioctl()
4283 if (&obj->base == NULL) { in i915_gem_busy_ioctl()
4293 ret = i915_gem_object_flush_active(obj); in i915_gem_busy_ioctl()
4298 args->busy = obj->active << 16; in i915_gem_busy_ioctl()
4299 if (obj->last_write_req) in i915_gem_busy_ioctl()
4300 args->busy |= obj->last_write_req->ring->id; in i915_gem_busy_ioctl()
4303 drm_gem_object_unreference(&obj->base); in i915_gem_busy_ioctl()
4322 struct drm_i915_gem_object *obj; in i915_gem_madvise_ioctl() local
4337 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); in i915_gem_madvise_ioctl()
4338 if (&obj->base == NULL) { in i915_gem_madvise_ioctl()
4343 if (i915_gem_obj_is_pinned(obj)) { in i915_gem_madvise_ioctl()
4348 if (obj->pages && in i915_gem_madvise_ioctl()
4349 obj->tiling_mode != I915_TILING_NONE && in i915_gem_madvise_ioctl()
4351 if (obj->madv == I915_MADV_WILLNEED) in i915_gem_madvise_ioctl()
4352 i915_gem_object_unpin_pages(obj); in i915_gem_madvise_ioctl()
4354 i915_gem_object_pin_pages(obj); in i915_gem_madvise_ioctl()
4357 if (obj->madv != __I915_MADV_PURGED) in i915_gem_madvise_ioctl()
4358 obj->madv = args->madv; in i915_gem_madvise_ioctl()
4361 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) in i915_gem_madvise_ioctl()
4362 i915_gem_object_truncate(obj); in i915_gem_madvise_ioctl()
4364 args->retained = obj->madv != __I915_MADV_PURGED; in i915_gem_madvise_ioctl()
4367 drm_gem_object_unreference(&obj->base); in i915_gem_madvise_ioctl()
4373 void i915_gem_object_init(struct drm_i915_gem_object *obj, in i915_gem_object_init() argument
4378 INIT_LIST_HEAD(&obj->global_list); in i915_gem_object_init()
4380 INIT_LIST_HEAD(&obj->ring_list[i]); in i915_gem_object_init()
4381 INIT_LIST_HEAD(&obj->obj_exec_link); in i915_gem_object_init()
4382 INIT_LIST_HEAD(&obj->vma_list); in i915_gem_object_init()
4383 INIT_LIST_HEAD(&obj->batch_pool_link); in i915_gem_object_init()
4385 obj->ops = ops; in i915_gem_object_init()
4387 obj->fence_reg = I915_FENCE_REG_NONE; in i915_gem_object_init()
4388 obj->madv = I915_MADV_WILLNEED; in i915_gem_object_init()
4390 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); in i915_gem_object_init()
4401 struct drm_i915_gem_object *obj; in i915_gem_alloc_object() local
4405 obj = i915_gem_object_alloc(dev); in i915_gem_alloc_object()
4406 if (obj == NULL) in i915_gem_alloc_object()
4409 if (drm_gem_object_init(dev, &obj->base, size) != 0) { in i915_gem_alloc_object()
4410 i915_gem_object_free(obj); in i915_gem_alloc_object()
4421 mapping = file_inode(obj->base.filp)->i_mapping; in i915_gem_alloc_object()
4424 i915_gem_object_init(obj, &i915_gem_object_ops); in i915_gem_alloc_object()
4426 obj->base.write_domain = I915_GEM_DOMAIN_CPU; in i915_gem_alloc_object()
4427 obj->base.read_domains = I915_GEM_DOMAIN_CPU; in i915_gem_alloc_object()
4442 obj->cache_level = I915_CACHE_LLC; in i915_gem_alloc_object()
4444 obj->cache_level = I915_CACHE_NONE; in i915_gem_alloc_object()
4446 trace_i915_gem_object_create(obj); in i915_gem_alloc_object()
4448 return obj; in i915_gem_alloc_object()
4451 static bool discard_backing_storage(struct drm_i915_gem_object *obj) in discard_backing_storage() argument
4459 if (obj->madv != I915_MADV_WILLNEED) in discard_backing_storage()
4462 if (obj->base.filp == NULL) in discard_backing_storage()
4472 return atomic_long_read(&obj->base.filp->f_count) == 1; in discard_backing_storage()
4477 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); in i915_gem_free_object() local
4478 struct drm_device *dev = obj->base.dev; in i915_gem_free_object()
4484 trace_i915_gem_object_destroy(obj); in i915_gem_free_object()
4486 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { in i915_gem_free_object()
4505 if (obj->stolen) in i915_gem_free_object()
4506 i915_gem_object_unpin_pages(obj); in i915_gem_free_object()
4508 WARN_ON(obj->frontbuffer_bits); in i915_gem_free_object()
4510 if (obj->pages && obj->madv == I915_MADV_WILLNEED && in i915_gem_free_object()
4512 obj->tiling_mode != I915_TILING_NONE) in i915_gem_free_object()
4513 i915_gem_object_unpin_pages(obj); in i915_gem_free_object()
4515 if (WARN_ON(obj->pages_pin_count)) in i915_gem_free_object()
4516 obj->pages_pin_count = 0; in i915_gem_free_object()
4517 if (discard_backing_storage(obj)) in i915_gem_free_object()
4518 obj->madv = I915_MADV_DONTNEED; in i915_gem_free_object()
4519 i915_gem_object_put_pages(obj); in i915_gem_free_object()
4520 i915_gem_object_free_mmap_offset(obj); in i915_gem_free_object()
4522 BUG_ON(obj->pages); in i915_gem_free_object()
4524 if (obj->base.import_attach) in i915_gem_free_object()
4525 drm_prime_gem_destroy(&obj->base, NULL); in i915_gem_free_object()
4527 if (obj->ops->release) in i915_gem_free_object()
4528 obj->ops->release(obj); in i915_gem_free_object()
4530 drm_gem_object_release(&obj->base); in i915_gem_free_object()
4531 i915_gem_info_remove_obj(dev_priv, obj->base.size); in i915_gem_free_object()
4533 kfree(obj->bit_17); in i915_gem_free_object()
4534 i915_gem_object_free(obj); in i915_gem_free_object()
4539 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, in i915_gem_obj_to_vma() argument
4543 list_for_each_entry(vma, &obj->vma_list, vma_link) { in i915_gem_obj_to_vma()
4553 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, in i915_gem_obj_to_ggtt_view() argument
4556 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); in i915_gem_obj_to_ggtt_view()
4562 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_obj_to_ggtt_view()
4585 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); in i915_gem_vma_destroy()
5233 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) in i915_gem_obj_is_pinned() argument
5236 list_for_each_entry(vma, &obj->vma_list, vma_link) in i915_gem_obj_is_pinned()
5248 struct drm_i915_gem_object *obj; in i915_gem_object_create_from_data() local
5253 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE)); in i915_gem_object_create_from_data()
5254 if (IS_ERR_OR_NULL(obj)) in i915_gem_object_create_from_data()
5255 return obj; in i915_gem_object_create_from_data()
5257 ret = i915_gem_object_set_to_cpu_domain(obj, true); in i915_gem_object_create_from_data()
5261 ret = i915_gem_object_get_pages(obj); in i915_gem_object_create_from_data()
5265 i915_gem_object_pin_pages(obj); in i915_gem_object_create_from_data()
5266 sg = obj->pages; in i915_gem_object_create_from_data()
5268 i915_gem_object_unpin_pages(obj); in i915_gem_object_create_from_data()
5276 return obj; in i915_gem_object_create_from_data()
5279 drm_gem_object_unreference(&obj->base); in i915_gem_object_create_from_data()