27 static dma_addr_t physaddr(struct drm_gem_object *obj)  in physaddr()  argument
29 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in physaddr()
30 	struct msm_drm_private *priv = obj->dev->dev_private;  in physaddr()
35 static bool use_pages(struct drm_gem_object *obj)  in use_pages()  argument
37 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in use_pages()
42 static struct page **get_pages_vram(struct drm_gem_object *obj,  in get_pages_vram()  argument
45 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages_vram()
46 	struct msm_drm_private *priv = obj->dev->dev_private;  in get_pages_vram()
62 	paddr = physaddr(obj);  in get_pages_vram()
72 static struct page **get_pages(struct drm_gem_object *obj)  in get_pages()  argument
74 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages()
77 		struct drm_device *dev = obj->dev;  in get_pages()
79 		int npages = obj->size >> PAGE_SHIFT;  in get_pages()
81 		if (use_pages(obj))  in get_pages()
82 			p = drm_gem_get_pages(obj);  in get_pages()
84 			p = get_pages_vram(obj, npages);  in get_pages()
111 static void put_pages(struct drm_gem_object *obj)  in put_pages()  argument
113 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages()
120 			dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,  in put_pages()
125 		if (use_pages(obj))  in put_pages()
126 			drm_gem_put_pages(obj, msm_obj->pages, true, false);  in put_pages()
136 struct page **msm_gem_get_pages(struct drm_gem_object *obj)  in msm_gem_get_pages()  argument
138 	struct drm_device *dev = obj->dev;  in msm_gem_get_pages()
141 	p = get_pages(obj);  in msm_gem_get_pages()
146 void msm_gem_put_pages(struct drm_gem_object *obj)  in msm_gem_put_pages()  argument
151 int msm_gem_mmap_obj(struct drm_gem_object *obj,  in msm_gem_mmap_obj()  argument
154 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_mmap_obj()
170 		get_file(obj->filp);  in msm_gem_mmap_obj()
172 		vma->vm_file  = obj->filp;  in msm_gem_mmap_obj()
195 	struct drm_gem_object *obj = vma->vm_private_data;  in msm_gem_fault()  local
196 	struct drm_device *dev = obj->dev;  in msm_gem_fault()
210 	pages = get_pages(obj);  in msm_gem_fault()
249 static uint64_t mmap_offset(struct drm_gem_object *obj)  in mmap_offset()  argument
251 	struct drm_device *dev = obj->dev;  in mmap_offset()
257 	ret = drm_gem_create_mmap_offset(obj);  in mmap_offset()
264 	return drm_vma_node_offset_addr(&obj->vma_node);  in mmap_offset()
267 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)  in msm_gem_mmap_offset()  argument
270 	mutex_lock(&obj->dev->struct_mutex);  in msm_gem_mmap_offset()
271 	offset = mmap_offset(obj);  in msm_gem_mmap_offset()
272 	mutex_unlock(&obj->dev->struct_mutex);  in msm_gem_mmap_offset()
283 int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,  in msm_gem_get_iova_locked()  argument
286 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_get_iova_locked()
290 		struct msm_drm_private *priv = obj->dev->dev_private;  in msm_gem_get_iova_locked()
291 		struct page **pages = get_pages(obj);  in msm_gem_get_iova_locked()
303 			offset = (uint32_t)mmap_offset(obj);  in msm_gem_get_iova_locked()
305 					obj->size, IOMMU_READ | IOMMU_WRITE);  in msm_gem_get_iova_locked()
308 			msm_obj->domain[id].iova = physaddr(obj);  in msm_gem_get_iova_locked()
319 int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)  in msm_gem_get_iova()  argument
321 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_get_iova()
332 	mutex_lock(&obj->dev->struct_mutex);  in msm_gem_get_iova()
333 	ret = msm_gem_get_iova_locked(obj, id, iova);  in msm_gem_get_iova()
334 	mutex_unlock(&obj->dev->struct_mutex);  in msm_gem_get_iova()
341 uint32_t msm_gem_iova(struct drm_gem_object *obj, int id)  in msm_gem_iova()  argument
343 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_iova()
348 void msm_gem_put_iova(struct drm_gem_object *obj, int id)  in msm_gem_put_iova()  argument
370 	struct drm_gem_object *obj;  in msm_gem_dumb_map_offset()  local
374 	obj = drm_gem_object_lookup(dev, file, handle);  in msm_gem_dumb_map_offset()
375 	if (obj == NULL) {  in msm_gem_dumb_map_offset()
380 	*offset = msm_gem_mmap_offset(obj);  in msm_gem_dumb_map_offset()
382 	drm_gem_object_unreference_unlocked(obj);  in msm_gem_dumb_map_offset()
388 void *msm_gem_vaddr_locked(struct drm_gem_object *obj)  in msm_gem_vaddr_locked()  argument
390 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_vaddr_locked()
391 	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));  in msm_gem_vaddr_locked()
393 		struct page **pages = get_pages(obj);  in msm_gem_vaddr_locked()
396 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,  in msm_gem_vaddr_locked()
402 void *msm_gem_vaddr(struct drm_gem_object *obj)  in msm_gem_vaddr()  argument
405 	mutex_lock(&obj->dev->struct_mutex);  in msm_gem_vaddr()
406 	ret = msm_gem_vaddr_locked(obj);  in msm_gem_vaddr()
407 	mutex_unlock(&obj->dev->struct_mutex);  in msm_gem_vaddr()
414 int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,  in msm_gem_queue_inactive_cb()  argument
417 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_queue_inactive_cb()
420 	return msm_queue_fence_cb(obj->dev, cb, fence);  in msm_gem_queue_inactive_cb()
423 void msm_gem_move_to_active(struct drm_gem_object *obj,  in msm_gem_move_to_active()  argument
426 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_move_to_active()
436 void msm_gem_move_to_inactive(struct drm_gem_object *obj)  in msm_gem_move_to_inactive()  argument
438 	struct drm_device *dev = obj->dev;  in msm_gem_move_to_inactive()
440 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_move_to_inactive()
451 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)  in msm_gem_cpu_prep()  argument
453 	struct drm_device *dev = obj->dev;  in msm_gem_cpu_prep()
454 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_cpu_prep()
471 int msm_gem_cpu_fini(struct drm_gem_object *obj)  in msm_gem_cpu_fini()  argument
478 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)  in msm_gem_describe()  argument
480 	struct drm_device *dev = obj->dev;  in msm_gem_describe()
481 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_describe()
482 	uint64_t off = drm_vma_node_start(&obj->vma_node);  in msm_gem_describe()
488 			obj->name, obj->refcount.refcount.counter,  in msm_gem_describe()
489 			off, msm_obj->vaddr, obj->size);  in msm_gem_describe()
499 		struct drm_gem_object *obj = &msm_obj->base;  in msm_gem_describe_objects()  local
501 		msm_gem_describe(obj, m);  in msm_gem_describe_objects()
503 		size += obj->size;  in msm_gem_describe_objects()
510 void msm_gem_free_object(struct drm_gem_object *obj)  in msm_gem_free_object()  argument
512 	struct drm_device *dev = obj->dev;  in msm_gem_free_object()
513 	struct msm_drm_private *priv = obj->dev->dev_private;  in msm_gem_free_object()
514 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_free_object()
528 			mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);  in msm_gem_free_object()
532 	if (obj->import_attach) {  in msm_gem_free_object()
534 			dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);  in msm_gem_free_object()
542 		drm_prime_gem_destroy(obj, msm_obj->sgt);  in msm_gem_free_object()
545 		put_pages(obj);  in msm_gem_free_object()
551 	drm_gem_object_release(obj);  in msm_gem_free_object()
560 	struct drm_gem_object *obj;  in msm_gem_new_handle()  local
567 	obj = msm_gem_new(dev, size, flags);  in msm_gem_new_handle()
571 	if (IS_ERR(obj))  in msm_gem_new_handle()
572 		return PTR_ERR(obj);  in msm_gem_new_handle()
574 	ret = drm_gem_handle_create(file, obj, handle);  in msm_gem_new_handle()
577 	drm_gem_object_unreference_unlocked(obj);  in msm_gem_new_handle()
584 		struct drm_gem_object **obj)  in msm_gem_new_impl()  argument
629 	*obj = &msm_obj->base;  in msm_gem_new_impl()
637 	struct drm_gem_object *obj = NULL;  in msm_gem_new()  local
644 	ret = msm_gem_new_impl(dev, size, flags, &obj);  in msm_gem_new()
648 	if (use_pages(obj)) {  in msm_gem_new()
649 		ret = drm_gem_object_init(dev, obj, size);  in msm_gem_new()
653 		drm_gem_private_object_init(dev, obj, size);  in msm_gem_new()
656 	return obj;  in msm_gem_new()
659 	if (obj)  in msm_gem_new()
660 		drm_gem_object_unreference(obj);  in msm_gem_new()
669 	struct drm_gem_object *obj;  in msm_gem_import()  local
680 	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);  in msm_gem_import()
684 	drm_gem_private_object_init(dev, obj, size);  in msm_gem_import()
688 	msm_obj = to_msm_bo(obj);  in msm_gem_import()
700 	return obj;  in msm_gem_import()
703 	if (obj)  in msm_gem_import()
704 		drm_gem_object_unreference_unlocked(obj);  in msm_gem_import()