72 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_cookie()  local
74 	return &buf->dma_addr;  in vb2_dc_cookie()
79 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_vaddr()  local
81 	if (!buf->vaddr && buf->db_attach)  in vb2_dc_vaddr()
82 		buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);  in vb2_dc_vaddr()
84 	return buf->vaddr;  in vb2_dc_vaddr()
89 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_num_users()  local
91 	return atomic_read(&buf->refcount);  in vb2_dc_num_users()
96 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_prepare()  local
97 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_prepare()
100 	if (!sgt || buf->db_attach)  in vb2_dc_prepare()
103 	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_prepare()
104 			       buf->dma_dir);  in vb2_dc_prepare()
109 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_finish()  local
110 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_finish()
113 	if (!sgt || buf->db_attach)  in vb2_dc_finish()
116 	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir);  in vb2_dc_finish()
125 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_put()  local
127 	if (!atomic_dec_and_test(&buf->refcount))  in vb2_dc_put()
130 	if (buf->sgt_base) {  in vb2_dc_put()
131 		sg_free_table(buf->sgt_base);  in vb2_dc_put()
132 		kfree(buf->sgt_base);  in vb2_dc_put()
134 	dma_free_coherent(buf->dev, buf->size, buf->vaddr, buf->dma_addr);  in vb2_dc_put()
135 	put_device(buf->dev);  in vb2_dc_put()
136 	kfree(buf);  in vb2_dc_put()
144 	struct vb2_dc_buf *buf;  in vb2_dc_alloc()  local
146 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dc_alloc()
147 	if (!buf)  in vb2_dc_alloc()
150 	buf->vaddr = dma_alloc_coherent(dev, size, &buf->dma_addr,  in vb2_dc_alloc()
152 	if (!buf->vaddr) {  in vb2_dc_alloc()
154 		kfree(buf);  in vb2_dc_alloc()
159 	buf->dev = get_device(dev);  in vb2_dc_alloc()
160 	buf->size = size;  in vb2_dc_alloc()
161 	buf->dma_dir = dma_dir;  in vb2_dc_alloc()
163 	buf->handler.refcount = &buf->refcount;  in vb2_dc_alloc()
164 	buf->handler.put = vb2_dc_put;  in vb2_dc_alloc()
165 	buf->handler.arg = buf;  in vb2_dc_alloc()
167 	atomic_inc(&buf->refcount);  in vb2_dc_alloc()
169 	return buf;  in vb2_dc_alloc()
174 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_mmap()  local
177 	if (!buf) {  in vb2_dc_mmap()
188 	ret = dma_mmap_coherent(buf->dev, vma, buf->vaddr,  in vb2_dc_mmap()
189 		buf->dma_addr, buf->size);  in vb2_dc_mmap()
197 	vma->vm_private_data	= &buf->handler;  in vb2_dc_mmap()
203 		__func__, (unsigned long)buf->dma_addr, vma->vm_start,  in vb2_dc_mmap()
204 		buf->size);  in vb2_dc_mmap()
225 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_attach()  local
236 	ret = sg_alloc_table(sgt, buf->sgt_base->orig_nents, GFP_KERNEL);  in vb2_dc_dmabuf_ops_attach()
242 	rd = buf->sgt_base->sgl;  in vb2_dc_dmabuf_ops_attach()
330 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_kmap()  local
332 	return buf->vaddr + pgnum * PAGE_SIZE;  in vb2_dc_dmabuf_ops_kmap()
337 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_vmap()  local
339 	return buf->vaddr;  in vb2_dc_dmabuf_ops_vmap()
360 static struct sg_table *vb2_dc_get_base_sgt(struct vb2_dc_buf *buf)  in vb2_dc_get_base_sgt()  argument
367 		dev_err(buf->dev, "failed to alloc sg table\n");  in vb2_dc_get_base_sgt()
371 	ret = dma_get_sgtable(buf->dev, sgt, buf->vaddr, buf->dma_addr,  in vb2_dc_get_base_sgt()
372 		buf->size);  in vb2_dc_get_base_sgt()
374 		dev_err(buf->dev, "failed to get scatterlist from DMA API\n");  in vb2_dc_get_base_sgt()
384 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_get_dmabuf()  local
389 	exp_info.size = buf->size;  in vb2_dc_get_dmabuf()
391 	exp_info.priv = buf;  in vb2_dc_get_dmabuf()
393 	if (!buf->sgt_base)  in vb2_dc_get_dmabuf()
394 		buf->sgt_base = vb2_dc_get_base_sgt(buf);  in vb2_dc_get_dmabuf()
396 	if (WARN_ON(!buf->sgt_base))  in vb2_dc_get_dmabuf()
404 	atomic_inc(&buf->refcount);  in vb2_dc_get_dmabuf()
415 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_put_userptr()  local
416 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_put_userptr()
428 		dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_put_userptr()
429 				   buf->dma_dir, &attrs);  in vb2_dc_put_userptr()
430 		pages = frame_vector_pages(buf->vec);  in vb2_dc_put_userptr()
433 		for (i = 0; i < frame_vector_count(buf->vec); i++)  in vb2_dc_put_userptr()
438 	vb2_destroy_framevec(buf->vec);  in vb2_dc_put_userptr()
439 	kfree(buf);  in vb2_dc_put_userptr()
477 	struct vb2_dc_buf *buf;  in vb2_dc_get_userptr()  local
500 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dc_get_userptr()
501 	if (!buf)  in vb2_dc_get_userptr()
504 	buf->dev = conf->dev;  in vb2_dc_get_userptr()
505 	buf->dma_dir = dma_dir;  in vb2_dc_get_userptr()
513 	buf->vec = vec;  in vb2_dc_get_userptr()
526 		buf->dma_addr = vb2_dc_pfn_to_dma(buf->dev, nums[0]);  in vb2_dc_get_userptr()
548 	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_get_userptr()
549 				      buf->dma_dir, &attrs);  in vb2_dc_get_userptr()
564 	buf->dma_addr = sg_dma_address(sgt->sgl);  in vb2_dc_get_userptr()
565 	buf->dma_sgt = sgt;  in vb2_dc_get_userptr()
567 	buf->size = size;  in vb2_dc_get_userptr()
569 	return buf;  in vb2_dc_get_userptr()
572 	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_get_userptr()
573 			   buf->dma_dir, &attrs);  in vb2_dc_get_userptr()
585 	kfree(buf);  in vb2_dc_get_userptr()
596 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_map_dmabuf()  local
600 	if (WARN_ON(!buf->db_attach)) {  in vb2_dc_map_dmabuf()
605 	if (WARN_ON(buf->dma_sgt)) {  in vb2_dc_map_dmabuf()
611 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);  in vb2_dc_map_dmabuf()
619 	if (contig_size < buf->size) {  in vb2_dc_map_dmabuf()
621 			contig_size, buf->size);  in vb2_dc_map_dmabuf()
622 		dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dc_map_dmabuf()
626 	buf->dma_addr = sg_dma_address(sgt->sgl);  in vb2_dc_map_dmabuf()
627 	buf->dma_sgt = sgt;  in vb2_dc_map_dmabuf()
628 	buf->vaddr = NULL;  in vb2_dc_map_dmabuf()
635 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_unmap_dmabuf()  local
636 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_unmap_dmabuf()
638 	if (WARN_ON(!buf->db_attach)) {  in vb2_dc_unmap_dmabuf()
648 	if (buf->vaddr) {  in vb2_dc_unmap_dmabuf()
649 		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);  in vb2_dc_unmap_dmabuf()
650 		buf->vaddr = NULL;  in vb2_dc_unmap_dmabuf()
652 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dc_unmap_dmabuf()
654 	buf->dma_addr = 0;  in vb2_dc_unmap_dmabuf()
655 	buf->dma_sgt = NULL;  in vb2_dc_unmap_dmabuf()
660 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_detach_dmabuf()  local
663 	if (WARN_ON(buf->dma_addr))  in vb2_dc_detach_dmabuf()
664 		vb2_dc_unmap_dmabuf(buf);  in vb2_dc_detach_dmabuf()
667 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);  in vb2_dc_detach_dmabuf()
668 	kfree(buf);  in vb2_dc_detach_dmabuf()
675 	struct vb2_dc_buf *buf;  in vb2_dc_attach_dmabuf()  local
681 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_dc_attach_dmabuf()
682 	if (!buf)  in vb2_dc_attach_dmabuf()
685 	buf->dev = conf->dev;  in vb2_dc_attach_dmabuf()
687 	dba = dma_buf_attach(dbuf, buf->dev);  in vb2_dc_attach_dmabuf()
690 		kfree(buf);  in vb2_dc_attach_dmabuf()
694 	buf->dma_dir = dma_dir;  in vb2_dc_attach_dmabuf()
695 	buf->size = size;  in vb2_dc_attach_dmabuf()
696 	buf->db_attach = dba;  in vb2_dc_attach_dmabuf()
698 	return buf;  in vb2_dc_attach_dmabuf()