92 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_cookie()  local
94 	return &buf->dma_addr;  in vb2_dc_cookie()
99 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_vaddr()  local
101 	if (!buf->vaddr && buf->db_attach)  in vb2_dc_vaddr()
102 		buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);  in vb2_dc_vaddr()
104 	return buf->vaddr;  in vb2_dc_vaddr()
109 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_num_users()  local
111 	return atomic_read(&buf->refcount);  in vb2_dc_num_users()
116 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_prepare()  local
117 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_prepare()
120 	if (!sgt || buf->db_attach)  in vb2_dc_prepare()
123 	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);  in vb2_dc_prepare()
128 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_finish()  local
129 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_finish()
132 	if (!sgt || buf->db_attach)  in vb2_dc_finish()
135 	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);  in vb2_dc_finish()
144 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_put()  local
146 	if (!atomic_dec_and_test(&buf->refcount))  in vb2_dc_put()
149 	if (buf->sgt_base) {  in vb2_dc_put()
150 		sg_free_table(buf->sgt_base);  in vb2_dc_put()
151 		kfree(buf->sgt_base);  in vb2_dc_put()
153 	dma_free_coherent(buf->dev, buf->size, buf->vaddr, buf->dma_addr);  in vb2_dc_put()
154 	put_device(buf->dev);  in vb2_dc_put()
155 	kfree(buf);  in vb2_dc_put()
163 	struct vb2_dc_buf *buf;  in vb2_dc_alloc()  local
165 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dc_alloc()
166 	if (!buf)  in vb2_dc_alloc()
169 	buf->vaddr = dma_alloc_coherent(dev, size, &buf->dma_addr,  in vb2_dc_alloc()
171 	if (!buf->vaddr) {  in vb2_dc_alloc()
173 		kfree(buf);  in vb2_dc_alloc()
178 	buf->dev = get_device(dev);  in vb2_dc_alloc()
179 	buf->size = size;  in vb2_dc_alloc()
180 	buf->dma_dir = dma_dir;  in vb2_dc_alloc()
182 	buf->handler.refcount = &buf->refcount;  in vb2_dc_alloc()
183 	buf->handler.put = vb2_dc_put;  in vb2_dc_alloc()
184 	buf->handler.arg = buf;  in vb2_dc_alloc()
186 	atomic_inc(&buf->refcount);  in vb2_dc_alloc()
188 	return buf;  in vb2_dc_alloc()
193 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_mmap()  local
196 	if (!buf) {  in vb2_dc_mmap()
207 	ret = dma_mmap_coherent(buf->dev, vma, buf->vaddr,  in vb2_dc_mmap()
208 		buf->dma_addr, buf->size);  in vb2_dc_mmap()
216 	vma->vm_private_data	= &buf->handler;  in vb2_dc_mmap()
222 		__func__, (unsigned long)buf->dma_addr, vma->vm_start,  in vb2_dc_mmap()
223 		buf->size);  in vb2_dc_mmap()
244 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_attach()  local
255 	ret = sg_alloc_table(sgt, buf->sgt_base->orig_nents, GFP_KERNEL);  in vb2_dc_dmabuf_ops_attach()
261 	rd = buf->sgt_base->sgl;  in vb2_dc_dmabuf_ops_attach()
349 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_kmap()  local
351 	return buf->vaddr + pgnum * PAGE_SIZE;  in vb2_dc_dmabuf_ops_kmap()
356 	struct vb2_dc_buf *buf = dbuf->priv;  in vb2_dc_dmabuf_ops_vmap()  local
358 	return buf->vaddr;  in vb2_dc_dmabuf_ops_vmap()
379 static struct sg_table *vb2_dc_get_base_sgt(struct vb2_dc_buf *buf)  in vb2_dc_get_base_sgt()  argument
386 		dev_err(buf->dev, "failed to alloc sg table\n");  in vb2_dc_get_base_sgt()
390 	ret = dma_get_sgtable(buf->dev, sgt, buf->vaddr, buf->dma_addr,  in vb2_dc_get_base_sgt()
391 		buf->size);  in vb2_dc_get_base_sgt()
393 		dev_err(buf->dev, "failed to get scatterlist from DMA API\n");  in vb2_dc_get_base_sgt()
403 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_get_dmabuf()  local
408 	exp_info.size = buf->size;  in vb2_dc_get_dmabuf()
410 	exp_info.priv = buf;  in vb2_dc_get_dmabuf()
412 	if (!buf->sgt_base)  in vb2_dc_get_dmabuf()
413 		buf->sgt_base = vb2_dc_get_base_sgt(buf);  in vb2_dc_get_dmabuf()
415 	if (WARN_ON(!buf->sgt_base))  in vb2_dc_get_dmabuf()
423 	atomic_inc(&buf->refcount);  in vb2_dc_get_dmabuf()
516 	struct vb2_dc_buf *buf = buf_priv;  in vb2_dc_put_userptr()  local
517 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_put_userptr()
527 		dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_put_userptr()
528 				   buf->dma_dir, &attrs);  in vb2_dc_put_userptr()
529 		if (!vma_is_io(buf->vma))  in vb2_dc_put_userptr()
535 	vb2_put_vma(buf->vma);  in vb2_dc_put_userptr()
536 	kfree(buf);  in vb2_dc_put_userptr()
574 	struct vb2_dc_buf *buf;  in vb2_dc_get_userptr()  local
600 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dc_get_userptr()
601 	if (!buf)  in vb2_dc_get_userptr()
604 	buf->dev = conf->dev;  in vb2_dc_get_userptr()
605 	buf->dma_dir = dma_dir;  in vb2_dc_get_userptr()
633 	buf->vma = vb2_get_vma(vma);  in vb2_dc_get_userptr()
634 	if (!buf->vma) {  in vb2_dc_get_userptr()
645 			buf->dma_addr = vb2_dc_pfn_to_dma(buf->dev, pfn);  in vb2_dc_get_userptr()
646 			buf->size = size;  in vb2_dc_get_userptr()
648 			return buf;  in vb2_dc_get_userptr()
677 	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_get_userptr()
678 				      buf->dma_dir, &attrs);  in vb2_dc_get_userptr()
693 	buf->dma_addr = sg_dma_address(sgt->sgl);  in vb2_dc_get_userptr()
694 	buf->size = size;  in vb2_dc_get_userptr()
695 	buf->dma_sgt = sgt;  in vb2_dc_get_userptr()
697 	return buf;  in vb2_dc_get_userptr()
700 	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dc_get_userptr()
701 			   buf->dma_dir, &attrs);  in vb2_dc_get_userptr()
704 	if (!vma_is_io(buf->vma))  in vb2_dc_get_userptr()
712 	if (pages && !vma_is_io(buf->vma))  in vb2_dc_get_userptr()
717 	vb2_put_vma(buf->vma);  in vb2_dc_get_userptr()
723 	kfree(buf);  in vb2_dc_get_userptr()
734 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_map_dmabuf()  local
738 	if (WARN_ON(!buf->db_attach)) {  in vb2_dc_map_dmabuf()
743 	if (WARN_ON(buf->dma_sgt)) {  in vb2_dc_map_dmabuf()
749 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);  in vb2_dc_map_dmabuf()
757 	if (contig_size < buf->size) {  in vb2_dc_map_dmabuf()
759 			contig_size, buf->size);  in vb2_dc_map_dmabuf()
760 		dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dc_map_dmabuf()
764 	buf->dma_addr = sg_dma_address(sgt->sgl);  in vb2_dc_map_dmabuf()
765 	buf->dma_sgt = sgt;  in vb2_dc_map_dmabuf()
766 	buf->vaddr = NULL;  in vb2_dc_map_dmabuf()
773 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_unmap_dmabuf()  local
774 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dc_unmap_dmabuf()
776 	if (WARN_ON(!buf->db_attach)) {  in vb2_dc_unmap_dmabuf()
786 	if (buf->vaddr) {  in vb2_dc_unmap_dmabuf()
787 		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);  in vb2_dc_unmap_dmabuf()
788 		buf->vaddr = NULL;  in vb2_dc_unmap_dmabuf()
790 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dc_unmap_dmabuf()
792 	buf->dma_addr = 0;  in vb2_dc_unmap_dmabuf()
793 	buf->dma_sgt = NULL;  in vb2_dc_unmap_dmabuf()
798 	struct vb2_dc_buf *buf = mem_priv;  in vb2_dc_detach_dmabuf()  local
801 	if (WARN_ON(buf->dma_addr))  in vb2_dc_detach_dmabuf()
802 		vb2_dc_unmap_dmabuf(buf);  in vb2_dc_detach_dmabuf()
805 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);  in vb2_dc_detach_dmabuf()
806 	kfree(buf);  in vb2_dc_detach_dmabuf()
813 	struct vb2_dc_buf *buf;  in vb2_dc_attach_dmabuf()  local
819 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_dc_attach_dmabuf()
820 	if (!buf)  in vb2_dc_attach_dmabuf()
823 	buf->dev = conf->dev;  in vb2_dc_attach_dmabuf()
825 	dba = dma_buf_attach(dbuf, buf->dev);  in vb2_dc_attach_dmabuf()
828 		kfree(buf);  in vb2_dc_attach_dmabuf()
832 	buf->dma_dir = dma_dir;  in vb2_dc_attach_dmabuf()
833 	buf->size = size;  in vb2_dc_attach_dmabuf()
834 	buf->db_attach = dba;  in vb2_dc_attach_dmabuf()
836 	return buf;  in vb2_dc_attach_dmabuf()