61 static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,  in vb2_dma_sg_alloc_compacted()  argument
65 	int size = buf->size;  in vb2_dma_sg_alloc_compacted()
86 					__free_page(buf->pages[last_page]);  in vb2_dma_sg_alloc_compacted()
94 			buf->pages[last_page++] = &pages[i];  in vb2_dma_sg_alloc_compacted()
106 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_alloc()  local
116 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dma_sg_alloc()
117 	if (!buf)  in vb2_dma_sg_alloc()
120 	buf->vaddr = NULL;  in vb2_dma_sg_alloc()
121 	buf->dma_dir = dma_dir;  in vb2_dma_sg_alloc()
122 	buf->offset = 0;  in vb2_dma_sg_alloc()
123 	buf->size = size;  in vb2_dma_sg_alloc()
125 	buf->num_pages = size >> PAGE_SHIFT;  in vb2_dma_sg_alloc()
126 	buf->dma_sgt = &buf->sg_table;  in vb2_dma_sg_alloc()
128 	buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),  in vb2_dma_sg_alloc()
130 	if (!buf->pages)  in vb2_dma_sg_alloc()
133 	ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);  in vb2_dma_sg_alloc()
137 	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_dma_sg_alloc()
138 			buf->num_pages, 0, size, GFP_KERNEL);  in vb2_dma_sg_alloc()
143 	buf->dev = get_device(conf->dev);  in vb2_dma_sg_alloc()
145 	sgt = &buf->sg_table;  in vb2_dma_sg_alloc()
150 	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dma_sg_alloc()
151 				      buf->dma_dir, &attrs);  in vb2_dma_sg_alloc()
155 	buf->handler.refcount = &buf->refcount;  in vb2_dma_sg_alloc()
156 	buf->handler.put = vb2_dma_sg_put;  in vb2_dma_sg_alloc()
157 	buf->handler.arg = buf;  in vb2_dma_sg_alloc()
159 	atomic_inc(&buf->refcount);  in vb2_dma_sg_alloc()
162 		__func__, buf->num_pages);  in vb2_dma_sg_alloc()
163 	return buf;  in vb2_dma_sg_alloc()
166 	put_device(buf->dev);  in vb2_dma_sg_alloc()
167 	sg_free_table(buf->dma_sgt);  in vb2_dma_sg_alloc()
169 	num_pages = buf->num_pages;  in vb2_dma_sg_alloc()
171 		__free_page(buf->pages[num_pages]);  in vb2_dma_sg_alloc()
173 	kfree(buf->pages);  in vb2_dma_sg_alloc()
175 	kfree(buf);  in vb2_dma_sg_alloc()
181 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_put()  local
182 	struct sg_table *sgt = &buf->sg_table;  in vb2_dma_sg_put()
183 	int i = buf->num_pages;  in vb2_dma_sg_put()
185 	if (atomic_dec_and_test(&buf->refcount)) {  in vb2_dma_sg_put()
190 			buf->num_pages);  in vb2_dma_sg_put()
191 		dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dma_sg_put()
192 				   buf->dma_dir, &attrs);  in vb2_dma_sg_put()
193 		if (buf->vaddr)  in vb2_dma_sg_put()
194 			vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_dma_sg_put()
195 		sg_free_table(buf->dma_sgt);  in vb2_dma_sg_put()
197 			__free_page(buf->pages[i]);  in vb2_dma_sg_put()
198 		kfree(buf->pages);  in vb2_dma_sg_put()
199 		put_device(buf->dev);  in vb2_dma_sg_put()
200 		kfree(buf);  in vb2_dma_sg_put()
206 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_prepare()  local
207 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_prepare()
210 	if (buf->db_attach)  in vb2_dma_sg_prepare()
213 	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dma_sg_prepare()
214 			       buf->dma_dir);  in vb2_dma_sg_prepare()
219 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_finish()  local
220 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_finish()
223 	if (buf->db_attach)  in vb2_dma_sg_finish()
226 	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir);  in vb2_dma_sg_finish()
234 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_get_userptr()  local
240 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dma_sg_get_userptr()
241 	if (!buf)  in vb2_dma_sg_get_userptr()
244 	buf->vaddr = NULL;  in vb2_dma_sg_get_userptr()
245 	buf->dev = conf->dev;  in vb2_dma_sg_get_userptr()
246 	buf->dma_dir = dma_dir;  in vb2_dma_sg_get_userptr()
247 	buf->offset = vaddr & ~PAGE_MASK;  in vb2_dma_sg_get_userptr()
248 	buf->size = size;  in vb2_dma_sg_get_userptr()
249 	buf->dma_sgt = &buf->sg_table;  in vb2_dma_sg_get_userptr()
250 	vec = vb2_create_framevec(vaddr, size, buf->dma_dir == DMA_FROM_DEVICE);  in vb2_dma_sg_get_userptr()
253 	buf->vec = vec;  in vb2_dma_sg_get_userptr()
255 	buf->pages = frame_vector_pages(vec);  in vb2_dma_sg_get_userptr()
256 	if (IS_ERR(buf->pages))  in vb2_dma_sg_get_userptr()
258 	buf->num_pages = frame_vector_count(vec);  in vb2_dma_sg_get_userptr()
260 	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_dma_sg_get_userptr()
261 			buf->num_pages, buf->offset, size, 0))  in vb2_dma_sg_get_userptr()
264 	sgt = &buf->sg_table;  in vb2_dma_sg_get_userptr()
269 	sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,  in vb2_dma_sg_get_userptr()
270 				      buf->dma_dir, &attrs);  in vb2_dma_sg_get_userptr()
274 	return buf;  in vb2_dma_sg_get_userptr()
277 	sg_free_table(&buf->sg_table);  in vb2_dma_sg_get_userptr()
281 	kfree(buf);  in vb2_dma_sg_get_userptr()
291 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_put_userptr()  local
292 	struct sg_table *sgt = &buf->sg_table;  in vb2_dma_sg_put_userptr()
293 	int i = buf->num_pages;  in vb2_dma_sg_put_userptr()
299 	       __func__, buf->num_pages);  in vb2_dma_sg_put_userptr()
300 	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,  in vb2_dma_sg_put_userptr()
302 	if (buf->vaddr)  in vb2_dma_sg_put_userptr()
303 		vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_dma_sg_put_userptr()
304 	sg_free_table(buf->dma_sgt);  in vb2_dma_sg_put_userptr()
306 		if (buf->dma_dir == DMA_FROM_DEVICE)  in vb2_dma_sg_put_userptr()
307 			set_page_dirty_lock(buf->pages[i]);  in vb2_dma_sg_put_userptr()
309 	vb2_destroy_framevec(buf->vec);  in vb2_dma_sg_put_userptr()
310 	kfree(buf);  in vb2_dma_sg_put_userptr()
315 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_vaddr()  local
317 	BUG_ON(!buf);  in vb2_dma_sg_vaddr()
319 	if (!buf->vaddr) {  in vb2_dma_sg_vaddr()
320 		if (buf->db_attach)  in vb2_dma_sg_vaddr()
321 			buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);  in vb2_dma_sg_vaddr()
323 			buf->vaddr = vm_map_ram(buf->pages,  in vb2_dma_sg_vaddr()
324 					buf->num_pages, -1, PAGE_KERNEL);  in vb2_dma_sg_vaddr()
328 	return buf->vaddr ? buf->vaddr + buf->offset : NULL;  in vb2_dma_sg_vaddr()
333 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_num_users()  local
335 	return atomic_read(&buf->refcount);  in vb2_dma_sg_num_users()
340 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_mmap()  local
345 	if (!buf) {  in vb2_dma_sg_mmap()
353 		ret = vm_insert_page(vma, uaddr, buf->pages[i++]);  in vb2_dma_sg_mmap()
367 	vma->vm_private_data	= &buf->handler;  in vb2_dma_sg_mmap()
391 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_attach()  local
402 	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);  in vb2_dma_sg_dmabuf_ops_attach()
408 	rd = buf->dma_sgt->sgl;  in vb2_dma_sg_dmabuf_ops_attach()
496 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_kmap()  local
498 	return buf->vaddr ? buf->vaddr + pgnum * PAGE_SIZE : NULL;  in vb2_dma_sg_dmabuf_ops_kmap()
503 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_vmap()  local
505 	return vb2_dma_sg_vaddr(buf);  in vb2_dma_sg_dmabuf_ops_vmap()
528 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_get_dmabuf()  local
533 	exp_info.size = buf->size;  in vb2_dma_sg_get_dmabuf()
535 	exp_info.priv = buf;  in vb2_dma_sg_get_dmabuf()
537 	if (WARN_ON(!buf->dma_sgt))  in vb2_dma_sg_get_dmabuf()
545 	atomic_inc(&buf->refcount);  in vb2_dma_sg_get_dmabuf()
556 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_map_dmabuf()  local
559 	if (WARN_ON(!buf->db_attach)) {  in vb2_dma_sg_map_dmabuf()
564 	if (WARN_ON(buf->dma_sgt)) {  in vb2_dma_sg_map_dmabuf()
570 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);  in vb2_dma_sg_map_dmabuf()
576 	buf->dma_sgt = sgt;  in vb2_dma_sg_map_dmabuf()
577 	buf->vaddr = NULL;  in vb2_dma_sg_map_dmabuf()
584 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_unmap_dmabuf()  local
585 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_unmap_dmabuf()
587 	if (WARN_ON(!buf->db_attach)) {  in vb2_dma_sg_unmap_dmabuf()
597 	if (buf->vaddr) {  in vb2_dma_sg_unmap_dmabuf()
598 		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);  in vb2_dma_sg_unmap_dmabuf()
599 		buf->vaddr = NULL;  in vb2_dma_sg_unmap_dmabuf()
601 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dma_sg_unmap_dmabuf()
603 	buf->dma_sgt = NULL;  in vb2_dma_sg_unmap_dmabuf()
608 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_detach_dmabuf()  local
611 	if (WARN_ON(buf->dma_sgt))  in vb2_dma_sg_detach_dmabuf()
612 		vb2_dma_sg_unmap_dmabuf(buf);  in vb2_dma_sg_detach_dmabuf()
615 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);  in vb2_dma_sg_detach_dmabuf()
616 	kfree(buf);  in vb2_dma_sg_detach_dmabuf()
623 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_attach_dmabuf()  local
629 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_dma_sg_attach_dmabuf()
630 	if (!buf)  in vb2_dma_sg_attach_dmabuf()
633 	buf->dev = conf->dev;  in vb2_dma_sg_attach_dmabuf()
635 	dba = dma_buf_attach(dbuf, buf->dev);  in vb2_dma_sg_attach_dmabuf()
638 		kfree(buf);  in vb2_dma_sg_attach_dmabuf()
642 	buf->dma_dir = dma_dir;  in vb2_dma_sg_attach_dmabuf()
643 	buf->size = size;  in vb2_dma_sg_attach_dmabuf()
644 	buf->db_attach = dba;  in vb2_dma_sg_attach_dmabuf()
646 	return buf;  in vb2_dma_sg_attach_dmabuf()
651 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_cookie()  local
653 	return buf->dma_sgt;  in vb2_dma_sg_cookie()