61 static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,  in vb2_dma_sg_alloc_compacted()  argument
65 	int size = buf->size;  in vb2_dma_sg_alloc_compacted()
86 					__free_page(buf->pages[last_page]);  in vb2_dma_sg_alloc_compacted()
94 			buf->pages[last_page++] = &pages[i];  in vb2_dma_sg_alloc_compacted()
106 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_alloc()  local
116 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dma_sg_alloc()
117 	if (!buf)  in vb2_dma_sg_alloc()
120 	buf->vaddr = NULL;  in vb2_dma_sg_alloc()
121 	buf->dma_dir = dma_dir;  in vb2_dma_sg_alloc()
122 	buf->offset = 0;  in vb2_dma_sg_alloc()
123 	buf->size = size;  in vb2_dma_sg_alloc()
125 	buf->num_pages = size >> PAGE_SHIFT;  in vb2_dma_sg_alloc()
126 	buf->dma_sgt = &buf->sg_table;  in vb2_dma_sg_alloc()
128 	buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),  in vb2_dma_sg_alloc()
130 	if (!buf->pages)  in vb2_dma_sg_alloc()
133 	ret = vb2_dma_sg_alloc_compacted(buf, gfp_flags);  in vb2_dma_sg_alloc()
137 	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_dma_sg_alloc()
138 			buf->num_pages, 0, size, GFP_KERNEL);  in vb2_dma_sg_alloc()
143 	buf->dev = get_device(conf->dev);  in vb2_dma_sg_alloc()
145 	sgt = &buf->sg_table;  in vb2_dma_sg_alloc()
150 	if (dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->nents,  in vb2_dma_sg_alloc()
151 			     buf->dma_dir, &attrs) == 0)  in vb2_dma_sg_alloc()
154 	buf->handler.refcount = &buf->refcount;  in vb2_dma_sg_alloc()
155 	buf->handler.put = vb2_dma_sg_put;  in vb2_dma_sg_alloc()
156 	buf->handler.arg = buf;  in vb2_dma_sg_alloc()
158 	atomic_inc(&buf->refcount);  in vb2_dma_sg_alloc()
161 		__func__, buf->num_pages);  in vb2_dma_sg_alloc()
162 	return buf;  in vb2_dma_sg_alloc()
165 	put_device(buf->dev);  in vb2_dma_sg_alloc()
166 	sg_free_table(buf->dma_sgt);  in vb2_dma_sg_alloc()
168 	num_pages = buf->num_pages;  in vb2_dma_sg_alloc()
170 		__free_page(buf->pages[num_pages]);  in vb2_dma_sg_alloc()
172 	kfree(buf->pages);  in vb2_dma_sg_alloc()
174 	kfree(buf);  in vb2_dma_sg_alloc()
180 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_put()  local
181 	struct sg_table *sgt = &buf->sg_table;  in vb2_dma_sg_put()
182 	int i = buf->num_pages;  in vb2_dma_sg_put()
184 	if (atomic_dec_and_test(&buf->refcount)) {  in vb2_dma_sg_put()
189 			buf->num_pages);  in vb2_dma_sg_put()
190 		dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->nents,  in vb2_dma_sg_put()
191 				   buf->dma_dir, &attrs);  in vb2_dma_sg_put()
192 		if (buf->vaddr)  in vb2_dma_sg_put()
193 			vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_dma_sg_put()
194 		sg_free_table(buf->dma_sgt);  in vb2_dma_sg_put()
196 			__free_page(buf->pages[i]);  in vb2_dma_sg_put()
197 		kfree(buf->pages);  in vb2_dma_sg_put()
198 		put_device(buf->dev);  in vb2_dma_sg_put()
199 		kfree(buf);  in vb2_dma_sg_put()
205 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_prepare()  local
206 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_prepare()
209 	if (buf->db_attach)  in vb2_dma_sg_prepare()
212 	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);  in vb2_dma_sg_prepare()
217 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_finish()  local
218 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_finish()
221 	if (buf->db_attach)  in vb2_dma_sg_finish()
224 	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);  in vb2_dma_sg_finish()
237 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_get_userptr()  local
246 	buf = kzalloc(sizeof *buf, GFP_KERNEL);  in vb2_dma_sg_get_userptr()
247 	if (!buf)  in vb2_dma_sg_get_userptr()
250 	buf->vaddr = NULL;  in vb2_dma_sg_get_userptr()
251 	buf->dev = conf->dev;  in vb2_dma_sg_get_userptr()
252 	buf->dma_dir = dma_dir;  in vb2_dma_sg_get_userptr()
253 	buf->offset = vaddr & ~PAGE_MASK;  in vb2_dma_sg_get_userptr()
254 	buf->size = size;  in vb2_dma_sg_get_userptr()
255 	buf->dma_sgt = &buf->sg_table;  in vb2_dma_sg_get_userptr()
259 	buf->num_pages = last - first + 1;  in vb2_dma_sg_get_userptr()
261 	buf->pages = kzalloc(buf->num_pages * sizeof(struct page *),  in vb2_dma_sg_get_userptr()
263 	if (!buf->pages)  in vb2_dma_sg_get_userptr()
278 	buf->vma = vb2_get_vma(vma);  in vb2_dma_sg_get_userptr()
279 	if (!buf->vma) {  in vb2_dma_sg_get_userptr()
284 	if (vma_is_io(buf->vma)) {  in vb2_dma_sg_get_userptr()
286 		     num_pages_from_user < buf->num_pages;  in vb2_dma_sg_get_userptr()
294 			buf->pages[num_pages_from_user] = pfn_to_page(pfn);  in vb2_dma_sg_get_userptr()
299 					     buf->num_pages,  in vb2_dma_sg_get_userptr()
300 					     buf->dma_dir == DMA_FROM_DEVICE,  in vb2_dma_sg_get_userptr()
302 					     buf->pages,  in vb2_dma_sg_get_userptr()
305 	if (num_pages_from_user != buf->num_pages)  in vb2_dma_sg_get_userptr()
308 	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_dma_sg_get_userptr()
309 			buf->num_pages, buf->offset, size, 0))  in vb2_dma_sg_get_userptr()
312 	sgt = &buf->sg_table;  in vb2_dma_sg_get_userptr()
317 	if (dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->nents,  in vb2_dma_sg_get_userptr()
318 			     buf->dma_dir, &attrs) == 0)  in vb2_dma_sg_get_userptr()
320 	return buf;  in vb2_dma_sg_get_userptr()
323 	sg_free_table(&buf->sg_table);  in vb2_dma_sg_get_userptr()
327 		buf->num_pages, num_pages_from_user);  in vb2_dma_sg_get_userptr()
328 	if (!vma_is_io(buf->vma))  in vb2_dma_sg_get_userptr()
330 			put_page(buf->pages[num_pages_from_user]);  in vb2_dma_sg_get_userptr()
331 	vb2_put_vma(buf->vma);  in vb2_dma_sg_get_userptr()
333 	kfree(buf->pages);  in vb2_dma_sg_get_userptr()
335 	kfree(buf);  in vb2_dma_sg_get_userptr()
345 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_put_userptr()  local
346 	struct sg_table *sgt = &buf->sg_table;  in vb2_dma_sg_put_userptr()
347 	int i = buf->num_pages;  in vb2_dma_sg_put_userptr()
353 	       __func__, buf->num_pages);  in vb2_dma_sg_put_userptr()
354 	dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir, &attrs);  in vb2_dma_sg_put_userptr()
355 	if (buf->vaddr)  in vb2_dma_sg_put_userptr()
356 		vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_dma_sg_put_userptr()
357 	sg_free_table(buf->dma_sgt);  in vb2_dma_sg_put_userptr()
359 		if (buf->dma_dir == DMA_FROM_DEVICE)  in vb2_dma_sg_put_userptr()
360 			set_page_dirty_lock(buf->pages[i]);  in vb2_dma_sg_put_userptr()
361 		if (!vma_is_io(buf->vma))  in vb2_dma_sg_put_userptr()
362 			put_page(buf->pages[i]);  in vb2_dma_sg_put_userptr()
364 	kfree(buf->pages);  in vb2_dma_sg_put_userptr()
365 	vb2_put_vma(buf->vma);  in vb2_dma_sg_put_userptr()
366 	kfree(buf);  in vb2_dma_sg_put_userptr()
371 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_vaddr()  local
373 	BUG_ON(!buf);  in vb2_dma_sg_vaddr()
375 	if (!buf->vaddr) {  in vb2_dma_sg_vaddr()
376 		if (buf->db_attach)  in vb2_dma_sg_vaddr()
377 			buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);  in vb2_dma_sg_vaddr()
379 			buf->vaddr = vm_map_ram(buf->pages,  in vb2_dma_sg_vaddr()
380 					buf->num_pages, -1, PAGE_KERNEL);  in vb2_dma_sg_vaddr()
384 	return buf->vaddr ? buf->vaddr + buf->offset : NULL;  in vb2_dma_sg_vaddr()
389 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_num_users()  local
391 	return atomic_read(&buf->refcount);  in vb2_dma_sg_num_users()
396 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_mmap()  local
401 	if (!buf) {  in vb2_dma_sg_mmap()
409 		ret = vm_insert_page(vma, uaddr, buf->pages[i++]);  in vb2_dma_sg_mmap()
423 	vma->vm_private_data	= &buf->handler;  in vb2_dma_sg_mmap()
447 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_attach()  local
458 	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);  in vb2_dma_sg_dmabuf_ops_attach()
464 	rd = buf->dma_sgt->sgl;  in vb2_dma_sg_dmabuf_ops_attach()
552 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_kmap()  local
554 	return buf->vaddr ? buf->vaddr + pgnum * PAGE_SIZE : NULL;  in vb2_dma_sg_dmabuf_ops_kmap()
559 	struct vb2_dma_sg_buf *buf = dbuf->priv;  in vb2_dma_sg_dmabuf_ops_vmap()  local
561 	return vb2_dma_sg_vaddr(buf);  in vb2_dma_sg_dmabuf_ops_vmap()
584 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_get_dmabuf()  local
589 	exp_info.size = buf->size;  in vb2_dma_sg_get_dmabuf()
591 	exp_info.priv = buf;  in vb2_dma_sg_get_dmabuf()
593 	if (WARN_ON(!buf->dma_sgt))  in vb2_dma_sg_get_dmabuf()
601 	atomic_inc(&buf->refcount);  in vb2_dma_sg_get_dmabuf()
612 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_map_dmabuf()  local
615 	if (WARN_ON(!buf->db_attach)) {  in vb2_dma_sg_map_dmabuf()
620 	if (WARN_ON(buf->dma_sgt)) {  in vb2_dma_sg_map_dmabuf()
626 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);  in vb2_dma_sg_map_dmabuf()
632 	buf->dma_sgt = sgt;  in vb2_dma_sg_map_dmabuf()
633 	buf->vaddr = NULL;  in vb2_dma_sg_map_dmabuf()
640 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_unmap_dmabuf()  local
641 	struct sg_table *sgt = buf->dma_sgt;  in vb2_dma_sg_unmap_dmabuf()
643 	if (WARN_ON(!buf->db_attach)) {  in vb2_dma_sg_unmap_dmabuf()
653 	if (buf->vaddr) {  in vb2_dma_sg_unmap_dmabuf()
654 		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);  in vb2_dma_sg_unmap_dmabuf()
655 		buf->vaddr = NULL;  in vb2_dma_sg_unmap_dmabuf()
657 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_dma_sg_unmap_dmabuf()
659 	buf->dma_sgt = NULL;  in vb2_dma_sg_unmap_dmabuf()
664 	struct vb2_dma_sg_buf *buf = mem_priv;  in vb2_dma_sg_detach_dmabuf()  local
667 	if (WARN_ON(buf->dma_sgt))  in vb2_dma_sg_detach_dmabuf()
668 		vb2_dma_sg_unmap_dmabuf(buf);  in vb2_dma_sg_detach_dmabuf()
671 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);  in vb2_dma_sg_detach_dmabuf()
672 	kfree(buf);  in vb2_dma_sg_detach_dmabuf()
679 	struct vb2_dma_sg_buf *buf;  in vb2_dma_sg_attach_dmabuf()  local
685 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_dma_sg_attach_dmabuf()
686 	if (!buf)  in vb2_dma_sg_attach_dmabuf()
689 	buf->dev = conf->dev;  in vb2_dma_sg_attach_dmabuf()
691 	dba = dma_buf_attach(dbuf, buf->dev);  in vb2_dma_sg_attach_dmabuf()
694 		kfree(buf);  in vb2_dma_sg_attach_dmabuf()
698 	buf->dma_dir = dma_dir;  in vb2_dma_sg_attach_dmabuf()
699 	buf->size = size;  in vb2_dma_sg_attach_dmabuf()
700 	buf->db_attach = dba;  in vb2_dma_sg_attach_dmabuf()
702 	return buf;  in vb2_dma_sg_attach_dmabuf()
707 	struct vb2_dma_sg_buf *buf = buf_priv;  in vb2_dma_sg_cookie()  local
709 	return buf->dma_sgt;  in vb2_dma_sg_cookie()