90 #define to_usb_dmac_chan(c) container_of(c, struct usb_dmac_chan, vc.chan)
164 static u32 usb_dmac_chan_read(struct usb_dmac_chan *chan, u32 reg)  in usb_dmac_chan_read()  argument
166 	return readl(chan->iomem + reg);  in usb_dmac_chan_read()
169 static void usb_dmac_chan_write(struct usb_dmac_chan *chan, u32 reg, u32 data)  in usb_dmac_chan_write()  argument
171 	writel(data, chan->iomem + reg);  in usb_dmac_chan_write()
178 static bool usb_dmac_chan_is_busy(struct usb_dmac_chan *chan)  in usb_dmac_chan_is_busy()  argument
180 	u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR);  in usb_dmac_chan_is_busy()
196 static void usb_dmac_chan_start_sg(struct usb_dmac_chan *chan,  in usb_dmac_chan_start_sg()  argument
199 	struct usb_dmac_desc *desc = chan->desc;  in usb_dmac_chan_start_sg()
203 	WARN_ON_ONCE(usb_dmac_chan_is_busy(chan));  in usb_dmac_chan_start_sg()
210 	dev_dbg(chan->vc.chan.device->dev,  in usb_dmac_chan_start_sg()
212 		chan->index, sg, sg->size, &src_addr, &dst_addr);  in usb_dmac_chan_start_sg()
214 	usb_dmac_chan_write(chan, USB_DMASAR, src_addr & 0xffffffff);  in usb_dmac_chan_start_sg()
215 	usb_dmac_chan_write(chan, USB_DMADAR, dst_addr & 0xffffffff);  in usb_dmac_chan_start_sg()
216 	usb_dmac_chan_write(chan, USB_DMATCR,  in usb_dmac_chan_start_sg()
218 	usb_dmac_chan_write(chan, USB_DMATEND, usb_dmac_calc_tend(sg->size));  in usb_dmac_chan_start_sg()
220 	usb_dmac_chan_write(chan, USB_DMACHCR, USB_DMAC_CHCR_TS |  in usb_dmac_chan_start_sg()
225 static void usb_dmac_chan_start_desc(struct usb_dmac_chan *chan)  in usb_dmac_chan_start_desc()  argument
229 	vd = vchan_next_desc(&chan->vc);  in usb_dmac_chan_start_desc()
231 		chan->desc = NULL;  in usb_dmac_chan_start_desc()
242 	chan->desc = to_usb_dmac_desc(vd);  in usb_dmac_chan_start_desc()
243 	chan->desc->sg_index = 0;  in usb_dmac_chan_start_desc()
244 	usb_dmac_chan_start_sg(chan, 0);  in usb_dmac_chan_start_desc()
266 static int usb_dmac_desc_alloc(struct usb_dmac_chan *chan, unsigned int sg_len,  in usb_dmac_desc_alloc()  argument
279 	spin_lock_irqsave(&chan->vc.lock, flags);  in usb_dmac_desc_alloc()
280 	list_add_tail(&desc->node, &chan->desc_freed);  in usb_dmac_desc_alloc()
281 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in usb_dmac_desc_alloc()
286 static void usb_dmac_desc_free(struct usb_dmac_chan *chan)  in usb_dmac_desc_free()  argument
291 	list_splice_init(&chan->desc_freed, &list);  in usb_dmac_desc_free()
292 	list_splice_init(&chan->desc_got, &list);  in usb_dmac_desc_free()
298 	chan->descs_allocated = 0;  in usb_dmac_desc_free()
301 static struct usb_dmac_desc *usb_dmac_desc_get(struct usb_dmac_chan *chan,  in usb_dmac_desc_get()  argument
308 	spin_lock_irqsave(&chan->vc.lock, flags);  in usb_dmac_desc_get()
309 	list_for_each_entry(desc, &chan->desc_freed, node) {  in usb_dmac_desc_get()
311 			list_move_tail(&desc->node, &chan->desc_got);  in usb_dmac_desc_get()
312 			spin_unlock_irqrestore(&chan->vc.lock, flags);  in usb_dmac_desc_get()
316 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in usb_dmac_desc_get()
319 	if (!usb_dmac_desc_alloc(chan, sg_len, gfp)) {  in usb_dmac_desc_get()
321 		spin_lock_irqsave(&chan->vc.lock, flags);  in usb_dmac_desc_get()
322 		desc = list_last_entry(&chan->desc_freed, struct usb_dmac_desc,  in usb_dmac_desc_get()
324 		list_move_tail(&desc->node, &chan->desc_got);  in usb_dmac_desc_get()
325 		spin_unlock_irqrestore(&chan->vc.lock, flags);  in usb_dmac_desc_get()
332 static void usb_dmac_desc_put(struct usb_dmac_chan *chan,  in usb_dmac_desc_put()  argument
337 	spin_lock_irqsave(&chan->vc.lock, flags);  in usb_dmac_desc_put()
338 	list_move_tail(&desc->node, &chan->desc_freed);  in usb_dmac_desc_put()
339 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in usb_dmac_desc_put()
348 	struct dma_chan *chan = &uchan->vc.chan;  in usb_dmac_soft_reset()  local
349 	struct usb_dmac *dmac = to_usb_dmac(chan->device);  in usb_dmac_soft_reset()
365 static void usb_dmac_chan_halt(struct usb_dmac_chan *chan)  in usb_dmac_chan_halt()  argument
367 	u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR);  in usb_dmac_chan_halt()
370 	usb_dmac_chan_write(chan, USB_DMACHCR, chcr);  in usb_dmac_chan_halt()
372 	usb_dmac_soft_reset(chan);  in usb_dmac_chan_halt()
384 static int usb_dmac_alloc_chan_resources(struct dma_chan *chan)  in usb_dmac_alloc_chan_resources()  argument
386 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_alloc_chan_resources()
399 	return pm_runtime_get_sync(chan->device->dev);  in usb_dmac_alloc_chan_resources()
402 static void usb_dmac_free_chan_resources(struct dma_chan *chan)  in usb_dmac_free_chan_resources()  argument
404 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_free_chan_resources()
415 	pm_runtime_put(chan->device->dev);  in usb_dmac_free_chan_resources()
419 usb_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in usb_dmac_prep_slave_sg()  argument
423 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_prep_slave_sg()
429 		dev_warn(chan->device->dev,  in usb_dmac_prep_slave_sg()
448 static int usb_dmac_chan_terminate_all(struct dma_chan *chan)  in usb_dmac_chan_terminate_all()  argument
450 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_chan_terminate_all()
470 static unsigned int usb_dmac_get_current_residue(struct usb_dmac_chan *chan,  in usb_dmac_get_current_residue()  argument
483 		residue -= usb_dmac_chan_read(chan, USB_DMADAR) - mem_addr;  in usb_dmac_get_current_residue()
485 		residue -= usb_dmac_chan_read(chan, USB_DMASAR) - mem_addr;  in usb_dmac_get_current_residue()
490 static u32 usb_dmac_chan_get_residue_if_complete(struct usb_dmac_chan *chan,  in usb_dmac_chan_get_residue_if_complete()  argument
496 	list_for_each_entry_reverse(desc, &chan->desc_freed, node) {  in usb_dmac_chan_get_residue_if_complete()
506 static u32 usb_dmac_chan_get_residue(struct usb_dmac_chan *chan,  in usb_dmac_chan_get_residue()  argument
511 	struct usb_dmac_desc *desc = chan->desc;  in usb_dmac_chan_get_residue()
515 		vd = vchan_find_desc(&chan->vc, cookie);  in usb_dmac_chan_get_residue()
526 	residue += usb_dmac_get_current_residue(chan, desc, desc->sg_index);  in usb_dmac_chan_get_residue()
531 static enum dma_status usb_dmac_tx_status(struct dma_chan *chan,  in usb_dmac_tx_status()  argument
535 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_tx_status()
540 	status = dma_cookie_status(chan, cookie, txstate);  in usb_dmac_tx_status()
557 static void usb_dmac_issue_pending(struct dma_chan *chan)  in usb_dmac_issue_pending()  argument
559 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_issue_pending()
571 	struct usb_dmac_chan *chan = to_usb_dmac_chan(vd->tx.chan);  in usb_dmac_virt_desc_free()  local
573 	usb_dmac_desc_put(chan, desc);  in usb_dmac_virt_desc_free()
580 static void usb_dmac_isr_transfer_end(struct usb_dmac_chan *chan)  in usb_dmac_isr_transfer_end()  argument
582 	struct usb_dmac_desc *desc = chan->desc;  in usb_dmac_isr_transfer_end()
587 		usb_dmac_chan_start_sg(chan, desc->sg_index);  in usb_dmac_isr_transfer_end()
589 		desc->residue = usb_dmac_get_current_residue(chan, desc,  in usb_dmac_isr_transfer_end()
595 		usb_dmac_chan_start_desc(chan);  in usb_dmac_isr_transfer_end()
601 	struct usb_dmac_chan *chan = dev;  in usb_dmac_isr_channel()  local
607 	spin_lock(&chan->vc.lock);  in usb_dmac_isr_channel()
609 	chcr = usb_dmac_chan_read(chan, USB_DMACHCR);  in usb_dmac_isr_channel()
618 	usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask);  in usb_dmac_isr_channel()
621 		usb_dmac_isr_transfer_end(chan);  in usb_dmac_isr_channel()
625 	spin_unlock(&chan->vc.lock);  in usb_dmac_isr_channel()
634 static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg)  in usb_dmac_chan_filter()  argument
636 	struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);  in usb_dmac_chan_filter()
639 	if (dma_spec->np != chan->device->dev->of_node)  in usb_dmac_chan_filter()
653 	struct dma_chan *chan;  in usb_dmac_of_xlate()  local
663 	chan = dma_request_channel(mask, usb_dmac_chan_filter, dma_spec);  in usb_dmac_of_xlate()
664 	if (!chan)  in usb_dmac_of_xlate()
667 	uchan = to_usb_dmac_chan(chan);  in usb_dmac_of_xlate()
669 	return chan;  in usb_dmac_of_xlate()