110 			      struct usbhs_fifo *fifo);
112 				 struct usbhs_fifo *fifo);
113 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
121 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhs_pkt_pop()  local
135 		if (fifo)  in usbhs_pkt_pop()
136 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
139 			usbhsf_fifo_clear(pipe, fifo);  in usbhs_pkt_pop()
146 	if (fifo)  in usbhs_pkt_pop()
147 		usbhsf_fifo_unselect(pipe, fifo);  in usbhs_pkt_pop()
261 				   struct usbhs_fifo *fifo)  in usbhsf_send_terminator()  argument
265 	usbhs_bset(priv, fifo->ctr, BVAL, BVAL);  in usbhsf_send_terminator()
269 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_barrier()  argument
275 		if (usbhs_read(priv, fifo->ctr) & FRDY)  in usbhsf_fifo_barrier()
285 			      struct usbhs_fifo *fifo)  in usbhsf_fifo_clear()  argument
290 		usbhsf_fifo_barrier(priv, fifo);  in usbhsf_fifo_clear()
292 	usbhs_write(priv, fifo->ctr, BCLR);  in usbhsf_fifo_clear()
296 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_rcv_len()  argument
298 	return usbhs_read(priv, fifo->ctr) & DTLN_MASK;  in usbhsf_fifo_rcv_len()
302 				 struct usbhs_fifo *fifo)  in usbhsf_fifo_unselect()  argument
307 	usbhs_write(priv, fifo->sel, 0);  in usbhsf_fifo_unselect()
311 			      struct usbhs_fifo *fifo,  in usbhsf_fifo_select()  argument
321 	    usbhsf_fifo_is_busy(fifo))  in usbhsf_fifo_select()
332 	if (usbhs_get_dparam(priv, has_sudmac) && !usbhsf_is_cfifo(priv, fifo))  in usbhsf_fifo_select()
333 		usbhs_write(priv, fifo->sel, base);  in usbhsf_fifo_select()
335 		usbhs_write(priv, fifo->sel, base | MBW_32);  in usbhsf_fifo_select()
339 		if (base == (mask & usbhs_read(priv, fifo->sel))) {  in usbhsf_fifo_select()
340 			usbhs_pipe_select_fifo(pipe, fifo);  in usbhsf_fifo_select()
358 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_write()  local
364 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhs_dcp_dir_switch_to_write()
372 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
373 	usbhsf_send_terminator(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
375 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
387 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_read()  local
393 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhs_dcp_dir_switch_to_read()
400 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
402 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
465 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_dcp_data_stage_prepare_pop()  local
480 	usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dcp_data_stage_prepare_pop()
481 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
482 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
504 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_push()  local
505 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_push()
517 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhsf_pio_try_push()
528 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_push()
569 		usbhsf_send_terminator(pipe, fifo);  in usbhsf_pio_try_push()
579 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
584 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
616 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_prepare_pop()  local
631 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_prepare_pop()
646 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_pop()  local
647 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_pop()
655 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_pio_try_pop()
659 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_pop()
663 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_pio_try_pop()
700 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_pio_try_pop()
729 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_pop()
759 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,  in usbhsf_dma_chan_get()  argument
763 		return fifo->tx_chan;  in usbhsf_dma_chan_get()
766 		return fifo->rx_chan;  in usbhsf_dma_chan_get()
774 	struct usbhs_fifo *fifo;  in usbhsf_get_dma_fifo()  local
777 	usbhs_for_each_dfifo(priv, fifo, i) {  in usbhsf_get_dma_fifo()
778 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
779 		    !usbhsf_fifo_is_busy(fifo))  in usbhsf_get_dma_fifo()
780 			return fifo;  in usbhsf_get_dma_fifo()
789 			      struct usbhs_fifo *fifo,  in __usbhsf_dma_ctrl()  argument
794 	usbhs_bset(priv, fifo->sel, DREQE, dreqe);  in __usbhsf_dma_ctrl()
811 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in xfer_work()  local
814 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in xfer_work()
836 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in xfer_work()
839 	usbhsf_dma_start(pipe, fifo);  in xfer_work()
852 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_push()  local
880 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
881 	if (!fifo)  in usbhsf_dma_prepare_push()
887 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_push()
926 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_push_done()
928 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_push_done()
959 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_pop_with_usb_dmac()  local
970 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
971 	if (!fifo)  in usbhsf_dma_prepare_pop_with_usb_dmac()
979 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1003 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1029 	struct usbhs_fifo *fifo;  in usbhsf_dma_try_pop_with_rx_irq()  local
1039 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1040 	if (!fifo)  in usbhsf_dma_try_pop_with_rx_irq()
1046 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1051 	len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1059 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1083 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1108 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1110 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1152 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()  local
1153 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1163 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1164 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1167 	usbhsf_dma_stop(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1169 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1213 static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo)  in usbhsf_dma_quit()  argument
1215 	if (fifo->tx_chan)  in usbhsf_dma_quit()
1216 		dma_release_channel(fifo->tx_chan);  in usbhsf_dma_quit()
1217 	if (fifo->rx_chan)  in usbhsf_dma_quit()
1218 		dma_release_channel(fifo->rx_chan);  in usbhsf_dma_quit()
1220 	fifo->tx_chan = NULL;  in usbhsf_dma_quit()
1221 	fifo->rx_chan = NULL;  in usbhsf_dma_quit()
1224 static void usbhsf_dma_init_pdev(struct usbhs_fifo *fifo)  in usbhsf_dma_init_pdev()  argument
1230 	fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1231 					    &fifo->tx_slave);  in usbhsf_dma_init_pdev()
1235 	fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1236 					    &fifo->rx_slave);  in usbhsf_dma_init_pdev()
1239 static void usbhsf_dma_init_dt(struct device *dev, struct usbhs_fifo *fifo,  in usbhsf_dma_init_dt()  argument
1251 		fifo->tx_chan = dma_request_slave_channel_reason(dev, name);  in usbhsf_dma_init_dt()
1252 		if (IS_ERR(fifo->tx_chan))  in usbhsf_dma_init_dt()
1253 			fifo->tx_chan = NULL;  in usbhsf_dma_init_dt()
1255 		fifo->rx_chan = dma_request_slave_channel_reason(dev, name);  in usbhsf_dma_init_dt()
1256 		if (IS_ERR(fifo->rx_chan))  in usbhsf_dma_init_dt()
1257 			fifo->rx_chan = NULL;  in usbhsf_dma_init_dt()
1261 static void usbhsf_dma_init(struct usbhs_priv *priv, struct usbhs_fifo *fifo,  in usbhsf_dma_init()  argument
1267 		usbhsf_dma_init_dt(dev, fifo, channel);  in usbhsf_dma_init()
1269 		usbhsf_dma_init_pdev(fifo);  in usbhsf_dma_init()
1271 	if (fifo->tx_chan || fifo->rx_chan)  in usbhsf_dma_init()
1273 			 fifo->name,  in usbhsf_dma_init()
1274 			 fifo->tx_chan ? "[TX]" : "    ",  in usbhsf_dma_init()
1275 			 fifo->rx_chan ? "[RX]" : "    ");  in usbhsf_dma_init()
1357 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_fifo_clear_dcp()  local
1360 	if (usbhsf_fifo_select(pipe, fifo, 1) < 0)  in usbhs_fifo_clear_dcp()
1362 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1363 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1366 	if (usbhsf_fifo_select(pipe, fifo, 0) < 0)  in usbhs_fifo_clear_dcp()
1368 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1369 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1402 #define __USBHS_DFIFO_INIT(priv, fifo, channel, fifo_port)		\  argument
1404 	fifo = usbhsf_get_dnfifo(priv, channel);			\
1405 	fifo->name	= "D"#channel"FIFO";				\
1406 	fifo->port	= fifo_port;					\
1407 	fifo->sel	= D##channel##FIFOSEL;				\
1408 	fifo->ctr	= D##channel##FIFOCTR;				\
1409 	fifo->tx_slave.shdma_slave.slave_id =				\
1411 	fifo->rx_slave.shdma_slave.slave_id =				\
1413 	usbhsf_dma_init(priv, fifo, channel);				\
1416 #define USBHS_DFIFO_INIT(priv, fifo, channel)				\  argument
1417 		__USBHS_DFIFO_INIT(priv, fifo, channel, D##channel##FIFO)
1418 #define USBHS_DFIFO_INIT_NO_PORT(priv, fifo, channel)			\  argument
1419 		__USBHS_DFIFO_INIT(priv, fifo, channel, 0)
1423 	struct usbhs_fifo *fifo;  in usbhs_fifo_probe()  local
1426 	fifo = usbhsf_get_cfifo(priv);  in usbhs_fifo_probe()
1427 	fifo->name	= "CFIFO";  in usbhs_fifo_probe()
1428 	fifo->port	= CFIFO;  in usbhs_fifo_probe()
1429 	fifo->sel	= CFIFOSEL;  in usbhs_fifo_probe()
1430 	fifo->ctr	= CFIFOCTR;  in usbhs_fifo_probe()
1433 	USBHS_DFIFO_INIT(priv, fifo, 0);  in usbhs_fifo_probe()
1434 	USBHS_DFIFO_INIT(priv, fifo, 1);  in usbhs_fifo_probe()
1435 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 2);  in usbhs_fifo_probe()
1436 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 3);  in usbhs_fifo_probe()
1443 	struct usbhs_fifo *fifo;  in usbhs_fifo_remove()  local
1446 	usbhs_for_each_dfifo(priv, fifo, i)  in usbhs_fifo_remove()
1447 		usbhsf_dma_quit(priv, fifo);  in usbhs_fifo_remove()