26 #define usbhsf_fifo_is_busy(f)	((f)->pipe) /* see usbhs_pipe_select_fifo */
41 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
54 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
59 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_pkt_push()
71 	if (!pipe->handler) {  in usbhs_pkt_push()
73 		pipe->handler = &usbhsf_null_handler;  in usbhs_pkt_push()
76 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
83 	pkt->pipe	= pipe;  in usbhs_pkt_push()
85 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
101 static struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe)  in __usbhsf_pkt_get()  argument
103 	if (list_empty(&pipe->list))  in __usbhsf_pkt_get()
106 	return list_first_entry(&pipe->list, struct usbhs_pkt, node);  in __usbhsf_pkt_get()
109 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,
111 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,
118 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
120 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_pkt_pop()
121 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhs_pkt_pop()
127 	usbhs_pipe_disable(pipe);  in usbhs_pkt_pop()
130 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
139 			usbhsf_fifo_clear(pipe, fifo);  in usbhs_pkt_pop()
147 		usbhsf_fifo_unselect(pipe, fifo);  in usbhs_pkt_pop()
161 static int usbhsf_pkt_handler(struct usbhs_pipe *pipe, int type)  in usbhsf_pkt_handler()  argument
163 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pkt_handler()
174 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
205 		usbhs_pkt_start(pipe);  in usbhsf_pkt_handler()
211 void usbhs_pkt_start(struct usbhs_pipe *pipe)  in usbhs_pkt_start()  argument
213 	usbhsf_pkt_handler(pipe, USBHSF_PKT_PREPARE);  in usbhs_pkt_start()
221 #define usbhsf_irq_callback_ctrl(pipe, status, enable)			\  argument
223 		struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);	\
225 		u16 status = (1 << usbhs_pipe_number(pipe));		\
235 static void usbhsf_tx_irq_ctrl(struct usbhs_pipe *pipe, int enable)  in usbhsf_tx_irq_ctrl()  argument
246 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_tx_irq_ctrl()
247 		usbhsf_irq_empty_ctrl(pipe, enable);  in usbhsf_tx_irq_ctrl()
249 		usbhsf_irq_ready_ctrl(pipe, enable);  in usbhsf_tx_irq_ctrl()
252 static void usbhsf_rx_irq_ctrl(struct usbhs_pipe *pipe, int enable)  in usbhsf_rx_irq_ctrl()  argument
254 	usbhsf_irq_ready_ctrl(pipe, enable);  in usbhsf_rx_irq_ctrl()
260 static void usbhsf_send_terminator(struct usbhs_pipe *pipe,  in usbhsf_send_terminator()  argument
263 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_send_terminator()
284 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,  in usbhsf_fifo_clear()  argument
287 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_clear()
289 	if (!usbhs_pipe_is_dcp(pipe))  in usbhsf_fifo_clear()
301 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,  in usbhsf_fifo_unselect()  argument
304 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_unselect()
306 	usbhs_pipe_select_fifo(pipe, NULL);  in usbhsf_fifo_unselect()
310 static int usbhsf_fifo_select(struct usbhs_pipe *pipe,  in usbhsf_fifo_select()  argument
314 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_select()
318 	u16 base = usbhs_pipe_number(pipe);	/* CURPIPE */  in usbhsf_fifo_select()
320 	if (usbhs_pipe_is_busy(pipe) ||  in usbhsf_fifo_select()
324 	if (usbhs_pipe_is_dcp(pipe)) {  in usbhsf_fifo_select()
328 			usbhs_dcp_dir_for_host(pipe, write);  in usbhsf_fifo_select()
340 			usbhs_pipe_select_fifo(pipe, fifo);  in usbhsf_fifo_select()
356 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()  local
357 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_dcp_dir_switch_to_write()
362 	usbhs_pipe_disable(pipe);  in usbhs_dcp_dir_switch_to_write()
364 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhs_dcp_dir_switch_to_write()
370 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhs_dcp_dir_switch_to_write()
372 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
373 	usbhsf_send_terminator(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
375 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
377 	usbhsf_tx_irq_ctrl(pipe, 1);  in usbhs_dcp_dir_switch_to_write()
378 	usbhs_pipe_enable(pipe);  in usbhs_dcp_dir_switch_to_write()
385 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()  local
386 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_dcp_dir_switch_to_read()
391 	usbhs_pipe_disable(pipe);  in usbhs_dcp_dir_switch_to_read()
393 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhs_dcp_dir_switch_to_read()
399 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhs_dcp_dir_switch_to_read()
400 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
402 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
404 	usbhsf_rx_irq_ctrl(pipe, 1);  in usbhs_dcp_dir_switch_to_read()
405 	usbhs_pipe_enable(pipe);  in usbhs_dcp_dir_switch_to_read()
413 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()  local
416 		usbhsf_tx_irq_ctrl(pipe, 0);  in usbhs_dcp_dir_switch_done()
418 		usbhsf_rx_irq_ctrl(pipe, 0);  in usbhs_dcp_dir_switch_done()
441 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()  local
443 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhsf_dcp_data_stage_try_push()
463 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()  local
464 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dcp_data_stage_prepare_pop()
467 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dcp_data_stage_prepare_pop()
476 	usbhs_pipe_disable(pipe);  in usbhsf_dcp_data_stage_prepare_pop()
478 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhsf_dcp_data_stage_prepare_pop()
480 	usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dcp_data_stage_prepare_pop()
481 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
482 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
501 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()  local
502 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pio_try_push()
507 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_pio_try_push()
512 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
515 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
517 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhsf_pio_try_push()
521 	ret = usbhs_pipe_is_accessible(pipe);  in usbhsf_pio_try_push()
569 		usbhsf_send_terminator(pipe, fifo);  in usbhsf_pio_try_push()
571 	usbhsf_tx_irq_ctrl(pipe, !*is_done);  in usbhsf_pio_try_push()
572 	usbhs_pipe_running(pipe, !*is_done);  in usbhsf_pio_try_push()
573 	usbhs_pipe_enable(pipe);  in usbhsf_pio_try_push()
576 		usbhs_pipe_number(pipe),  in usbhsf_pio_try_push()
579 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
584 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
590 	usbhsf_tx_irq_ctrl(pipe, 1);  in usbhsf_pio_try_push()
591 	usbhs_pipe_running(pipe, 1);  in usbhsf_pio_try_push()
598 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
614 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()  local
615 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_prepare_pop()
618 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_prepare_pop()
621 	if (usbhs_pipe_is_running(pipe))  in usbhsf_prepare_pop()
627 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
630 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_prepare_pop()
631 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_prepare_pop()
633 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
634 	usbhs_pipe_enable(pipe);  in usbhsf_prepare_pop()
635 	usbhs_pipe_running(pipe, 1);  in usbhsf_prepare_pop()
636 	usbhsf_rx_irq_ctrl(pipe, 1);  in usbhsf_prepare_pop()
643 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()  local
644 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pio_try_pop()
650 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_pio_try_pop()
655 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_pio_try_pop()
680 		usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_pio_try_pop()
681 		usbhs_pipe_running(pipe, 0);  in usbhsf_pio_try_pop()
688 		if (!usbhs_mod_is_host(priv) && !usbhs_pipe_is_dcp(pipe))  in usbhsf_pio_try_pop()
689 			usbhs_pipe_disable(pipe);	/* disable pipe first */  in usbhsf_pio_try_pop()
700 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_pio_try_pop()
725 		usbhs_pipe_number(pipe),  in usbhsf_pio_try_pop()
729 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_pop()
744 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
788 static void __usbhsf_dma_ctrl(struct usbhs_pipe *pipe,  in __usbhsf_dma_ctrl()  argument
792 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in __usbhsf_dma_ctrl()
799 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()  local
800 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in __usbhsf_dma_map_ctrl()
810 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()  local
811 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in xfer_work()
812 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in xfer_work()
818 	dir = usbhs_pipe_is_dir_in(pipe) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;  in xfer_work()
827 	desc->callback_param	= pipe;  in xfer_work()
836 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in xfer_work()
838 	usbhs_pipe_running(pipe, 1);  in xfer_work()
839 	usbhsf_dma_start(pipe, fifo);  in xfer_work()
840 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in xfer_work()
842 	usbhs_pipe_enable(pipe);  in xfer_work()
850 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()  local
851 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_prepare_push()
857 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_prepare_push()
862 	    usbhs_pipe_is_dcp(pipe))  in usbhsf_dma_prepare_push()
876 	if (usbhs_pipe_is_running(pipe))  in usbhsf_dma_prepare_push()
887 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_push()
893 	usbhsf_tx_irq_ctrl(pipe, 0);  in usbhsf_dma_prepare_push()
912 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()  local
913 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
924 	usbhs_pipe_running(pipe, !*is_done);  in usbhsf_dma_push_done()
926 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_push_done()
928 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_push_done()
957 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()  local
958 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_prepare_pop_with_usb_dmac()
962 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_prepare_pop_with_usb_dmac()
967 	    usbhs_pipe_is_dcp(pipe))  in usbhsf_dma_prepare_pop_with_usb_dmac()
977 	usbhs_pipe_config_change_bfre(pipe, 1);  in usbhsf_dma_prepare_pop_with_usb_dmac()
979 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
993 	usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1003 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1010 	usbhs_pipe_config_change_bfre(pipe, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1017 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1027 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()  local
1028 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_try_pop_with_rx_irq()
1032 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_try_pop_with_rx_irq()
1035 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_dma_try_pop_with_rx_irq()
1046 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1073 	usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1083 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1096 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1105 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()  local
1106 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_pop_done_with_rx_irq()
1108 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1110 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1117 		usbhs_pipe_running(pipe, 0);  in usbhsf_dma_pop_done_with_rx_irq()
1120 		usbhs_pipe_running(pipe, 0);  in usbhsf_dma_pop_done_with_rx_irq()
1130 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()  local
1133 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhs_dma_calc_received_size()
1150 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()  local
1151 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()
1152 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()
1161 	usbhs_xxxsts_clear(priv, BRDYSTS, usbhs_pipe_number(pipe));  in usbhsf_dma_pop_done_with_usb_dmac()
1164 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1167 	usbhsf_dma_stop(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1169 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1179 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1284 	struct usbhs_pipe *pipe;  in usbhsf_irq_empty()  local
1299 	usbhs_for_each_pipe_with_dcp(pipe, priv, i) {  in usbhsf_irq_empty()
1303 		ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);  in usbhsf_irq_empty()
1314 	struct usbhs_pipe *pipe;  in usbhsf_irq_ready()  local
1329 	usbhs_for_each_pipe_with_dcp(pipe, priv, i) {  in usbhsf_irq_ready()
1333 		ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);  in usbhsf_irq_ready()
1343 	struct usbhs_pipe *pipe = arg;  in usbhsf_dma_complete()  local
1344 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_complete()
1348 	ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_DMA_DONE);  in usbhsf_dma_complete()
1351 			usbhs_pipe_number(pipe), ret);  in usbhsf_dma_complete()
1354 void usbhs_fifo_clear_dcp(struct usbhs_pipe *pipe)  in usbhs_fifo_clear_dcp()  argument
1356 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_fifo_clear_dcp()
1360 	if (usbhsf_fifo_select(pipe, fifo, 1) < 0)  in usbhs_fifo_clear_dcp()
1362 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1363 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1366 	if (usbhsf_fifo_select(pipe, fifo, 0) < 0)  in usbhs_fifo_clear_dcp()
1368 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1369 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1387 	cfifo->pipe	= NULL;  in usbhs_fifo_init()
1389 		dfifo->pipe	= NULL;  in usbhs_fifo_init()