31 void usbhs_pkt_init(struct usbhs_pkt *pkt)  in usbhs_pkt_init()  argument
33 	INIT_LIST_HEAD(&pkt->node);  in usbhs_pkt_init()
39 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_null_handle()  argument
41 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
54 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
56 				 struct usbhs_pkt *pkt),  in usbhs_pkt_push()  argument
76 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
83 	pkt->pipe	= pipe;  in usbhs_pkt_push()
84 	pkt->buf	= buf;  in usbhs_pkt_push()
85 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
86 	pkt->length	= len;  in usbhs_pkt_push()
87 	pkt->zero	= zero;  in usbhs_pkt_push()
88 	pkt->actual	= 0;  in usbhs_pkt_push()
89 	pkt->done	= done;  in usbhs_pkt_push()
90 	pkt->sequence	= sequence;  in usbhs_pkt_push()
96 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)  in __usbhsf_pkt_del()  argument
98 	list_del_init(&pkt->node);  in __usbhsf_pkt_del()
114 					    struct usbhs_pkt *pkt);
117 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map);
118 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
129 	if (!pkt)  in usbhs_pkt_pop()
130 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
132 	if (pkt) {  in usbhs_pkt_pop()
136 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
140 			usbhsf_dma_unmap(pkt);  in usbhs_pkt_pop()
143 		__usbhsf_pkt_del(pkt);  in usbhs_pkt_pop()
152 	return pkt;  in usbhs_pkt_pop()
164 	struct usbhs_pkt *pkt;  in usbhsf_pkt_handler()  local
166 	int (*func)(struct usbhs_pkt *pkt, int *is_done);  in usbhsf_pkt_handler()
174 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
175 	if (!pkt)  in usbhsf_pkt_handler()
180 		func = pkt->handler->prepare;  in usbhsf_pkt_handler()
183 		func = pkt->handler->try_run;  in usbhsf_pkt_handler()
186 		func = pkt->handler->dma_done;  in usbhsf_pkt_handler()
194 		ret = func(pkt, &is_done);  in usbhsf_pkt_handler()
197 		__usbhsf_pkt_del(pkt);  in usbhsf_pkt_handler()
204 		pkt->done(priv, pkt);  in usbhsf_pkt_handler()
354 static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_write()  argument
356 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()
383 static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_read()  argument
385 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()
411 static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_done()  argument
413 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()
415 	if (pkt->handler == &usbhs_dcp_status_stage_in_handler)  in usbhs_dcp_dir_switch_done()
420 	pkt->actual = pkt->length;  in usbhs_dcp_dir_switch_done()
439 static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dcp_data_stage_try_push()  argument
441 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()
448 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dcp_data_stage_try_push()
450 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_try_push()
460 static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt,  in usbhsf_dcp_data_stage_prepare_pop()  argument
463 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()
487 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dcp_data_stage_prepare_pop()
489 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_prepare_pop()
499 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_push()  argument
501 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()
512 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
513 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_pio_try_push()
515 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
532 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_push()
533 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_push()
556 	pkt->actual += total_len;  in usbhsf_pio_try_push()
558 	if (pkt->actual < pkt->length)  in usbhsf_pio_try_push()
563 		*is_done = !pkt->zero;	/* send zero packet ? */  in usbhsf_pio_try_push()
577 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_push()
596 static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_prepare_push()  argument
598 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
601 	return usbhsf_pio_try_push(pkt, is_done);  in usbhsf_pio_prepare_push()
612 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_prepare_pop()  argument
614 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()
627 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
628 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_prepare_pop()
633 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
641 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_pop()  argument
643 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()
665 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_pop()
666 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_pop()
675 	pkt->actual += total_len;  in usbhsf_pio_try_pop()
677 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_pio_try_pop()
699 		pkt->zero = 1;  in usbhsf_pio_try_pop()
726 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_pop()
742 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_ctrl_stage_end()  argument
744 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
760 					    struct usbhs_pkt *pkt)  in usbhsf_dma_chan_get()  argument
762 	if (&usbhs_fifo_dma_push_handler == pkt->handler)  in usbhsf_dma_chan_get()
765 	if (&usbhs_fifo_dma_pop_handler == pkt->handler)  in usbhsf_dma_chan_get()
772 					      struct usbhs_pkt *pkt)  in usbhsf_get_dma_fifo()  argument
778 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
797 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)  in __usbhsf_dma_map_ctrl()  argument
799 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()
803 	return info->dma_map_ctrl(pkt, map);  in __usbhsf_dma_map_ctrl()
809 	struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work);  in xfer_work()  local
810 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()
814 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in xfer_work()
820 	desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual,  in xfer_work()
821 					pkt->trans, dir,  in xfer_work()
829 	pkt->cookie = dmaengine_submit(desc);  in xfer_work()
830 	if (pkt->cookie < 0) {  in xfer_work()
836 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in xfer_work()
840 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in xfer_work()
848 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_push()  argument
850 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()
853 	int len = pkt->length - pkt->actual;  in usbhsf_dma_prepare_push()
872 	if ((uintptr_t)(pkt->buf + pkt->actual) & align_mask)  in usbhsf_dma_prepare_push()
880 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
884 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_push()
891 	pkt->trans = len;  in usbhsf_dma_prepare_push()
894 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_push()
895 	schedule_work(&pkt->work);  in usbhsf_dma_prepare_push()
900 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_prepare_push()
905 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_prepare_push()
907 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_push()
910 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_push_done()  argument
912 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()
913 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
915 	pkt->actual += pkt->trans;  in usbhsf_dma_push_done()
917 	if (pkt->actual < pkt->length)  in usbhsf_dma_push_done()
922 		*is_done = !pkt->zero;	/* send zero packet? */  in usbhsf_dma_push_done()
927 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_push_done()
932 		pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_push_done()
933 		return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_push_done()
948 static int usbhsf_dma_prepare_pop_with_rx_irq(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_rx_irq()  argument
951 	return usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_prepare_pop_with_rx_irq()
954 static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_usb_dmac()  argument
957 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()
966 	if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||  in usbhsf_dma_prepare_pop_with_usb_dmac()
970 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
974 	if ((uintptr_t)pkt->buf & (USBHS_USB_DMAC_XFER_SIZE - 1))  in usbhsf_dma_prepare_pop_with_usb_dmac()
983 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_pop_with_usb_dmac()
995 	pkt->trans = pkt->length;  in usbhsf_dma_prepare_pop_with_usb_dmac()
997 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_pop_with_usb_dmac()
998 	schedule_work(&pkt->work);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1009 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1012 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1015 static int usbhsf_dma_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_pop()  argument
1017 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1020 		return usbhsf_dma_prepare_pop_with_usb_dmac(pkt, is_done);  in usbhsf_dma_prepare_pop()
1022 		return usbhsf_dma_prepare_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_prepare_pop()
1025 static int usbhsf_dma_try_pop_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop_with_rx_irq()  argument
1027 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()
1039 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1043 	if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */  in usbhsf_dma_try_pop_with_rx_irq()
1052 	len = min(pkt->length - pkt->actual, len);  in usbhsf_dma_try_pop_with_rx_irq()
1063 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_try_pop_with_rx_irq()
1075 	pkt->trans = len;  in usbhsf_dma_try_pop_with_rx_irq()
1077 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_try_pop_with_rx_irq()
1078 	schedule_work(&pkt->work);  in usbhsf_dma_try_pop_with_rx_irq()
1089 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_try_pop_with_rx_irq()
1091 	return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_try_pop_with_rx_irq()
1094 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop()  argument
1096 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1100 	return usbhsf_dma_try_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_try_pop()
1103 static int usbhsf_dma_pop_done_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done_with_rx_irq()  argument
1105 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()
1109 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_rx_irq()
1112 	pkt->actual += pkt->trans;  in usbhsf_dma_pop_done_with_rx_irq()
1114 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_dma_pop_done_with_rx_irq()
1115 	    (pkt->trans < maxp)) {		/* short packet */  in usbhsf_dma_pop_done_with_rx_irq()
1121 		usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_pop_done_with_rx_irq()
1127 static size_t usbhs_dma_calc_received_size(struct usbhs_pkt *pkt,  in usbhs_dma_calc_received_size()  argument
1130 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()
1135 	dmaengine_tx_status(chan, pkt->cookie, &state);  in usbhs_dma_calc_received_size()
1136 	received_size = pkt->length - state.residue;  in usbhs_dma_calc_received_size()
1147 static int usbhsf_dma_pop_done_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_pop_done_with_usb_dmac()  argument
1150 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()
1153 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1165 	pkt->actual = usbhs_dma_calc_received_size(pkt, chan, rcv_len);  in usbhsf_dma_pop_done_with_usb_dmac()
1168 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1177 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done()  argument
1179 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1182 		return usbhsf_dma_pop_done_with_usb_dmac(pkt, is_done);  in usbhsf_dma_pop_done()
1184 		return usbhsf_dma_pop_done_with_rx_irq(pkt, is_done);  in usbhsf_dma_pop_done()