59 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)  in save_rx_toggle()  argument
64 	if (cppi41_channel->is_tx)  in save_rx_toggle()
66 	if (!is_host_active(cppi41_channel->controller->musb))  in save_rx_toggle()
69 	csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);  in save_rx_toggle()
72 	cppi41_channel->usb_toggle = toggle;  in save_rx_toggle()
75 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)  in update_rx_toggle()  argument
77 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in update_rx_toggle()
82 	if (cppi41_channel->is_tx)  in update_rx_toggle()
96 	if (!toggle && toggle == cppi41_channel->usb_toggle) {  in update_rx_toggle()
98 		musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);  in update_rx_toggle()
99 		dev_dbg(cppi41_channel->controller->musb->controller,  in update_rx_toggle()
103 	cppi41_channel->usb_toggle = toggle;  in update_rx_toggle()
122 static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)  in cppi41_trans_done()  argument
124 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_trans_done()
129 	if (!cppi41_channel->prog_len ||  in cppi41_trans_done()
130 	    (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) {  in cppi41_trans_done()
133 		cppi41_channel->channel.actual_len =  in cppi41_trans_done()
134 			cppi41_channel->transferred;  in cppi41_trans_done()
135 		cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;  in cppi41_trans_done()
136 		cppi41_channel->channel.rx_packet_done = true;  in cppi41_trans_done()
142 		if (cppi41_channel->tx_zlp && (cppi41_channel->transferred %  in cppi41_trans_done()
143 					cppi41_channel->packet_sz) == 0) {  in cppi41_trans_done()
148 		musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);  in cppi41_trans_done()
151 		struct dma_chan *dc = cppi41_channel->dc;  in cppi41_trans_done()
156 		cppi41_channel->buf_addr += cppi41_channel->packet_sz;  in cppi41_trans_done()
158 		remain_bytes = cppi41_channel->total_len;  in cppi41_trans_done()
159 		remain_bytes -= cppi41_channel->transferred;  in cppi41_trans_done()
160 		remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);  in cppi41_trans_done()
161 		cppi41_channel->prog_len = remain_bytes;  in cppi41_trans_done()
163 		direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV  in cppi41_trans_done()
166 				cppi41_channel->buf_addr,  in cppi41_trans_done()
174 		dma_desc->callback_param = &cppi41_channel->channel;  in cppi41_trans_done()
175 		cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);  in cppi41_trans_done()
178 		if (!cppi41_channel->is_tx) {  in cppi41_trans_done()
190 	struct cppi41_dma_channel *cppi41_channel, *n;  in cppi41_recheck_tx_req()  local
200 	list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,  in cppi41_recheck_tx_req()
203 		struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_recheck_tx_req()
207 			list_del_init(&cppi41_channel->tx_check);  in cppi41_recheck_tx_req()
208 			cppi41_trans_done(cppi41_channel);  in cppi41_recheck_tx_req()
226 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_callback()  local
227 	struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;  in cppi41_dma_callback()
238 	dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,  in cppi41_dma_callback()
240 	transferred = cppi41_channel->prog_len - txstate.residue;  in cppi41_dma_callback()
241 	cppi41_channel->transferred += transferred;  in cppi41_dma_callback()
244 		hw_ep->epnum, cppi41_channel->transferred,  in cppi41_dma_callback()
245 		cppi41_channel->total_len);  in cppi41_dma_callback()
247 	update_rx_toggle(cppi41_channel);  in cppi41_dma_callback()
249 	if (cppi41_channel->transferred == cppi41_channel->total_len ||  in cppi41_dma_callback()
250 			transferred < cppi41_channel->packet_sz)  in cppi41_dma_callback()
251 		cppi41_channel->prog_len = 0;  in cppi41_dma_callback()
253 	if (cppi41_channel->is_tx)  in cppi41_dma_callback()
256 	if (!cppi41_channel->is_tx || empty) {  in cppi41_dma_callback()
257 		cppi41_trans_done(cppi41_channel);  in cppi41_dma_callback()
271 	controller = cppi41_channel->controller;  in cppi41_dma_callback()
286 				cppi41_trans_done(cppi41_channel);  in cppi41_dma_callback()
295 	list_add_tail(&cppi41_channel->tx_check,  in cppi41_dma_callback()
298 		unsigned long usecs = cppi41_channel->total_len / 10;  in cppi41_dma_callback()
320 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,  in cppi41_set_dma_mode()  argument
323 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_set_dma_mode()
328 	if (cppi41_channel->is_tx)  in cppi41_set_dma_mode()
332 	port = cppi41_channel->port_num;  in cppi41_set_dma_mode()
337 	if (cppi41_channel->is_tx) {  in cppi41_set_dma_mode()
348 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,  in cppi41_set_autoreq_mode()  argument
351 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_set_autoreq_mode()
357 	port = cppi41_channel->port_num;  in cppi41_set_autoreq_mode()
370 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_configure_channel()  local
371 	struct dma_chan *dc = cppi41_channel->dc;  in cppi41_configure_channel()
374 	struct musb *musb = cppi41_channel->controller->musb;  in cppi41_configure_channel()
379 		cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num),  in cppi41_configure_channel()
381 		len, cppi41_channel->is_tx);  in cppi41_configure_channel()
383 	cppi41_channel->buf_addr = dma_addr;  in cppi41_configure_channel()
384 	cppi41_channel->total_len = len;  in cppi41_configure_channel()
385 	cppi41_channel->transferred = 0;  in cppi41_configure_channel()
386 	cppi41_channel->packet_sz = packet_sz;  in cppi41_configure_channel()
387 	cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0;  in cppi41_configure_channel()
393 	if (cppi41_channel->is_tx)  in cppi41_configure_channel()
400 				RNDIS_REG(cppi41_channel->port_num), len);  in cppi41_configure_channel()
402 			cppi41_set_dma_mode(cppi41_channel,  in cppi41_configure_channel()
406 			cppi41_set_autoreq_mode(cppi41_channel,  in cppi41_configure_channel()
410 					RNDIS_REG(cppi41_channel->port_num), 0);  in cppi41_configure_channel()
411 			cppi41_set_dma_mode(cppi41_channel,  in cppi41_configure_channel()
413 			cppi41_set_autoreq_mode(cppi41_channel,  in cppi41_configure_channel()
418 		cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT);  in cppi41_configure_channel()
419 		cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);  in cppi41_configure_channel()
422 	cppi41_channel->prog_len = len;  in cppi41_configure_channel()
423 	direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;  in cppi41_configure_channel()
431 	cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);  in cppi41_configure_channel()
432 	cppi41_channel->channel.rx_packet_done = false;  in cppi41_configure_channel()
434 	save_rx_toggle(cppi41_channel);  in cppi41_configure_channel()
444 	struct cppi41_dma_channel *cppi41_channel = NULL;  in cppi41_dma_channel_allocate()  local
451 		cppi41_channel = &controller->tx_channel[ch_num];  in cppi41_dma_channel_allocate()
453 		cppi41_channel = &controller->rx_channel[ch_num];  in cppi41_dma_channel_allocate()
455 	if (!cppi41_channel->dc)  in cppi41_dma_channel_allocate()
458 	if (cppi41_channel->is_allocated)  in cppi41_dma_channel_allocate()
461 	cppi41_channel->hw_ep = hw_ep;  in cppi41_dma_channel_allocate()
462 	cppi41_channel->is_allocated = 1;  in cppi41_dma_channel_allocate()
464 	return &cppi41_channel->channel;  in cppi41_dma_channel_allocate()
469 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_release()  local
471 	if (cppi41_channel->is_allocated) {  in cppi41_dma_channel_release()
472 		cppi41_channel->is_allocated = 0;  in cppi41_dma_channel_release()
483 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_program()  local
489 	if (is_host_active(cppi41_channel->controller->musb)) {  in cppi41_dma_channel_program()
490 		if (cppi41_channel->is_tx)  in cppi41_dma_channel_program()
491 			hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;  in cppi41_dma_channel_program()
493 			hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;  in cppi41_dma_channel_program()
512 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_is_compatible()  local
513 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_is_compatible()
520 	if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)  in cppi41_is_compatible()
522 	if (cppi41_channel->is_tx)  in cppi41_is_compatible()
530 	struct cppi41_dma_channel *cppi41_channel = channel->private_data;  in cppi41_dma_channel_abort()  local
531 	struct cppi41_dma_controller *controller = cppi41_channel->controller;  in cppi41_dma_channel_abort()
533 	void __iomem *epio = cppi41_channel->hw_ep->regs;  in cppi41_dma_channel_abort()
539 	is_tx = cppi41_channel->is_tx;  in cppi41_dma_channel_abort()
541 			cppi41_channel->port_num, is_tx);  in cppi41_dma_channel_abort()
543 	if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)  in cppi41_dma_channel_abort()
546 	list_del_init(&cppi41_channel->tx_check);  in cppi41_dma_channel_abort()
552 		cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);  in cppi41_dma_channel_abort()
572 	tdbit = 1 << cppi41_channel->port_num;  in cppi41_dma_channel_abort()
579 		ret = dmaengine_terminate_all(cppi41_channel->dc);  in cppi41_dma_channel_abort()
592 	cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;  in cppi41_dma_channel_abort()
621 	struct cppi41_dma_channel *cppi41_channel;  in cppi41_dma_controller_start()  local
656 			cppi41_channel = &controller->tx_channel[port - 1];  in cppi41_dma_controller_start()
658 			cppi41_channel = &controller->rx_channel[port - 1];  in cppi41_dma_controller_start()
660 		cppi41_channel->controller = controller;  in cppi41_dma_controller_start()
661 		cppi41_channel->port_num = port;  in cppi41_dma_controller_start()
662 		cppi41_channel->is_tx = is_tx;  in cppi41_dma_controller_start()
663 		INIT_LIST_HEAD(&cppi41_channel->tx_check);  in cppi41_dma_controller_start()
665 		musb_dma = &cppi41_channel->channel;  in cppi41_dma_controller_start()
666 		musb_dma->private_data = cppi41_channel;  in cppi41_dma_controller_start()
676 		cppi41_channel->dc = dc;  in cppi41_dma_controller_start()