74 static u16 dwc2_max_desc_num(struct dwc2_qh *qh)  in dwc2_max_desc_num()  argument
76 	return (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&  in dwc2_max_desc_num()
77 		qh->dev_speed == USB_SPEED_HIGH) ?  in dwc2_max_desc_num()
81 static u16 dwc2_frame_incr_val(struct dwc2_qh *qh)  in dwc2_frame_incr_val()  argument
83 	return qh->dev_speed == USB_SPEED_HIGH ?  in dwc2_frame_incr_val()
84 	       (qh->interval + 8 - 1) / 8 : qh->interval;  in dwc2_frame_incr_val()
87 static int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_desc_list_alloc()  argument
90 	qh->desc_list = dma_alloc_coherent(hsotg->dev,  in dwc2_desc_list_alloc()
92 				dwc2_max_desc_num(qh), &qh->desc_list_dma,  in dwc2_desc_list_alloc()
95 	if (!qh->desc_list)  in dwc2_desc_list_alloc()
98 	memset(qh->desc_list, 0,  in dwc2_desc_list_alloc()
99 	       sizeof(struct dwc2_hcd_dma_desc) * dwc2_max_desc_num(qh));  in dwc2_desc_list_alloc()
101 	qh->n_bytes = kzalloc(sizeof(u32) * dwc2_max_desc_num(qh), flags);  in dwc2_desc_list_alloc()
102 	if (!qh->n_bytes) {  in dwc2_desc_list_alloc()
104 				  * dwc2_max_desc_num(qh), qh->desc_list,  in dwc2_desc_list_alloc()
105 				  qh->desc_list_dma);  in dwc2_desc_list_alloc()
106 		qh->desc_list = NULL;  in dwc2_desc_list_alloc()
113 static void dwc2_desc_list_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_desc_list_free()  argument
115 	if (qh->desc_list) {  in dwc2_desc_list_free()
117 				  * dwc2_max_desc_num(qh), qh->desc_list,  in dwc2_desc_list_free()
118 				  qh->desc_list_dma);  in dwc2_desc_list_free()
119 		qh->desc_list = NULL;  in dwc2_desc_list_free()
122 	kfree(qh->n_bytes);  in dwc2_desc_list_free()
123 	qh->n_bytes = NULL;  in dwc2_desc_list_free()
214 static void dwc2_update_frame_list(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_update_frame_list()  argument
225 	if (!qh->channel) {  in dwc2_update_frame_list()
226 		dev_err(hsotg->dev, "qh->channel = %p\n", qh->channel);  in dwc2_update_frame_list()
236 	chan = qh->channel;  in dwc2_update_frame_list()
237 	inc = dwc2_frame_incr_val(qh);  in dwc2_update_frame_list()
238 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC)  in dwc2_update_frame_list()
239 		i = dwc2_frame_list_idx(qh->sched_frame);  in dwc2_update_frame_list()
256 	if (chan->speed == USB_SPEED_HIGH && qh->interval) {  in dwc2_update_frame_list()
259 		inc = (8 + qh->interval - 1) / qh->interval;  in dwc2_update_frame_list()
262 			j = j << qh->interval;  in dwc2_update_frame_list()
270 				      struct dwc2_qh *qh)  in dwc2_release_channel_ddma()  argument
272 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_release_channel_ddma()
274 	if (dwc2_qh_is_non_per(qh)) {  in dwc2_release_channel_ddma()
280 		dwc2_update_frame_list(hsotg, qh, 0);  in dwc2_release_channel_ddma()
287 	if (chan->qh) {  in dwc2_release_channel_ddma()
292 		chan->qh = NULL;  in dwc2_release_channel_ddma()
295 	qh->channel = NULL;  in dwc2_release_channel_ddma()
296 	qh->ntd = 0;  in dwc2_release_channel_ddma()
298 	if (qh->desc_list)  in dwc2_release_channel_ddma()
299 		memset(qh->desc_list, 0, sizeof(struct dwc2_hcd_dma_desc) *  in dwc2_release_channel_ddma()
300 		       dwc2_max_desc_num(qh));  in dwc2_release_channel_ddma()
315 int dwc2_hcd_qh_init_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_hcd_qh_init_ddma()  argument
320 	if (qh->do_split) {  in dwc2_hcd_qh_init_ddma()
327 	retval = dwc2_desc_list_alloc(hsotg, qh, mem_flags);  in dwc2_hcd_qh_init_ddma()
331 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC ||  in dwc2_hcd_qh_init_ddma()
332 	    qh->ep_type == USB_ENDPOINT_XFER_INT) {  in dwc2_hcd_qh_init_ddma()
342 	qh->ntd = 0;  in dwc2_hcd_qh_init_ddma()
346 	dwc2_desc_list_free(hsotg, qh);  in dwc2_hcd_qh_init_ddma()
361 void dwc2_hcd_qh_free_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_qh_free_ddma()  argument
363 	dwc2_desc_list_free(hsotg, qh);  in dwc2_hcd_qh_free_ddma()
372 	if (qh->channel)  in dwc2_hcd_qh_free_ddma()
373 		dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_qh_free_ddma()
375 	if ((qh->ep_type == USB_ENDPOINT_XFER_ISOC ||  in dwc2_hcd_qh_free_ddma()
376 	     qh->ep_type == USB_ENDPOINT_XFER_INT) &&  in dwc2_hcd_qh_free_ddma()
384 static u8 dwc2_frame_to_desc_idx(struct dwc2_qh *qh, u16 frame_idx)  in dwc2_frame_to_desc_idx()  argument
386 	if (qh->dev_speed == USB_SPEED_HIGH)  in dwc2_frame_to_desc_idx()
398 				    struct dwc2_qh *qh, u16 *skip_frames)  in dwc2_calc_starting_frame()  argument
420 	if (qh->dev_speed == USB_SPEED_HIGH) {  in dwc2_calc_starting_frame()
457 					struct dwc2_qh *qh)  in dwc2_recalc_initial_desc_idx()  argument
479 	if (qh->channel) {  in dwc2_recalc_initial_desc_idx()
480 		frame = dwc2_calc_starting_frame(hsotg, qh, &skip_frames);  in dwc2_recalc_initial_desc_idx()
487 			  dwc2_frame_list_idx(qh->sched_frame) - fr_idx_tmp)  in dwc2_recalc_initial_desc_idx()
488 			 % dwc2_frame_incr_val(qh);  in dwc2_recalc_initial_desc_idx()
491 		qh->sched_frame = dwc2_calc_starting_frame(hsotg, qh,  in dwc2_recalc_initial_desc_idx()
493 		fr_idx = dwc2_frame_list_idx(qh->sched_frame);  in dwc2_recalc_initial_desc_idx()
496 	qh->td_first = qh->td_last = dwc2_frame_to_desc_idx(qh, fr_idx);  in dwc2_recalc_initial_desc_idx()
509 					 struct dwc2_qh *qh, u32 max_xfer_size,  in dwc2_fill_host_isoc_dma_desc()  argument
512 	struct dwc2_hcd_dma_desc *dma_desc = &qh->desc_list[idx];  in dwc2_fill_host_isoc_dma_desc()
519 		qh->n_bytes[idx] = max_xfer_size;  in dwc2_fill_host_isoc_dma_desc()
521 		qh->n_bytes[idx] = frame_desc->length;  in dwc2_fill_host_isoc_dma_desc()
524 	dma_desc->status = qh->n_bytes[idx] << HOST_DMA_ISOC_NBYTES_SHIFT &  in dwc2_fill_host_isoc_dma_desc()
533 	qh->ntd++;  in dwc2_fill_host_isoc_dma_desc()
538 				    struct dwc2_qh *qh, u16 skip_frames)  in dwc2_init_isoc_dma_desc()  argument
544 	idx = qh->td_last;  in dwc2_init_isoc_dma_desc()
545 	inc = qh->interval;  in dwc2_init_isoc_dma_desc()
548 	if (qh->interval) {  in dwc2_init_isoc_dma_desc()
549 		ntd_max = (dwc2_max_desc_num(qh) + qh->interval - 1) /  in dwc2_init_isoc_dma_desc()
550 				qh->interval;  in dwc2_init_isoc_dma_desc()
551 		if (skip_frames && !qh->channel)  in dwc2_init_isoc_dma_desc()
552 			ntd_max -= skip_frames / qh->interval;  in dwc2_init_isoc_dma_desc()
555 	max_xfer_size = qh->dev_speed == USB_SPEED_HIGH ?  in dwc2_init_isoc_dma_desc()
558 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {  in dwc2_init_isoc_dma_desc()
559 		while (qh->ntd < ntd_max && qtd->isoc_frame_index_last <  in dwc2_init_isoc_dma_desc()
562 				qh->desc_list[n_desc - 1].status |= HOST_DMA_A;  in dwc2_init_isoc_dma_desc()
563 			dwc2_fill_host_isoc_dma_desc(hsotg, qtd, qh,  in dwc2_init_isoc_dma_desc()
565 			idx = dwc2_desclist_idx_inc(idx, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
571 	qh->td_last = idx;  in dwc2_init_isoc_dma_desc()
575 	if (qh->ntd == ntd_max) {  in dwc2_init_isoc_dma_desc()
576 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
577 		qh->desc_list[idx].status |= HOST_DMA_IOC;  in dwc2_init_isoc_dma_desc()
596 		idx = dwc2_desclist_idx_dec(idx, inc * ((qh->ntd + 1) / 2),  in dwc2_init_isoc_dma_desc()
597 					    qh->dev_speed);  in dwc2_init_isoc_dma_desc()
604 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
606 	qh->desc_list[idx].status |= HOST_DMA_IOC;  in dwc2_init_isoc_dma_desc()
610 		qh->desc_list[n_desc - 1].status |= HOST_DMA_A;  in dwc2_init_isoc_dma_desc()
612 			qh->desc_list[0].status |= HOST_DMA_A;  in dwc2_init_isoc_dma_desc()
618 				    struct dwc2_qtd *qtd, struct dwc2_qh *qh,  in dwc2_fill_host_dma_desc()  argument
621 	struct dwc2_hcd_dma_desc *dma_desc = &qh->desc_list[n_desc];  in dwc2_fill_host_dma_desc()
642 	qh->n_bytes[n_desc] = len;  in dwc2_fill_host_dma_desc()
644 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL &&  in dwc2_fill_host_dma_desc()
663 					struct dwc2_qh *qh)  in dwc2_init_non_isoc_dma_desc()  argument
666 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_init_non_isoc_dma_desc()
669 	dev_vdbg(hsotg->dev, "%s(): qh=%p dma=%08lx len=%d\n", __func__, qh,  in dwc2_init_non_isoc_dma_desc()
679 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {  in dwc2_init_non_isoc_dma_desc()
695 				qh->desc_list[n_desc - 1].status |= HOST_DMA_A;  in dwc2_init_non_isoc_dma_desc()
699 					 &qh->desc_list[n_desc - 1]);  in dwc2_init_non_isoc_dma_desc()
701 			dwc2_fill_host_dma_desc(hsotg, chan, qtd, qh, n_desc);  in dwc2_init_non_isoc_dma_desc()
704 				 n_desc, &qh->desc_list[n_desc],  in dwc2_init_non_isoc_dma_desc()
705 				 qh->desc_list[n_desc].buf,  in dwc2_init_non_isoc_dma_desc()
706 				 qh->desc_list[n_desc].status);  in dwc2_init_non_isoc_dma_desc()
714 		if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL)  in dwc2_init_non_isoc_dma_desc()
721 		qh->desc_list[n_desc - 1].status |=  in dwc2_init_non_isoc_dma_desc()
724 			 n_desc - 1, &qh->desc_list[n_desc - 1]);  in dwc2_init_non_isoc_dma_desc()
726 			qh->desc_list[0].status |= HOST_DMA_A;  in dwc2_init_non_isoc_dma_desc()
728 				 &qh->desc_list[0]);  in dwc2_init_non_isoc_dma_desc()
753 void dwc2_hcd_start_xfer_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_start_xfer_ddma()  argument
756 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_hcd_start_xfer_ddma()
762 		dwc2_init_non_isoc_dma_desc(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
766 		dwc2_init_non_isoc_dma_desc(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
767 		dwc2_update_frame_list(hsotg, qh, 1);  in dwc2_hcd_start_xfer_ddma()
771 		if (!qh->ntd)  in dwc2_hcd_start_xfer_ddma()
772 			skip_frames = dwc2_recalc_initial_desc_idx(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
773 		dwc2_init_isoc_dma_desc(hsotg, qh, skip_frames);  in dwc2_hcd_start_xfer_ddma()
776 			dwc2_update_frame_list(hsotg, qh, 1);  in dwc2_hcd_start_xfer_ddma()
783 			chan->ntd = dwc2_max_desc_num(qh);  in dwc2_hcd_start_xfer_ddma()
801 					struct dwc2_qh *qh, u16 idx)  in dwc2_cmpl_host_isoc_dma_desc()  argument
803 	struct dwc2_hcd_dma_desc *dma_desc = &qh->desc_list[idx];  in dwc2_cmpl_host_isoc_dma_desc()
824 		frame_desc->actual_length = qh->n_bytes[idx] - remain;  in dwc2_cmpl_host_isoc_dma_desc()
828 		frame_desc->actual_length = qh->n_bytes[idx] - remain;  in dwc2_cmpl_host_isoc_dma_desc()
838 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_cmpl_host_isoc_dma_desc()
851 	qh->ntd--;  in dwc2_cmpl_host_isoc_dma_desc()
866 	struct dwc2_qh *qh;  in dwc2_complete_isoc_xfer_ddma()  local
870 	qh = chan->qh;  in dwc2_complete_isoc_xfer_ddma()
871 	idx = qh->td_first;  in dwc2_complete_isoc_xfer_ddma()
874 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)  in dwc2_complete_isoc_xfer_ddma()
893 		list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list,  in dwc2_complete_isoc_xfer_ddma()
905 			dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_complete_isoc_xfer_ddma()
911 	list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list, qtd_list_entry) {  in dwc2_complete_isoc_xfer_ddma()
915 			rc = dwc2_cmpl_host_isoc_dma_desc(hsotg, chan, qtd, qh,  in dwc2_complete_isoc_xfer_ddma()
919 			idx = dwc2_desclist_idx_inc(idx, qh->interval,  in dwc2_complete_isoc_xfer_ddma()
925 		} while (idx != qh->td_first);  in dwc2_complete_isoc_xfer_ddma()
929 	qh->td_first = idx;  in dwc2_complete_isoc_xfer_ddma()
1021 	struct dwc2_qh *qh = chan->qh;  in dwc2_process_non_isoc_desc()  local
1032 	dma_desc = &qh->desc_list[desc_num];  in dwc2_process_non_isoc_desc()
1033 	n_bytes = qh->n_bytes[desc_num];  in dwc2_process_non_isoc_desc()
1042 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_process_non_isoc_desc()
1048 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL) {  in dwc2_process_non_isoc_desc()
1086 	struct dwc2_qh *qh = chan->qh;  in dwc2_complete_non_isoc_xfer_ddma()  local
1092 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)  in dwc2_complete_non_isoc_xfer_ddma()
1097 	list_for_each_safe(qtd_item, qtd_tmp, &qh->qtd_list) {  in dwc2_complete_non_isoc_xfer_ddma()
1114 	if (qh->ep_type != USB_ENDPOINT_XFER_CONTROL) {  in dwc2_complete_non_isoc_xfer_ddma()
1120 			qh->data_toggle = DWC2_HC_PID_DATA0;  in dwc2_complete_non_isoc_xfer_ddma()
1132 			qh->ping_state = 1;  in dwc2_complete_non_isoc_xfer_ddma()
1158 	struct dwc2_qh *qh = chan->qh;  in dwc2_hcd_complete_xfer_ddma()  local
1167 		    list_empty(&qh->qtd_list)) {  in dwc2_hcd_complete_xfer_ddma()
1171 			dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1172 			dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1175 			list_move(&qh->qh_list_entry,  in dwc2_hcd_complete_xfer_ddma()
1190 		dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1191 		dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1193 		if (!list_empty(&qh->qtd_list)) {  in dwc2_hcd_complete_xfer_ddma()
1198 			dwc2_hcd_qh_add(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()