46 qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,  in qtd_fill()  argument
53 	qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
54 	qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));  in qtd_fill()
65 			qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);  in qtd_fill()
66 			qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,  in qtd_fill()
79 	qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);  in qtd_fill()
88 qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)  in qh_update()  argument
95 	hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_update()
96 	hw->hw_alt_next = EHCI_LIST_END(ehci);  in qh_update()
103 	if (!(hw->hw_info1 & cpu_to_hc32(ehci, QH_TOGGLE_CTL))) {  in qh_update()
107 		epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;  in qh_update()
109 			hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);  in qh_update()
114 	hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);  in qh_update()
122 qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_refresh()  argument
135 	if (qh->hw->hw_token & ACTIVE_BIT(ehci))  in qh_refresh()
138 		qh_update(ehci, qh, qtd);  in qh_refresh()
143 static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
148 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_clear_tt_buffer_complete()  local
152 	spin_lock_irqsave(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
155 			&& ehci->rh_state == EHCI_RH_RUNNING)  in ehci_clear_tt_buffer_complete()
156 		qh_link_async(ehci, qh);  in ehci_clear_tt_buffer_complete()
157 	spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_clear_tt_buffer_complete()
160 static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,  in ehci_clear_tt_buffer()  argument
178 		if (!ehci_is_TDI(ehci)  in ehci_clear_tt_buffer()
180 				   ehci_to_hcd(ehci)->self.root_hub) {  in ehci_clear_tt_buffer()
193 	struct ehci_hcd *ehci,  in qtd_copy_status()  argument
235 			ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",  in qtd_copy_status()
249 ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)  in ehci_urb_done()  argument
253 		ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;  in ehci_urb_done()
257 		COUNT(ehci->stats.unlink);  in ehci_urb_done()
262 		COUNT(ehci->stats.complete);  in ehci_urb_done()
266 	ehci_dbg (ehci,  in ehci_urb_done()
275 	usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in ehci_urb_done()
276 	usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);  in ehci_urb_done()
279 static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
287 qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_completions()  argument
331 				ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
334 			ehci_qtd_free (ehci, last);  in qh_completions()
344 		token = hc32_to_cpu(ehci, qtd->hw_token);  in qh_completions()
352 				ehci_dbg(ehci,  in qh_completions()
373 					ehci_dbg(ehci,  in qh_completions()
385 					qtd->hw_token = cpu_to_hc32(ehci,  in qh_completions()
388 					hw->hw_token = cpu_to_hc32(ehci,  in qh_completions()
405 						& EHCI_LIST_END(ehci))) {  in qh_completions()
411 				&& ehci->rh_state >= EHCI_RH_RUNNING)) {  in qh_completions()
419 			if (ehci->rh_state < EHCI_RH_RUNNING)  in qh_completions()
438 					(hw->hw_token & ACTIVE_BIT(ehci))) {  in qh_completions()
439 				token = hc32_to_cpu(ehci, hw->hw_token);  in qh_completions()
440 				hw->hw_token &= ~ACTIVE_BIT(ehci);  in qh_completions()
446 				ehci_clear_tt_buffer(ehci, qh, urb, token);  in qh_completions()
458 			last_status = qtd_copy_status(ehci, urb,  in qh_completions()
462 						& EHCI_LIST_END(ehci)))  in qh_completions()
480 					ehci_clear_tt_buffer(ehci, qh, urb,  in qh_completions()
504 		ehci_urb_done(ehci, last->urb, last_status);  in qh_completions()
505 		ehci_qtd_free (ehci, last);  in qh_completions()
535 	if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci))  in qh_completions()
554 	struct ehci_hcd		*ehci,  in qtd_list_free()  argument
565 		ehci_qtd_free (ehci, qtd);  in qtd_list_free()
574 	struct ehci_hcd		*ehci,  in qh_urb_transaction()  argument
590 	qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
604 		qtd_fill(ehci, qtd, urb->setup_dma,  in qh_urb_transaction()
611 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
615 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
655 		this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,  in qh_urb_transaction()
667 			qtd->hw_alt_next = ehci->async->hw->hw_alt_next;  in qh_urb_transaction()
682 		qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
686 		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
697 		qtd->hw_alt_next = EHCI_LIST_END(ehci);  in qh_urb_transaction()
718 			qtd = ehci_qtd_alloc (ehci, flags);  in qh_urb_transaction()
722 			qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in qh_urb_transaction()
726 			qtd_fill(ehci, qtd, 0, 0, token, 0);  in qh_urb_transaction()
732 		qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);  in qh_urb_transaction()
736 	qtd_list_free (ehci, urb, head);  in qh_urb_transaction()
759 	struct ehci_hcd		*ehci,  in qh_make()  argument
763 	struct ehci_qh		*qh = ehci_qh_alloc (ehci, flags);  in qh_make()
787 		ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));  in qh_make()
817 			} else if (urb->interval > ehci->periodic_size << 3) {  in qh_make()
818 				urb->interval = ehci->periodic_size << 3;  in qh_make()
849 			if (urb->interval > ehci->periodic_size)  in qh_make()
850 				urb->interval = ehci->periodic_size;  in qh_make()
889 		if (ehci_has_fsl_portno_bug(ehci))  in qh_make()
897 		if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)  in qh_make()
927 		ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev,  in qh_make()
930 		qh_destroy(ehci, qh);  in qh_make()
939 	hw->hw_info1 = cpu_to_hc32(ehci, info1);  in qh_make()
940 	hw->hw_info2 = cpu_to_hc32(ehci, info2);  in qh_make()
948 static void enable_async(struct ehci_hcd *ehci)  in enable_async()  argument
950 	if (ehci->async_count++)  in enable_async()
954 	ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC);  in enable_async()
957 	ehci_poll_ASS(ehci);  in enable_async()
958 	turn_on_io_watchdog(ehci);  in enable_async()
961 static void disable_async(struct ehci_hcd *ehci)  in disable_async()  argument
963 	if (--ehci->async_count)  in disable_async()
967 	WARN_ON(ehci->async->qh_next.qh || !list_empty(&ehci->async_unlink) ||  in disable_async()
968 			!list_empty(&ehci->async_idle));  in disable_async()
971 	ehci_poll_ASS(ehci);  in disable_async()
976 static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)  in qh_link_async()  argument
978 	__hc32		dma = QH_NEXT(ehci, qh->qh_dma);  in qh_link_async()
988 	qh_refresh(ehci, qh);  in qh_link_async()
991 	head = ehci->async;  in qh_link_async()
1004 	enable_async(ehci);  in qh_link_async()
1016 	struct ehci_hcd		*ehci,  in qh_append_tds()  argument
1024 	__hc32			qh_addr_mask = cpu_to_hc32(ehci, 0x7f);  in qh_append_tds()
1029 		qh = qh_make (ehci, urb, GFP_ATOMIC);  in qh_append_tds()
1063 			qtd->hw_token = HALT_BIT(ehci);  in qh_append_tds()
1075 			ehci_qtd_init(ehci, qtd, qtd->qtd_dma);  in qh_append_tds()
1082 			qtd->hw_next = QTD_NEXT(ehci, dma);  in qh_append_tds()
1098 	struct ehci_hcd		*ehci,  in submit_async()  argument
1114 		ehci_dbg(ehci,  in submit_async()
1123 	spin_lock_irqsave (&ehci->lock, flags);  in submit_async()
1124 	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {  in submit_async()
1128 	rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1132 	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);  in submit_async()
1134 		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);  in submit_async()
1143 		qh_link_async(ehci, qh);  in submit_async()
1145 	spin_unlock_irqrestore (&ehci->lock, flags);  in submit_async()
1147 		qtd_list_free (ehci, urb, qtd_list);  in submit_async()
1168 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in submit_single_step_set_feature()  local
1181 	qtd = ehci_qtd_alloc(ehci, GFP_KERNEL);  in submit_single_step_set_feature()
1198 		qtd_fill(ehci, qtd, urb->setup_dma,  in submit_single_step_set_feature()
1202 		submit_async(ehci, urb, &qtd_list, GFP_ATOMIC);  in submit_single_step_set_feature()
1217 	qtd_fill(ehci, qtd, buf, len, token, maxpacket);  in submit_single_step_set_feature()
1223 	qtd->hw_alt_next = EHCI_LIST_END(ehci);  in submit_single_step_set_feature()
1230 	qtd = ehci_qtd_alloc(ehci, GFP_ATOMIC);  in submit_single_step_set_feature()
1234 	qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);  in submit_single_step_set_feature()
1238 	qtd_fill(ehci, qtd, 0, 0, token, 0);  in submit_single_step_set_feature()
1242 		qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);  in submit_single_step_set_feature()
1244 	submit_async(ehci, urb, &qtd_list, GFP_KERNEL);  in submit_single_step_set_feature()
1249 	qtd_list_free(ehci, urb, head);  in submit_single_step_set_feature()
1256 static void single_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)  in single_unlink_async()  argument
1262 	list_add_tail(&qh->unlink_node, &ehci->async_unlink);  in single_unlink_async()
1265 	prev = ehci->async;  in single_unlink_async()
1271 	if (ehci->qh_scan_next == qh)  in single_unlink_async()
1272 		ehci->qh_scan_next = qh->qh_next.qh;  in single_unlink_async()
1275 static void start_iaa_cycle(struct ehci_hcd *ehci)  in start_iaa_cycle()  argument
1278 	if (ehci->iaa_in_progress)  in start_iaa_cycle()
1280 	ehci->iaa_in_progress = true;  in start_iaa_cycle()
1283 	if (unlikely(ehci->rh_state < EHCI_RH_RUNNING)) {  in start_iaa_cycle()
1284 		end_unlink_async(ehci);  in start_iaa_cycle()
1287 	} else if (likely(ehci->rh_state == EHCI_RH_RUNNING)) {  in start_iaa_cycle()
1292 		ehci_writel(ehci, ehci->command | CMD_IAAD,  in start_iaa_cycle()
1293 				&ehci->regs->command);  in start_iaa_cycle()
1294 		ehci_readl(ehci, &ehci->regs->command);  in start_iaa_cycle()
1295 		ehci_enable_event(ehci, EHCI_HRTIMER_IAA_WATCHDOG, true);  in start_iaa_cycle()
1301 static void end_unlink_async(struct ehci_hcd *ehci)  in end_unlink_async()  argument
1306 	if (ehci->has_synopsys_hc_bug)  in end_unlink_async()
1307 		ehci_writel(ehci, (u32) ehci->async->qh_dma,  in end_unlink_async()
1308 			    &ehci->regs->async_next);  in end_unlink_async()
1311 	ehci->iaa_in_progress = false;  in end_unlink_async()
1313 	if (list_empty(&ehci->async_unlink))  in end_unlink_async()
1315 	qh = list_first_entry(&ehci->async_unlink, struct ehci_qh,  in end_unlink_async()
1322 	early_exit = ehci->async_unlinking;  in end_unlink_async()
1325 	if (ehci->rh_state < EHCI_RH_RUNNING)  in end_unlink_async()
1326 		list_splice_tail_init(&ehci->async_unlink, &ehci->async_idle);  in end_unlink_async()
1340 		list_move_tail(&qh->unlink_node, &ehci->async_idle);  in end_unlink_async()
1343 	if (!list_empty(&ehci->async_unlink))  in end_unlink_async()
1344 		start_iaa_cycle(ehci);  in end_unlink_async()
1354 	ehci->async_unlinking = true;  in end_unlink_async()
1355 	while (!list_empty(&ehci->async_idle)) {  in end_unlink_async()
1356 		qh = list_first_entry(&ehci->async_idle, struct ehci_qh,  in end_unlink_async()
1364 			qh_completions(ehci, qh);  in end_unlink_async()
1366 				ehci->rh_state == EHCI_RH_RUNNING)  in end_unlink_async()
1367 			qh_link_async(ehci, qh);  in end_unlink_async()
1368 		disable_async(ehci);  in end_unlink_async()
1370 	ehci->async_unlinking = false;  in end_unlink_async()
1373 static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
1375 static void unlink_empty_async(struct ehci_hcd *ehci)  in unlink_empty_async()  argument
1382 	for (qh = ehci->async->qh_next.qh; qh; qh = qh->qh_next.qh) {  in unlink_empty_async()
1386 			if (qh->unlink_cycle != ehci->async_unlink_cycle)  in unlink_empty_async()
1392 	if (list_empty(&ehci->async_unlink) && qh_to_unlink) {  in unlink_empty_async()
1393 		start_unlink_async(ehci, qh_to_unlink);  in unlink_empty_async()
1399 		ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);  in unlink_empty_async()
1400 		++ehci->async_unlink_cycle;  in unlink_empty_async()
1405 static void __maybe_unused unlink_empty_async_suspended(struct ehci_hcd *ehci)  in unlink_empty_async_suspended()  argument
1409 	while (ehci->async->qh_next.qh) {  in unlink_empty_async_suspended()
1410 		qh = ehci->async->qh_next.qh;  in unlink_empty_async_suspended()
1412 		single_unlink_async(ehci, qh);  in unlink_empty_async_suspended()
1414 	start_iaa_cycle(ehci);  in unlink_empty_async_suspended()
1420 static void start_unlink_async(struct ehci_hcd *ehci, struct ehci_qh *qh)  in start_unlink_async()  argument
1426 	single_unlink_async(ehci, qh);  in start_unlink_async()
1427 	start_iaa_cycle(ehci);  in start_unlink_async()
1432 static void scan_async (struct ehci_hcd *ehci)  in scan_async()  argument
1437 	ehci->qh_scan_next = ehci->async->qh_next.qh;  in scan_async()
1438 	while (ehci->qh_scan_next) {  in scan_async()
1439 		qh = ehci->qh_scan_next;  in scan_async()
1440 		ehci->qh_scan_next = qh->qh_next.qh;  in scan_async()
1453 			temp = qh_completions(ehci, qh);  in scan_async()
1455 				start_unlink_async(ehci, qh);  in scan_async()
1458 				qh->unlink_cycle = ehci->async_unlink_cycle;  in scan_async()
1470 	if (check_unlinks_later && ehci->rh_state == EHCI_RH_RUNNING &&  in scan_async()
1471 			!(ehci->enabled_hrtimer_events &  in scan_async()
1473 		ehci_enable_event(ehci, EHCI_HRTIMER_ASYNC_UNLINKS, true);  in scan_async()
1474 		++ehci->async_unlink_cycle;  in scan_async()