122 static unsigned ehci_moschip_read_frame_index(struct ehci_hcd *ehci)  in ehci_moschip_read_frame_index()  argument
126 	uf = ehci_readl(ehci, &ehci->regs->frame_index);  in ehci_moschip_read_frame_index()
128 		uf = ehci_readl(ehci, &ehci->regs->frame_index);  in ehci_moschip_read_frame_index()
132 static inline unsigned ehci_read_frame_index(struct ehci_hcd *ehci)  in ehci_read_frame_index()  argument
134 	if (ehci->frame_index_bug)  in ehci_read_frame_index()
135 		return ehci_moschip_read_frame_index(ehci);  in ehci_read_frame_index()
136 	return ehci_readl(ehci, &ehci->regs->frame_index);  in ehci_read_frame_index()
160 int ehci_handshake(struct ehci_hcd *ehci, void __iomem *ptr,  in ehci_handshake()  argument
166 		result = ehci_readl(ehci, ptr);  in ehci_handshake()
180 static int tdi_in_host_mode (struct ehci_hcd *ehci)  in tdi_in_host_mode()  argument
184 	tmp = ehci_readl(ehci, &ehci->regs->usbmode);  in tdi_in_host_mode()
192 static int ehci_halt (struct ehci_hcd *ehci)  in ehci_halt()  argument
196 	spin_lock_irq(&ehci->lock);  in ehci_halt()
199 	ehci_writel(ehci, 0, &ehci->regs->intr_enable);  in ehci_halt()
201 	if (ehci_is_TDI(ehci) && !tdi_in_host_mode(ehci)) {  in ehci_halt()
202 		spin_unlock_irq(&ehci->lock);  in ehci_halt()
210 	ehci->command &= ~CMD_RUN;  in ehci_halt()
211 	temp = ehci_readl(ehci, &ehci->regs->command);  in ehci_halt()
213 	ehci_writel(ehci, temp, &ehci->regs->command);  in ehci_halt()
215 	spin_unlock_irq(&ehci->lock);  in ehci_halt()
216 	synchronize_irq(ehci_to_hcd(ehci)->irq);  in ehci_halt()
218 	return ehci_handshake(ehci, &ehci->regs->status,  in ehci_halt()
223 static void tdi_reset (struct ehci_hcd *ehci)  in tdi_reset()  argument
227 	tmp = ehci_readl(ehci, &ehci->regs->usbmode);  in tdi_reset()
233 	if (ehci_big_endian_mmio(ehci))  in tdi_reset()
235 	ehci_writel(ehci, tmp, &ehci->regs->usbmode);  in tdi_reset()
242 int ehci_reset(struct ehci_hcd *ehci)  in ehci_reset()  argument
245 	u32	command = ehci_readl(ehci, &ehci->regs->command);  in ehci_reset()
249 	if (ehci->debug && !dbgp_reset_prep(ehci_to_hcd(ehci)))  in ehci_reset()
250 		ehci->debug = NULL;  in ehci_reset()
253 	dbg_cmd (ehci, "reset", command);  in ehci_reset()
254 	ehci_writel(ehci, command, &ehci->regs->command);  in ehci_reset()
255 	ehci->rh_state = EHCI_RH_HALTED;  in ehci_reset()
256 	ehci->next_statechange = jiffies;  in ehci_reset()
257 	retval = ehci_handshake(ehci, &ehci->regs->command,  in ehci_reset()
260 	if (ehci->has_hostpc) {  in ehci_reset()
261 		ehci_writel(ehci, USBMODE_EX_HC | USBMODE_EX_VBPS,  in ehci_reset()
262 				&ehci->regs->usbmode_ex);  in ehci_reset()
263 		ehci_writel(ehci, TXFIFO_DEFAULT, &ehci->regs->txfill_tuning);  in ehci_reset()
268 	if (ehci_is_TDI(ehci))  in ehci_reset()
269 		tdi_reset (ehci);  in ehci_reset()
271 	if (ehci->debug)  in ehci_reset()
272 		dbgp_external_startup(ehci_to_hcd(ehci));  in ehci_reset()
274 	ehci->port_c_suspend = ehci->suspended_ports =  in ehci_reset()
275 			ehci->resuming_ports = 0;  in ehci_reset()
284 static void ehci_quiesce (struct ehci_hcd *ehci)  in ehci_quiesce()  argument
288 	if (ehci->rh_state != EHCI_RH_RUNNING)  in ehci_quiesce()
292 	temp = (ehci->command << 10) & (STS_ASS | STS_PSS);  in ehci_quiesce()
293 	ehci_handshake(ehci, &ehci->regs->status, STS_ASS | STS_PSS, temp,  in ehci_quiesce()
297 	spin_lock_irq(&ehci->lock);  in ehci_quiesce()
298 	ehci->command &= ~(CMD_ASE | CMD_PSE);  in ehci_quiesce()
299 	ehci_writel(ehci, ehci->command, &ehci->regs->command);  in ehci_quiesce()
300 	spin_unlock_irq(&ehci->lock);  in ehci_quiesce()
303 	ehci_handshake(ehci, &ehci->regs->status, STS_ASS | STS_PSS, 0,  in ehci_quiesce()
309 static void end_unlink_async(struct ehci_hcd *ehci);
310 static void unlink_empty_async(struct ehci_hcd *ehci);
311 static void unlink_empty_async_suspended(struct ehci_hcd *ehci);
312 static void ehci_work(struct ehci_hcd *ehci);
313 static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh);
314 static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh);
315 static int ehci_port_power(struct ehci_hcd *ehci, int portnum, bool enable);
330 static void ehci_turn_off_all_ports(struct ehci_hcd *ehci)  in ehci_turn_off_all_ports()  argument
332 	int	port = HCS_N_PORTS(ehci->hcs_params);  in ehci_turn_off_all_ports()
335 		ehci_writel(ehci, PORT_RWC_BITS,  in ehci_turn_off_all_ports()
336 				&ehci->regs->port_status[port]);  in ehci_turn_off_all_ports()
337 		spin_unlock_irq(&ehci->lock);  in ehci_turn_off_all_ports()
338 		ehci_port_power(ehci, port, false);  in ehci_turn_off_all_ports()
339 		spin_lock_irq(&ehci->lock);  in ehci_turn_off_all_ports()
347 static void ehci_silence_controller(struct ehci_hcd *ehci)  in ehci_silence_controller()  argument
349 	ehci_halt(ehci);  in ehci_silence_controller()
351 	spin_lock_irq(&ehci->lock);  in ehci_silence_controller()
352 	ehci->rh_state = EHCI_RH_HALTED;  in ehci_silence_controller()
353 	ehci_turn_off_all_ports(ehci);  in ehci_silence_controller()
356 	ehci_writel(ehci, 0, &ehci->regs->configured_flag);  in ehci_silence_controller()
359 	ehci_readl(ehci, &ehci->regs->configured_flag);  in ehci_silence_controller()
360 	spin_unlock_irq(&ehci->lock);  in ehci_silence_controller()
369 	struct ehci_hcd	*ehci = hcd_to_ehci(hcd);  in ehci_shutdown()  local
371 	spin_lock_irq(&ehci->lock);  in ehci_shutdown()
372 	ehci->shutdown = true;  in ehci_shutdown()
373 	ehci->rh_state = EHCI_RH_STOPPING;  in ehci_shutdown()
374 	ehci->enabled_hrtimer_events = 0;  in ehci_shutdown()
375 	spin_unlock_irq(&ehci->lock);  in ehci_shutdown()
377 	ehci_silence_controller(ehci);  in ehci_shutdown()
379 	hrtimer_cancel(&ehci->hrtimer);  in ehci_shutdown()
388 static void ehci_work (struct ehci_hcd *ehci)  in ehci_work()  argument
394 	if (ehci->scanning) {  in ehci_work()
395 		ehci->need_rescan = true;  in ehci_work()
398 	ehci->scanning = true;  in ehci_work()
401 	ehci->need_rescan = false;  in ehci_work()
402 	if (ehci->async_count)  in ehci_work()
403 		scan_async(ehci);  in ehci_work()
404 	if (ehci->intr_count > 0)  in ehci_work()
405 		scan_intr(ehci);  in ehci_work()
406 	if (ehci->isoc_count > 0)  in ehci_work()
407 		scan_isoc(ehci);  in ehci_work()
408 	if (ehci->need_rescan)  in ehci_work()
410 	ehci->scanning = false;  in ehci_work()
416 	turn_on_io_watchdog(ehci);  in ehci_work()
424 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_stop()  local
426 	ehci_dbg (ehci, "stop\n");  in ehci_stop()
430 	spin_lock_irq(&ehci->lock);  in ehci_stop()
431 	ehci->enabled_hrtimer_events = 0;  in ehci_stop()
432 	spin_unlock_irq(&ehci->lock);  in ehci_stop()
434 	ehci_quiesce(ehci);  in ehci_stop()
435 	ehci_silence_controller(ehci);  in ehci_stop()
436 	ehci_reset (ehci);  in ehci_stop()
438 	hrtimer_cancel(&ehci->hrtimer);  in ehci_stop()
439 	remove_sysfs_files(ehci);  in ehci_stop()
440 	remove_debug_files (ehci);  in ehci_stop()
443 	spin_lock_irq (&ehci->lock);  in ehci_stop()
444 	end_free_itds(ehci);  in ehci_stop()
445 	spin_unlock_irq (&ehci->lock);  in ehci_stop()
446 	ehci_mem_cleanup (ehci);  in ehci_stop()
448 	if (ehci->amd_pll_fix == 1)  in ehci_stop()
451 	dbg_status (ehci, "ehci_stop completed",  in ehci_stop()
452 		    ehci_readl(ehci, &ehci->regs->status));  in ehci_stop()
458 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_init()  local
464 	spin_lock_init(&ehci->lock);  in ehci_init()
469 	ehci->need_io_watchdog = 1;  in ehci_init()
471 	hrtimer_init(&ehci->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in ehci_init()
472 	ehci->hrtimer.function = ehci_hrtimer_func;  in ehci_init()
473 	ehci->next_hrtimer_event = EHCI_HRTIMER_NO_EVENT;  in ehci_init()
475 	hcc_params = ehci_readl(ehci, &ehci->caps->hcc_params);  in ehci_init()
481 	ehci->uframe_periodic_max = 100;  in ehci_init()
487 	ehci->periodic_size = DEFAULT_I_TDPS;  in ehci_init()
488 	INIT_LIST_HEAD(&ehci->async_unlink);  in ehci_init()
489 	INIT_LIST_HEAD(&ehci->async_idle);  in ehci_init()
490 	INIT_LIST_HEAD(&ehci->intr_unlink_wait);  in ehci_init()
491 	INIT_LIST_HEAD(&ehci->intr_unlink);  in ehci_init()
492 	INIT_LIST_HEAD(&ehci->intr_qh_list);  in ehci_init()
493 	INIT_LIST_HEAD(&ehci->cached_itd_list);  in ehci_init()
494 	INIT_LIST_HEAD(&ehci->cached_sitd_list);  in ehci_init()
495 	INIT_LIST_HEAD(&ehci->tt_list);  in ehci_init()
500 		case 0: ehci->periodic_size = 1024; break;  in ehci_init()
501 		case 1: ehci->periodic_size = 512; break;  in ehci_init()
502 		case 2: ehci->periodic_size = 256; break;  in ehci_init()
506 	if ((retval = ehci_mem_init(ehci, GFP_KERNEL)) < 0)  in ehci_init()
511 		ehci->i_thresh = 0;  in ehci_init()
513 		ehci->i_thresh = 2 + HCC_ISOC_THRES(hcc_params);  in ehci_init()
522 	ehci->async->qh_next.qh = NULL;  in ehci_init()
523 	hw = ehci->async->hw;  in ehci_init()
524 	hw->hw_next = QH_NEXT(ehci, ehci->async->qh_dma);  in ehci_init()
525 	hw->hw_info1 = cpu_to_hc32(ehci, QH_HEAD);  in ehci_init()
527 	hw->hw_info1 |= cpu_to_hc32(ehci, QH_INACTIVATE);  in ehci_init()
529 	hw->hw_token = cpu_to_hc32(ehci, QTD_STS_HALT);  in ehci_init()
530 	hw->hw_qtd_next = EHCI_LIST_END(ehci);  in ehci_init()
531 	ehci->async->qh_state = QH_STATE_LINKED;  in ehci_init()
532 	hw->hw_alt_next = QTD_NEXT(ehci, ehci->async->dummy->qtd_dma);  in ehci_init()
539 		ehci->has_ppcd = 1;  in ehci_init()
540 		ehci_dbg(ehci, "enable per-port change event\n");  in ehci_init()
556 		ehci_dbg(ehci, "park %d\n", park);  in ehci_init()
563 	ehci->command = temp;  in ehci_init()
574 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_run()  local
582 	ehci_writel(ehci, ehci->periodic_dma, &ehci->regs->frame_list);  in ehci_run()
583 	ehci_writel(ehci, (u32)ehci->async->qh_dma, &ehci->regs->async_next);  in ehci_run()
597 	hcc_params = ehci_readl(ehci, &ehci->caps->hcc_params);  in ehci_run()
599 		ehci_writel(ehci, 0, &ehci->regs->segment);  in ehci_run()
603 			ehci_info(ehci, "enabled 64bit DMA\n");  in ehci_run()
610 	ehci->command &= ~(CMD_LRESET|CMD_IAAD|CMD_PSE|CMD_ASE|CMD_RESET);  in ehci_run()
611 	ehci->command |= CMD_RUN;  in ehci_run()
612 	ehci_writel(ehci, ehci->command, &ehci->regs->command);  in ehci_run()
613 	dbg_cmd (ehci, "init", ehci->command);  in ehci_run()
630 	ehci->rh_state = EHCI_RH_RUNNING;  in ehci_run()
631 	ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);  in ehci_run()
632 	ehci_readl(ehci, &ehci->regs->command);	/* unblock posted writes */  in ehci_run()
635 	ehci->last_periodic_enable = ktime_get_real();  in ehci_run()
637 	temp = HC_VERSION(ehci, ehci_readl(ehci, &ehci->caps->hc_capbase));  in ehci_run()
638 	ehci_info (ehci,  in ehci_run()
640 		((ehci->sbrn & 0xf0)>>4), (ehci->sbrn & 0x0f),  in ehci_run()
644 	ehci_writel(ehci, INTR_MASK,  in ehci_run()
645 		    &ehci->regs->intr_enable); /* Turn On Interrupts */  in ehci_run()
651 	create_debug_files(ehci);  in ehci_run()
652 	create_sysfs_files(ehci);  in ehci_run()
659 	struct ehci_hcd *ehci = hcd_to_ehci(hcd);  in ehci_setup()  local
662 	ehci->regs = (void __iomem *)ehci->caps +  in ehci_setup()
663 	    HC_LENGTH(ehci, ehci_readl(ehci, &ehci->caps->hc_capbase));  in ehci_setup()
664 	dbg_hcs_params(ehci, "reset");  in ehci_setup()
665 	dbg_hcc_params(ehci, "reset");  in ehci_setup()
668 	ehci->hcs_params = ehci_readl(ehci, &ehci->caps->hcs_params);  in ehci_setup()
670 	ehci->sbrn = HCD_USB2;  in ehci_setup()
677 	retval = ehci_halt(ehci);  in ehci_setup()
681 	ehci_reset(ehci);  in ehci_setup()
691 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_irq()  local
702 	spin_lock_irqsave(&ehci->lock, flags);  in ehci_irq()
704 	status = ehci_readl(ehci, &ehci->regs->status);  in ehci_irq()
708 		ehci_dbg (ehci, "device removed\n");  in ehci_irq()
719 	if (!masked_status || unlikely(ehci->rh_state == EHCI_RH_HALTED)) {  in ehci_irq()
720 		spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_irq()
725 	ehci_writel(ehci, masked_status, &ehci->regs->status);  in ehci_irq()
726 	cmd = ehci_readl(ehci, &ehci->regs->command);  in ehci_irq()
732 			COUNT (ehci->stats.normal);  in ehci_irq()
734 			COUNT (ehci->stats.error);  in ehci_irq()
742 		ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_IAA_WATCHDOG);  in ehci_irq()
751 		if (ehci->next_hrtimer_event == EHCI_HRTIMER_IAA_WATCHDOG)  in ehci_irq()
752 			++ehci->next_hrtimer_event;  in ehci_irq()
756 			ehci_dbg(ehci, "IAA with IAAD still set?\n");  in ehci_irq()
757 		if (ehci->iaa_in_progress)  in ehci_irq()
758 			COUNT(ehci->stats.iaa);  in ehci_irq()
759 		end_unlink_async(ehci);  in ehci_irq()
764 		unsigned	i = HCS_N_PORTS (ehci->hcs_params);  in ehci_irq()
771 		if (ehci->rh_state == EHCI_RH_SUSPENDED)  in ehci_irq()
775 		if (ehci->has_ppcd)  in ehci_irq()
784 			pstatus = ehci_readl(ehci,  in ehci_irq()
785 					 &ehci->regs->port_status[i]);  in ehci_irq()
789 			if (!(test_bit(i, &ehci->suspended_ports) &&  in ehci_irq()
793 					ehci->reset_done[i] == 0))  in ehci_irq()
800 			ehci->reset_done[i] = jiffies +  in ehci_irq()
802 			set_bit(i, &ehci->resuming_ports);  in ehci_irq()
803 			ehci_dbg (ehci, "port %d remote wakeup\n", i + 1);  in ehci_irq()
805 			mod_timer(&hcd->rh_timer, ehci->reset_done[i]);  in ehci_irq()
811 		ehci_err(ehci, "fatal error\n");  in ehci_irq()
812 		dbg_cmd(ehci, "fatal", cmd);  in ehci_irq()
813 		dbg_status(ehci, "fatal", status);  in ehci_irq()
818 		ehci->shutdown = true;  in ehci_irq()
819 		ehci->rh_state = EHCI_RH_STOPPING;  in ehci_irq()
820 		ehci->command &= ~(CMD_RUN | CMD_ASE | CMD_PSE);  in ehci_irq()
821 		ehci_writel(ehci, ehci->command, &ehci->regs->command);  in ehci_irq()
822 		ehci_writel(ehci, 0, &ehci->regs->intr_enable);  in ehci_irq()
823 		ehci_handle_controller_death(ehci);  in ehci_irq()
830 		ehci_work (ehci);  in ehci_irq()
831 	spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_irq()
856 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_urb_enqueue()  local
871 		if (!qh_urb_transaction (ehci, urb, &qtd_list, mem_flags))  in ehci_urb_enqueue()
873 		return submit_async(ehci, urb, &qtd_list, mem_flags);  in ehci_urb_enqueue()
876 		if (!qh_urb_transaction (ehci, urb, &qtd_list, mem_flags))  in ehci_urb_enqueue()
878 		return intr_submit(ehci, urb, &qtd_list, mem_flags);  in ehci_urb_enqueue()
882 			return itd_submit (ehci, urb, mem_flags);  in ehci_urb_enqueue()
884 			return sitd_submit (ehci, urb, mem_flags);  in ehci_urb_enqueue()
894 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_urb_dequeue()  local
899 	spin_lock_irqsave (&ehci->lock, flags);  in ehci_urb_dequeue()
916 				start_unlink_intr(ehci, qh);  in ehci_urb_dequeue()
918 				start_unlink_async(ehci, qh);  in ehci_urb_dequeue()
929 			qh_completions(ehci, qh);  in ehci_urb_dequeue()
934 	spin_unlock_irqrestore (&ehci->lock, flags);  in ehci_urb_dequeue()
945 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_endpoint_disable()  local
953 	spin_lock_irqsave (&ehci->lock, flags);  in ehci_endpoint_disable()
968 		reserve_release_iso_bandwidth(ehci, stream, -1);  in ehci_endpoint_disable()
978 			start_unlink_async(ehci, qh);  in ehci_endpoint_disable()
980 			start_unlink_intr(ehci, qh);  in ehci_endpoint_disable()
986 		spin_unlock_irqrestore (&ehci->lock, flags);  in ehci_endpoint_disable()
994 				reserve_release_intr_bandwidth(ehci, qh, -1);  in ehci_endpoint_disable()
995 			qh_destroy(ehci, qh);  in ehci_endpoint_disable()
1003 		ehci_err (ehci, "qh %p (#%02x) state %d%s\n",  in ehci_endpoint_disable()
1010 	spin_unlock_irqrestore (&ehci->lock, flags);  in ehci_endpoint_disable()
1016 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_endpoint_reset()  local
1026 	spin_lock_irqsave(&ehci->lock, flags);  in ehci_endpoint_reset()
1045 				start_unlink_async(ehci, qh);  in ehci_endpoint_reset()
1047 				start_unlink_intr(ehci, qh);  in ehci_endpoint_reset()
1050 	spin_unlock_irqrestore(&ehci->lock, flags);  in ehci_endpoint_reset()
1055 	struct ehci_hcd		*ehci = hcd_to_ehci (hcd);  in ehci_get_frame()  local
1056 	return (ehci_read_frame_index(ehci) >> 3) % ehci->periodic_size;  in ehci_get_frame()
1065 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_remove_device()  local
1067 	spin_lock_irq(&ehci->lock);  in ehci_remove_device()
1069 	spin_unlock_irq(&ehci->lock);  in ehci_remove_device()
1082 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_suspend()  local
1084 	if (time_before(jiffies, ehci->next_statechange))  in ehci_suspend()
1092 	ehci_prepare_ports_for_controller_suspend(ehci, do_wakeup);  in ehci_suspend()
1094 	spin_lock_irq(&ehci->lock);  in ehci_suspend()
1095 	ehci_writel(ehci, 0, &ehci->regs->intr_enable);  in ehci_suspend()
1096 	(void) ehci_readl(ehci, &ehci->regs->intr_enable);  in ehci_suspend()
1099 	spin_unlock_irq(&ehci->lock);  in ehci_suspend()
1116 	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);  in ehci_resume()  local
1118 	if (time_before(jiffies, ehci->next_statechange))  in ehci_resume()
1124 	if (ehci->shutdown)  in ehci_resume()
1132 	if (ehci_readl(ehci, &ehci->regs->configured_flag) == FLAG_CF &&  in ehci_resume()
1136 		ehci_prepare_ports_for_controller_resume(ehci);  in ehci_resume()
1138 		spin_lock_irq(&ehci->lock);  in ehci_resume()
1139 		if (ehci->shutdown)  in ehci_resume()
1144 		ehci_writel(ehci, mask, &ehci->regs->intr_enable);  in ehci_resume()
1145 		ehci_readl(ehci, &ehci->regs->intr_enable);  in ehci_resume()
1147 		spin_unlock_irq(&ehci->lock);  in ehci_resume()
1156 	(void) ehci_halt(ehci);  in ehci_resume()
1157 	(void) ehci_reset(ehci);  in ehci_resume()
1159 	spin_lock_irq(&ehci->lock);  in ehci_resume()
1160 	if (ehci->shutdown)  in ehci_resume()
1163 	ehci_writel(ehci, ehci->command, &ehci->regs->command);  in ehci_resume()
1164 	ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);  in ehci_resume()
1165 	ehci_readl(ehci, &ehci->regs->command);	/* unblock posted writes */  in ehci_resume()
1167 	ehci->rh_state = EHCI_RH_SUSPENDED;  in ehci_resume()
1168 	spin_unlock_irq(&ehci->lock);  in ehci_resume()