268 static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)  in is_ep_periodic()  argument
270 	return hs_ep->periodic;  in is_ep_periodic()
283 				struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_unmap_dma()  argument
292 	usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);  in dwc2_hsotg_unmap_dma()
312 				struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_write_fifo()  argument
315 	bool periodic = is_ep_periodic(hs_ep);  in dwc2_hsotg_write_fifo()
318 	int to_write = hs_ep->size_loaded;  in dwc2_hsotg_write_fifo()
324 	to_write -= (buf_pos - hs_ep->last_load);  in dwc2_hsotg_write_fifo()
331 		u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));  in dwc2_hsotg_write_fifo()
346 		if (hs_ep->fifo_load != 0) {  in dwc2_hsotg_write_fifo()
353 			hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);  in dwc2_hsotg_write_fifo()
356 		size_done = hs_ep->size_loaded - size_left;  in dwc2_hsotg_write_fifo()
359 		can_write = hs_ep->fifo_load - size_done;  in dwc2_hsotg_write_fifo()
363 		can_write = hs_ep->fifo_size - can_write;  in dwc2_hsotg_write_fifo()
371 	} else if (hsotg->dedicated_fifos && hs_ep->index != 0) {  in dwc2_hsotg_write_fifo()
372 		can_write = dwc2_readl(hsotg->regs + DTXFSTS(hs_ep->index));  in dwc2_hsotg_write_fifo()
390 	max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;  in dwc2_hsotg_write_fifo()
454 	hs_ep->total_data += to_write;  in dwc2_hsotg_write_fifo()
457 		hs_ep->fifo_load += to_write;  in dwc2_hsotg_write_fifo()
462 	iowrite32_rep(hsotg->regs + EPFIFO(hs_ep->index), data, to_write);  in dwc2_hsotg_write_fifo()
474 static unsigned get_ep_limit(struct dwc2_hsotg_ep *hs_ep)  in get_ep_limit()  argument
476 	int index = hs_ep->index;  in get_ep_limit()
485 		if (hs_ep->dir_in)  in get_ep_limit()
500 	if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)  in get_ep_limit()
501 		maxsize = maxpkt * hs_ep->ep.maxpacket;  in get_ep_limit()
517 				struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_start_req()  argument
522 	int index = hs_ep->index;  in dwc2_hsotg_start_req()
523 	int dir_in = hs_ep->dir_in;  in dwc2_hsotg_start_req()
533 		if (hs_ep->req && !continuing) {  in dwc2_hsotg_start_req()
537 		} else if (hs_ep->req != hs_req && continuing) {  in dwc2_hsotg_start_req()
550 		hs_ep->dir_in ? "in" : "out");  in dwc2_hsotg_start_req()
564 	maxreq = get_ep_limit(hs_ep);  in dwc2_hsotg_start_req()
566 		int round = maxreq % hs_ep->ep.maxpacket;  in dwc2_hsotg_start_req()
579 		packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);  in dwc2_hsotg_start_req()
583 	if (hs_ep->isochronous && length > (hs_ep->mc * hs_ep->ep.maxpacket)) {  in dwc2_hsotg_start_req()
589 		if (hs_ep->isochronous)  in dwc2_hsotg_start_req()
602 		if ((ureq->length >= hs_ep->ep.maxpacket) &&  in dwc2_hsotg_start_req()
603 					!(ureq->length % hs_ep->ep.maxpacket))  in dwc2_hsotg_start_req()
604 			hs_ep->send_zlp = 1;  in dwc2_hsotg_start_req()
614 	hs_ep->req = hs_req;  in dwc2_hsotg_start_req()
651 	hs_ep->size_loaded = length;  in dwc2_hsotg_start_req()
652 	hs_ep->last_load = ureq->actual;  in dwc2_hsotg_start_req()
656 		hs_ep->fifo_load = 0;  in dwc2_hsotg_start_req()
658 		dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);  in dwc2_hsotg_start_req()
684 	dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);  in dwc2_hsotg_start_req()
700 			     struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_map_dma()  argument
710 	ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);  in dwc2_hsotg_map_dma()
724 	struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req)  in dwc2_hsotg_handle_unaligned_buf_start()  argument
735 			hs_ep->ep.name, req_buf, hs_req->req.length);  in dwc2_hsotg_handle_unaligned_buf_start()
749 	if (hs_ep->dir_in)  in dwc2_hsotg_handle_unaligned_buf_start()
755 	struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req)  in dwc2_hsotg_handle_unaligned_buf_complete()  argument
762 		hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);  in dwc2_hsotg_handle_unaligned_buf_complete()
765 	if (!hs_ep->dir_in && !hs_req->req.status)  in dwc2_hsotg_handle_unaligned_buf_complete()
780 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_queue()  local
781 	struct dwc2_hsotg *hs = hs_ep->parent;  in dwc2_hsotg_ep_queue()
801 	ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);  in dwc2_hsotg_ep_queue()
807 		ret = dwc2_hsotg_map_dma(hs, hs_ep, req);  in dwc2_hsotg_ep_queue()
812 	first = list_empty(&hs_ep->queue);  in dwc2_hsotg_ep_queue()
813 	list_add_tail(&hs_req->queue, &hs_ep->queue);  in dwc2_hsotg_ep_queue()
816 		dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);  in dwc2_hsotg_ep_queue()
824 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_queue_lock()  local
825 	struct dwc2_hsotg *hs = hs_ep->parent;  in dwc2_hsotg_ep_queue_lock()
855 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_complete_oursetup()  local
856 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in dwc2_hsotg_complete_oursetup()
1029 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)  in get_ep_head()  argument
1031 	if (list_empty(&hs_ep->queue))  in get_ep_head()
1034 	return list_first_entry(&hs_ep->queue, struct dwc2_hsotg_req, queue);  in get_ep_head()
1277 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_complete_setup()  local
1278 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in dwc2_hsotg_complete_setup()
1333 					struct dwc2_hsotg_ep *hs_ep)  in dwc2_hsotg_program_zlp()  argument
1336 	u8 index = hs_ep->index;  in dwc2_hsotg_program_zlp()
1337 	u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);  in dwc2_hsotg_program_zlp()
1338 	u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);  in dwc2_hsotg_program_zlp()
1340 	if (hs_ep->dir_in)  in dwc2_hsotg_program_zlp()
1372 				       struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_complete_request()  argument
1384 		hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);  in dwc2_hsotg_complete_request()
1395 		dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);  in dwc2_hsotg_complete_request()
1397 	dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);  in dwc2_hsotg_complete_request()
1399 	hs_ep->req = NULL;  in dwc2_hsotg_complete_request()
1409 		usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);  in dwc2_hsotg_complete_request()
1419 	if (!hs_ep->req && result >= 0) {  in dwc2_hsotg_complete_request()
1420 		restart = !list_empty(&hs_ep->queue);  in dwc2_hsotg_complete_request()
1422 			hs_req = get_ep_head(hs_ep);  in dwc2_hsotg_complete_request()
1423 			dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);  in dwc2_hsotg_complete_request()
1440 	struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];  in dwc2_hsotg_rx_data()  local
1441 	struct dwc2_hsotg_req *hs_req = hs_ep->req;  in dwc2_hsotg_rx_data()
1480 	hs_ep->total_data += to_read;  in dwc2_hsotg_rx_data()
1537 	struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];  in dwc2_hsotg_handle_outdone()  local
1538 	struct dwc2_hsotg_req *hs_req = hs_ep->req;  in dwc2_hsotg_handle_outdone()
1550 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in dwc2_hsotg_handle_outdone()
1567 		size_done = hs_ep->size_loaded - size_left;  in dwc2_hsotg_handle_outdone()
1568 		size_done += hs_ep->last_load;  in dwc2_hsotg_handle_outdone()
1575 		dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);  in dwc2_hsotg_handle_outdone()
1600 		hs_ep->has_correct_parity = 1;  in dwc2_hsotg_handle_outdone()
1601 		if (hs_ep->isochronous && hs_ep->interval == 1)  in dwc2_hsotg_handle_outdone()
1605 	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);  in dwc2_hsotg_handle_outdone()
1742 	struct dwc2_hsotg_ep *hs_ep;  in dwc2_hsotg_set_ep_maxpacket()  local
1748 	hs_ep = index_to_ep(hsotg, ep, dir_in);  in dwc2_hsotg_set_ep_maxpacket()
1749 	if (!hs_ep)  in dwc2_hsotg_set_ep_maxpacket()
1757 		hs_ep->ep.maxpacket = mps;  in dwc2_hsotg_set_ep_maxpacket()
1758 		hs_ep->mc = 1;  in dwc2_hsotg_set_ep_maxpacket()
1764 		hs_ep->mc = mcval;  in dwc2_hsotg_set_ep_maxpacket()
1767 		hs_ep->ep.maxpacket = mpsval;  in dwc2_hsotg_set_ep_maxpacket()
1830 			   struct dwc2_hsotg_ep *hs_ep)  in dwc2_hsotg_trytx()  argument
1832 	struct dwc2_hsotg_req *hs_req = hs_ep->req;  in dwc2_hsotg_trytx()
1834 	if (!hs_ep->dir_in || !hs_req) {  in dwc2_hsotg_trytx()
1839 		if (hs_ep->index != 0)  in dwc2_hsotg_trytx()
1840 			dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,  in dwc2_hsotg_trytx()
1841 					     hs_ep->dir_in, 0);  in dwc2_hsotg_trytx()
1847 			hs_ep->index);  in dwc2_hsotg_trytx()
1848 		return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);  in dwc2_hsotg_trytx()
1863 				  struct dwc2_hsotg_ep *hs_ep)  in dwc2_hsotg_complete_in()  argument
1865 	struct dwc2_hsotg_req *hs_req = hs_ep->req;  in dwc2_hsotg_complete_in()
1866 	u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));  in dwc2_hsotg_complete_in()
1875 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {  in dwc2_hsotg_complete_in()
1877 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in dwc2_hsotg_complete_in()
1905 	size_done = hs_ep->size_loaded - size_left;  in dwc2_hsotg_complete_in()
1906 	size_done += hs_ep->last_load;  in dwc2_hsotg_complete_in()
1918 		dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);  in dwc2_hsotg_complete_in()
1923 	if (hs_ep->send_zlp) {  in dwc2_hsotg_complete_in()
1924 		dwc2_hsotg_program_zlp(hsotg, hs_ep);  in dwc2_hsotg_complete_in()
1925 		hs_ep->send_zlp = 0;  in dwc2_hsotg_complete_in()
1930 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {  in dwc2_hsotg_complete_in()
1936 	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in dwc2_hsotg_complete_in()
1950 	struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);  in dwc2_hsotg_epint()  local
1963 	if (!hs_ep) {  in dwc2_hsotg_epint()
1977 		hs_ep->has_correct_parity = 1;  in dwc2_hsotg_epint()
1978 		if (hs_ep->isochronous && hs_ep->interval == 1)  in dwc2_hsotg_epint()
1991 			dwc2_hsotg_complete_in(hsotg, hs_ep);  in dwc2_hsotg_epint()
1993 			if (idx == 0 && !hs_ep->req)  in dwc2_hsotg_epint()
2011 			dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);  in dwc2_hsotg_epint()
2047 	if (dir_in && !hs_ep->isochronous) {  in dwc2_hsotg_epint()
2066 				dwc2_hsotg_trytx(hsotg, hs_ep);  in dwc2_hsotg_epint()
2603 		struct dwc2_hsotg_ep *hs_ep;  in dwc2_hsotg_irq()  local
2607 			hs_ep = hsotg->eps_in[idx];  in dwc2_hsotg_irq()
2609 			if (!hs_ep->isochronous || hs_ep->has_correct_parity)  in dwc2_hsotg_irq()
2620 		struct dwc2_hsotg_ep *hs_ep;  in dwc2_hsotg_irq()  local
2624 			hs_ep = hsotg->eps_out[idx];  in dwc2_hsotg_irq()
2626 			if (!hs_ep->isochronous || hs_ep->has_correct_parity)  in dwc2_hsotg_irq()
2658 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_enable()  local
2659 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in dwc2_hsotg_ep_enable()
2661 	unsigned int index = hs_ep->index;  in dwc2_hsotg_ep_enable()
2678 	if (dir_in != hs_ep->dir_in) {  in dwc2_hsotg_ep_enable()
2714 	dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, dir_in);  in dwc2_hsotg_ep_enable()
2717 	hs_ep->isochronous = 0;  in dwc2_hsotg_ep_enable()
2718 	hs_ep->periodic = 0;  in dwc2_hsotg_ep_enable()
2719 	hs_ep->halted = 0;  in dwc2_hsotg_ep_enable()
2720 	hs_ep->interval = desc->bInterval;  in dwc2_hsotg_ep_enable()
2721 	hs_ep->has_correct_parity = 0;  in dwc2_hsotg_ep_enable()
2723 	if (hs_ep->interval > 1 && hs_ep->mc > 1)  in dwc2_hsotg_ep_enable()
2730 		hs_ep->isochronous = 1;  in dwc2_hsotg_ep_enable()
2732 			hs_ep->periodic = 1;  in dwc2_hsotg_ep_enable()
2741 			hs_ep->periodic = 1;  in dwc2_hsotg_ep_enable()
2752 	if (hs_ep->fifo_index) {  in dwc2_hsotg_ep_enable()
2753 		size =  hs_ep->ep.maxpacket * hs_ep->mc;  in dwc2_hsotg_ep_enable()
2755 		if (size > hs_ep->fifo_size) {  in dwc2_hsotg_ep_enable()
2756 			hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);  in dwc2_hsotg_ep_enable()
2757 			hs_ep->fifo_index = 0;  in dwc2_hsotg_ep_enable()
2758 			hs_ep->fifo_size = 0;  in dwc2_hsotg_ep_enable()
2766 	if (dir_in && hsotg->dedicated_fifos && !hs_ep->fifo_index) {  in dwc2_hsotg_ep_enable()
2769 		size = hs_ep->ep.maxpacket*hs_ep->mc;  in dwc2_hsotg_ep_enable()
2791 		hs_ep->fifo_index = fifo_index;  in dwc2_hsotg_ep_enable()
2792 		hs_ep->fifo_size = fifo_size;  in dwc2_hsotg_ep_enable()
2820 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_disable()  local
2821 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in dwc2_hsotg_ep_disable()
2822 	int dir_in = hs_ep->dir_in;  in dwc2_hsotg_ep_disable()
2823 	int index = hs_ep->index;  in dwc2_hsotg_ep_disable()
2839 	hsotg->fifo_map &= ~(1<<hs_ep->fifo_index);  in dwc2_hsotg_ep_disable()
2840 	hs_ep->fifo_index = 0;  in dwc2_hsotg_ep_disable()
2841 	hs_ep->fifo_size = 0;  in dwc2_hsotg_ep_disable()
2852 	dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);  in dwc2_hsotg_ep_disable()
2855 	kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);  in dwc2_hsotg_ep_disable()
2893 						struct dwc2_hsotg_ep *hs_ep)  in dwc2_hsotg_ep_stop_xfr()  argument
2898 	epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :  in dwc2_hsotg_ep_stop_xfr()
2899 		DOEPCTL(hs_ep->index);  in dwc2_hsotg_ep_stop_xfr()
2900 	epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :  in dwc2_hsotg_ep_stop_xfr()
2901 		DOEPINT(hs_ep->index);  in dwc2_hsotg_ep_stop_xfr()
2904 			hs_ep->name);  in dwc2_hsotg_ep_stop_xfr()
2905 	if (hs_ep->dir_in) {  in dwc2_hsotg_ep_stop_xfr()
2933 	if (hs_ep->dir_in) {  in dwc2_hsotg_ep_stop_xfr()
2935 			dwc2_writel(GRSTCTL_TXFNUM(hs_ep->fifo_index) |  in dwc2_hsotg_ep_stop_xfr()
2959 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_dequeue()  local
2960 	struct dwc2_hsotg *hs = hs_ep->parent;  in dwc2_hsotg_ep_dequeue()
2967 	if (!on_list(hs_ep, hs_req)) {  in dwc2_hsotg_ep_dequeue()
2973 	if (req == &hs_ep->req->req)  in dwc2_hsotg_ep_dequeue()
2974 		dwc2_hsotg_ep_stop_xfr(hs, hs_ep);  in dwc2_hsotg_ep_dequeue()
2976 	dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);  in dwc2_hsotg_ep_dequeue()
2989 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_sethalt()  local
2990 	struct dwc2_hsotg *hs = hs_ep->parent;  in dwc2_hsotg_ep_sethalt()
2991 	int index = hs_ep->index;  in dwc2_hsotg_ep_sethalt()
3007 	if (hs_ep->dir_in) {  in dwc2_hsotg_ep_sethalt()
3040 	hs_ep->halted = value;  in dwc2_hsotg_ep_sethalt()
3052 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);  in dwc2_hsotg_ep_sethalt_lock()  local
3053 	struct dwc2_hsotg *hs = hs_ep->parent;  in dwc2_hsotg_ep_sethalt_lock()
3335 				       struct dwc2_hsotg_ep *hs_ep,  in dwc2_hsotg_initep()  argument
3348 	hs_ep->dir_in = dir_in;  in dwc2_hsotg_initep()
3349 	hs_ep->index = epnum;  in dwc2_hsotg_initep()
3351 	snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);  in dwc2_hsotg_initep()
3353 	INIT_LIST_HEAD(&hs_ep->queue);  in dwc2_hsotg_initep()
3354 	INIT_LIST_HEAD(&hs_ep->ep.ep_list);  in dwc2_hsotg_initep()
3358 		list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);  in dwc2_hsotg_initep()
3360 	hs_ep->parent = hsotg;  in dwc2_hsotg_initep()
3361 	hs_ep->ep.name = hs_ep->name;  in dwc2_hsotg_initep()
3362 	usb_ep_set_maxpacket_limit(&hs_ep->ep, epnum ? 1024 : EP0_MPS_LIMIT);  in dwc2_hsotg_initep()
3363 	hs_ep->ep.ops = &dwc2_hsotg_ep_ops;  in dwc2_hsotg_initep()
3366 		hs_ep->ep.caps.type_control = true;  in dwc2_hsotg_initep()
3368 		hs_ep->ep.caps.type_iso = true;  in dwc2_hsotg_initep()
3369 		hs_ep->ep.caps.type_bulk = true;  in dwc2_hsotg_initep()
3370 		hs_ep->ep.caps.type_int = true;  in dwc2_hsotg_initep()
3374 		hs_ep->ep.caps.dir_in = true;  in dwc2_hsotg_initep()
3376 		hs_ep->ep.caps.dir_out = true;  in dwc2_hsotg_initep()