274 static inline int is_ep_periodic(struct s3c_hsotg_ep *hs_ep)  in is_ep_periodic()  argument
276 	return hs_ep->periodic;  in is_ep_periodic()
289 				struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_unmap_dma()  argument
298 	usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);  in s3c_hsotg_unmap_dma()
318 				struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_write_fifo()  argument
321 	bool periodic = is_ep_periodic(hs_ep);  in s3c_hsotg_write_fifo()
324 	int to_write = hs_ep->size_loaded;  in s3c_hsotg_write_fifo()
330 	to_write -= (buf_pos - hs_ep->last_load);  in s3c_hsotg_write_fifo()
337 		u32 epsize = readl(hsotg->regs + DIEPTSIZ(hs_ep->index));  in s3c_hsotg_write_fifo()
352 		if (hs_ep->fifo_load != 0) {  in s3c_hsotg_write_fifo()
359 			hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);  in s3c_hsotg_write_fifo()
362 		size_done = hs_ep->size_loaded - size_left;  in s3c_hsotg_write_fifo()
365 		can_write = hs_ep->fifo_load - size_done;  in s3c_hsotg_write_fifo()
369 		can_write = hs_ep->fifo_size - can_write;  in s3c_hsotg_write_fifo()
377 	} else if (hsotg->dedicated_fifos && hs_ep->index != 0) {  in s3c_hsotg_write_fifo()
378 		can_write = readl(hsotg->regs + DTXFSTS(hs_ep->index));  in s3c_hsotg_write_fifo()
396 	max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;  in s3c_hsotg_write_fifo()
460 	hs_ep->total_data += to_write;  in s3c_hsotg_write_fifo()
463 		hs_ep->fifo_load += to_write;  in s3c_hsotg_write_fifo()
468 	iowrite32_rep(hsotg->regs + EPFIFO(hs_ep->index), data, to_write);  in s3c_hsotg_write_fifo()
480 static unsigned get_ep_limit(struct s3c_hsotg_ep *hs_ep)  in get_ep_limit()  argument
482 	int index = hs_ep->index;  in get_ep_limit()
491 		if (hs_ep->dir_in)  in get_ep_limit()
506 	if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)  in get_ep_limit()
507 		maxsize = maxpkt * hs_ep->ep.maxpacket;  in get_ep_limit()
523 				struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_start_req()  argument
528 	int index = hs_ep->index;  in s3c_hsotg_start_req()
529 	int dir_in = hs_ep->dir_in;  in s3c_hsotg_start_req()
539 		if (hs_ep->req && !continuing) {  in s3c_hsotg_start_req()
543 		} else if (hs_ep->req != hs_req && continuing) {  in s3c_hsotg_start_req()
556 		hs_ep->dir_in ? "in" : "out");  in s3c_hsotg_start_req()
570 	maxreq = get_ep_limit(hs_ep);  in s3c_hsotg_start_req()
572 		int round = maxreq % hs_ep->ep.maxpacket;  in s3c_hsotg_start_req()
585 		packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);  in s3c_hsotg_start_req()
589 	if (hs_ep->isochronous && length > (hs_ep->mc * hs_ep->ep.maxpacket)) {  in s3c_hsotg_start_req()
595 		if (hs_ep->isochronous)  in s3c_hsotg_start_req()
608 		if ((ureq->length >= hs_ep->ep.maxpacket) &&  in s3c_hsotg_start_req()
609 					!(ureq->length % hs_ep->ep.maxpacket))  in s3c_hsotg_start_req()
610 			hs_ep->send_zlp = 1;  in s3c_hsotg_start_req()
620 	hs_ep->req = hs_req;  in s3c_hsotg_start_req()
657 	hs_ep->size_loaded = length;  in s3c_hsotg_start_req()
658 	hs_ep->last_load = ureq->actual;  in s3c_hsotg_start_req()
662 		hs_ep->fifo_load = 0;  in s3c_hsotg_start_req()
664 		s3c_hsotg_write_fifo(hsotg, hs_ep, hs_req);  in s3c_hsotg_start_req()
690 	s3c_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);  in s3c_hsotg_start_req()
706 			     struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_map_dma()  argument
716 	ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);  in s3c_hsotg_map_dma()
730 	struct s3c_hsotg_ep *hs_ep, struct s3c_hsotg_req *hs_req)  in s3c_hsotg_handle_unaligned_buf_start()  argument
741 			hs_ep->ep.name, req_buf, hs_req->req.length);  in s3c_hsotg_handle_unaligned_buf_start()
755 	if (hs_ep->dir_in)  in s3c_hsotg_handle_unaligned_buf_start()
761 	struct s3c_hsotg_ep *hs_ep, struct s3c_hsotg_req *hs_req)  in s3c_hsotg_handle_unaligned_buf_complete()  argument
768 		hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);  in s3c_hsotg_handle_unaligned_buf_complete()
771 	if (!hs_ep->dir_in && !hs_req->req.status)  in s3c_hsotg_handle_unaligned_buf_complete()
786 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_queue()  local
787 	struct dwc2_hsotg *hs = hs_ep->parent;  in s3c_hsotg_ep_queue()
800 	ret = s3c_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);  in s3c_hsotg_ep_queue()
806 		ret = s3c_hsotg_map_dma(hs, hs_ep, req);  in s3c_hsotg_ep_queue()
811 	first = list_empty(&hs_ep->queue);  in s3c_hsotg_ep_queue()
812 	list_add_tail(&hs_req->queue, &hs_ep->queue);  in s3c_hsotg_ep_queue()
815 		s3c_hsotg_start_req(hs, hs_ep, hs_req, false);  in s3c_hsotg_ep_queue()
823 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_queue_lock()  local
824 	struct dwc2_hsotg *hs = hs_ep->parent;  in s3c_hsotg_ep_queue_lock()
854 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_complete_oursetup()  local
855 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in s3c_hsotg_complete_oursetup()
1028 static struct s3c_hsotg_req *get_ep_head(struct s3c_hsotg_ep *hs_ep)  in get_ep_head()  argument
1030 	if (list_empty(&hs_ep->queue))  in get_ep_head()
1033 	return list_first_entry(&hs_ep->queue, struct s3c_hsotg_req, queue);  in get_ep_head()
1275 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_complete_setup()  local
1276 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in s3c_hsotg_complete_setup()
1331 					struct s3c_hsotg_ep *hs_ep)  in s3c_hsotg_program_zlp()  argument
1334 	u8 index = hs_ep->index;  in s3c_hsotg_program_zlp()
1335 	u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);  in s3c_hsotg_program_zlp()
1336 	u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);  in s3c_hsotg_program_zlp()
1338 	if (hs_ep->dir_in)  in s3c_hsotg_program_zlp()
1370 				       struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_complete_request()  argument
1382 		hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);  in s3c_hsotg_complete_request()
1392 	s3c_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);  in s3c_hsotg_complete_request()
1394 	hs_ep->req = NULL;  in s3c_hsotg_complete_request()
1398 		s3c_hsotg_unmap_dma(hsotg, hs_ep, hs_req);  in s3c_hsotg_complete_request()
1407 		usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);  in s3c_hsotg_complete_request()
1417 	if (!hs_ep->req && result >= 0) {  in s3c_hsotg_complete_request()
1418 		restart = !list_empty(&hs_ep->queue);  in s3c_hsotg_complete_request()
1420 			hs_req = get_ep_head(hs_ep);  in s3c_hsotg_complete_request()
1421 			s3c_hsotg_start_req(hsotg, hs_ep, hs_req, false);  in s3c_hsotg_complete_request()
1438 	struct s3c_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];  in s3c_hsotg_rx_data()  local
1439 	struct s3c_hsotg_req *hs_req = hs_ep->req;  in s3c_hsotg_rx_data()
1478 	hs_ep->total_data += to_read;  in s3c_hsotg_rx_data()
1522 	struct s3c_hsotg_ep *hs_ep = hsotg->eps_out[epnum];  in s3c_hsotg_handle_outdone()  local
1523 	struct s3c_hsotg_req *hs_req = hs_ep->req;  in s3c_hsotg_handle_outdone()
1535 		s3c_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in s3c_hsotg_handle_outdone()
1552 		size_done = hs_ep->size_loaded - size_left;  in s3c_hsotg_handle_outdone()
1553 		size_done += hs_ep->last_load;  in s3c_hsotg_handle_outdone()
1560 		s3c_hsotg_start_req(hsotg, hs_ep, hs_req, true);  in s3c_hsotg_handle_outdone()
1580 	s3c_hsotg_complete_request(hsotg, hs_ep, hs_req, result);  in s3c_hsotg_handle_outdone()
1717 	struct s3c_hsotg_ep *hs_ep;  in s3c_hsotg_set_ep_maxpacket()  local
1723 	hs_ep = index_to_ep(hsotg, ep, dir_in);  in s3c_hsotg_set_ep_maxpacket()
1724 	if (!hs_ep)  in s3c_hsotg_set_ep_maxpacket()
1732 		hs_ep->ep.maxpacket = mps;  in s3c_hsotg_set_ep_maxpacket()
1733 		hs_ep->mc = 1;  in s3c_hsotg_set_ep_maxpacket()
1739 		hs_ep->mc = mcval;  in s3c_hsotg_set_ep_maxpacket()
1742 		hs_ep->ep.maxpacket = mpsval;  in s3c_hsotg_set_ep_maxpacket()
1805 			   struct s3c_hsotg_ep *hs_ep)  in s3c_hsotg_trytx()  argument
1807 	struct s3c_hsotg_req *hs_req = hs_ep->req;  in s3c_hsotg_trytx()
1809 	if (!hs_ep->dir_in || !hs_req) {  in s3c_hsotg_trytx()
1814 		if (hs_ep->index != 0)  in s3c_hsotg_trytx()
1815 			s3c_hsotg_ctrl_epint(hsotg, hs_ep->index,  in s3c_hsotg_trytx()
1816 					     hs_ep->dir_in, 0);  in s3c_hsotg_trytx()
1822 			hs_ep->index);  in s3c_hsotg_trytx()
1823 		return s3c_hsotg_write_fifo(hsotg, hs_ep, hs_req);  in s3c_hsotg_trytx()
1838 				  struct s3c_hsotg_ep *hs_ep)  in s3c_hsotg_complete_in()  argument
1840 	struct s3c_hsotg_req *hs_req = hs_ep->req;  in s3c_hsotg_complete_in()
1841 	u32 epsize = readl(hsotg->regs + DIEPTSIZ(hs_ep->index));  in s3c_hsotg_complete_in()
1850 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {  in s3c_hsotg_complete_in()
1852 		s3c_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in s3c_hsotg_complete_in()
1880 	size_done = hs_ep->size_loaded - size_left;  in s3c_hsotg_complete_in()
1881 	size_done += hs_ep->last_load;  in s3c_hsotg_complete_in()
1893 		s3c_hsotg_start_req(hsotg, hs_ep, hs_req, true);  in s3c_hsotg_complete_in()
1898 	if (hs_ep->send_zlp) {  in s3c_hsotg_complete_in()
1899 		s3c_hsotg_program_zlp(hsotg, hs_ep);  in s3c_hsotg_complete_in()
1900 		hs_ep->send_zlp = 0;  in s3c_hsotg_complete_in()
1905 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {  in s3c_hsotg_complete_in()
1911 	s3c_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);  in s3c_hsotg_complete_in()
1925 	struct s3c_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);  in s3c_hsotg_epint()  local
1938 	if (!hs_ep) {  in s3c_hsotg_epint()
1952 		if (hs_ep->isochronous && hs_ep->interval == 1) {  in s3c_hsotg_epint()
1970 			s3c_hsotg_complete_in(hsotg, hs_ep);  in s3c_hsotg_epint()
1972 			if (idx == 0 && !hs_ep->req)  in s3c_hsotg_epint()
1990 			s3c_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);  in s3c_hsotg_epint()
2026 	if (dir_in && !hs_ep->isochronous) {  in s3c_hsotg_epint()
2045 				s3c_hsotg_trytx(hsotg, hs_ep);  in s3c_hsotg_epint()
2589 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_enable()  local
2590 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in s3c_hsotg_ep_enable()
2592 	unsigned int index = hs_ep->index;  in s3c_hsotg_ep_enable()
2609 	if (dir_in != hs_ep->dir_in) {  in s3c_hsotg_ep_enable()
2645 	s3c_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, dir_in);  in s3c_hsotg_ep_enable()
2648 	hs_ep->isochronous = 0;  in s3c_hsotg_ep_enable()
2649 	hs_ep->periodic = 0;  in s3c_hsotg_ep_enable()
2650 	hs_ep->halted = 0;  in s3c_hsotg_ep_enable()
2651 	hs_ep->interval = desc->bInterval;  in s3c_hsotg_ep_enable()
2653 	if (hs_ep->interval > 1 && hs_ep->mc > 1)  in s3c_hsotg_ep_enable()
2660 		hs_ep->isochronous = 1;  in s3c_hsotg_ep_enable()
2662 			hs_ep->periodic = 1;  in s3c_hsotg_ep_enable()
2671 			hs_ep->periodic = 1;  in s3c_hsotg_ep_enable()
2682 	if (hs_ep->fifo_index) {  in s3c_hsotg_ep_enable()
2683 		size =  hs_ep->ep.maxpacket * hs_ep->mc;  in s3c_hsotg_ep_enable()
2685 		if (size > hs_ep->fifo_size) {  in s3c_hsotg_ep_enable()
2686 			hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);  in s3c_hsotg_ep_enable()
2687 			hs_ep->fifo_index = 0;  in s3c_hsotg_ep_enable()
2688 			hs_ep->fifo_size = 0;  in s3c_hsotg_ep_enable()
2696 	if (dir_in && hsotg->dedicated_fifos && !hs_ep->fifo_index) {  in s3c_hsotg_ep_enable()
2699 		size = hs_ep->ep.maxpacket*hs_ep->mc;  in s3c_hsotg_ep_enable()
2721 		hs_ep->fifo_index = fifo_index;  in s3c_hsotg_ep_enable()
2722 		hs_ep->fifo_size = fifo_size;  in s3c_hsotg_ep_enable()
2750 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_disable_force()  local
2751 	struct dwc2_hsotg *hsotg = hs_ep->parent;  in s3c_hsotg_ep_disable_force()
2752 	int dir_in = hs_ep->dir_in;  in s3c_hsotg_ep_disable_force()
2753 	int index = hs_ep->index;  in s3c_hsotg_ep_disable_force()
2769 	hsotg->fifo_map &= ~(1<<hs_ep->fifo_index);  in s3c_hsotg_ep_disable_force()
2770 	hs_ep->fifo_index = 0;  in s3c_hsotg_ep_disable_force()
2771 	hs_ep->fifo_size = 0;  in s3c_hsotg_ep_disable_force()
2782 	s3c_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);  in s3c_hsotg_ep_disable_force()
2785 	kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);  in s3c_hsotg_ep_disable_force()
2820 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_dequeue()  local
2821 	struct dwc2_hsotg *hs = hs_ep->parent;  in s3c_hsotg_ep_dequeue()
2828 	if (!on_list(hs_ep, hs_req)) {  in s3c_hsotg_ep_dequeue()
2833 	s3c_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);  in s3c_hsotg_ep_dequeue()
2846 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_sethalt()  local
2847 	struct dwc2_hsotg *hs = hs_ep->parent;  in s3c_hsotg_ep_sethalt()
2848 	int index = hs_ep->index;  in s3c_hsotg_ep_sethalt()
2864 	if (hs_ep->dir_in) {  in s3c_hsotg_ep_sethalt()
2897 	hs_ep->halted = value;  in s3c_hsotg_ep_sethalt()
2909 	struct s3c_hsotg_ep *hs_ep = our_ep(ep);  in s3c_hsotg_ep_sethalt_lock()  local
2910 	struct dwc2_hsotg *hs = hs_ep->parent;  in s3c_hsotg_ep_sethalt_lock()
3240 				       struct s3c_hsotg_ep *hs_ep,  in s3c_hsotg_initep()  argument
3253 	hs_ep->dir_in = dir_in;  in s3c_hsotg_initep()
3254 	hs_ep->index = epnum;  in s3c_hsotg_initep()
3256 	snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);  in s3c_hsotg_initep()
3258 	INIT_LIST_HEAD(&hs_ep->queue);  in s3c_hsotg_initep()
3259 	INIT_LIST_HEAD(&hs_ep->ep.ep_list);  in s3c_hsotg_initep()
3263 		list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);  in s3c_hsotg_initep()
3265 	hs_ep->parent = hsotg;  in s3c_hsotg_initep()
3266 	hs_ep->ep.name = hs_ep->name;  in s3c_hsotg_initep()
3267 	usb_ep_set_maxpacket_limit(&hs_ep->ep, epnum ? 1024 : EP0_MPS_LIMIT);  in s3c_hsotg_initep()
3268 	hs_ep->ep.ops = &s3c_hsotg_ep_ops;  in s3c_hsotg_initep()