194 		struct dwc3_ep	*dep = dwc->eps[(num << 1) | 1];  in dwc3_gadget_resize_tx_fifos()  local
198 		if (!(dep->flags & DWC3_EP_ENABLED))  in dwc3_gadget_resize_tx_fifos()
201 		if (usb_endpoint_xfer_bulk(dep->endpoint.desc)  in dwc3_gadget_resize_tx_fifos()
202 				|| usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_gadget_resize_tx_fifos()
216 		tmp = mult * (dep->endpoint.maxpacket + mdwidth);  in dwc3_gadget_resize_tx_fifos()
224 				dep->name, last_fifo_depth, fifo_size & 0xffff);  in dwc3_gadget_resize_tx_fifos()
234 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,  in dwc3_gadget_giveback()  argument
237 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_giveback()
243 			dep->busy_slot++;  in dwc3_gadget_giveback()
249 			if (((dep->busy_slot & DWC3_TRB_MASK) ==  in dwc3_gadget_giveback()
251 				usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_gadget_giveback()
252 				dep->busy_slot++;  in dwc3_gadget_giveback()
262 	if (dwc->ep0_bounced && dep->number == 0)  in dwc3_gadget_giveback()
269 			req, dep->name, req->request.actual,  in dwc3_gadget_giveback()
274 	usb_gadget_giveback_request(&dep->endpoint, &req->request);  in dwc3_gadget_giveback()
316 	struct dwc3_ep		*dep = dwc->eps[ep];  in dwc3_send_gadget_ep_cmd()  local
320 	trace_dwc3_gadget_ep_cmd(dep, cmd, params);  in dwc3_send_gadget_ep_cmd()
353 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,  in dwc3_trb_dma_offset()  argument
356 	u32		offset = (char *) trb - (char *) dep->trb_pool;  in dwc3_trb_dma_offset()
358 	return dep->trb_pool_dma + offset;  in dwc3_trb_dma_offset()
361 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)  in dwc3_alloc_trb_pool()  argument
363 	struct dwc3		*dwc = dep->dwc;  in dwc3_alloc_trb_pool()
365 	if (dep->trb_pool)  in dwc3_alloc_trb_pool()
368 	dep->trb_pool = dma_alloc_coherent(dwc->dev,  in dwc3_alloc_trb_pool()
370 			&dep->trb_pool_dma, GFP_KERNEL);  in dwc3_alloc_trb_pool()
371 	if (!dep->trb_pool) {  in dwc3_alloc_trb_pool()
372 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",  in dwc3_alloc_trb_pool()
373 				dep->name);  in dwc3_alloc_trb_pool()
380 static void dwc3_free_trb_pool(struct dwc3_ep *dep)  in dwc3_free_trb_pool()  argument
382 	struct dwc3		*dwc = dep->dwc;  in dwc3_free_trb_pool()
385 			dep->trb_pool, dep->trb_pool_dma);  in dwc3_free_trb_pool()
387 	dep->trb_pool = NULL;  in dwc3_free_trb_pool()
388 	dep->trb_pool_dma = 0;  in dwc3_free_trb_pool()
391 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
425 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)  in dwc3_gadget_start_config()  argument
432 	if (dep->number)  in dwc3_gadget_start_config()
443 		struct dwc3_ep *dep = dwc->eps[i];  in dwc3_gadget_start_config()  local
445 		if (!dep)  in dwc3_gadget_start_config()
448 		ret = dwc3_gadget_set_xfer_resource(dwc, dep);  in dwc3_gadget_start_config()
456 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,  in dwc3_gadget_set_ep_config()  argument
470 		u32 burst = dep->endpoint.maxburst - 1;  in dwc3_gadget_set_ep_config()
480 		params.param2 |= dep->saved_state;  in dwc3_gadget_set_ep_config()
489 		dep->stream_capable = true;  in dwc3_gadget_set_ep_config()
501 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);  in dwc3_gadget_set_ep_config()
507 	if (dep->direction)  in dwc3_gadget_set_ep_config()
508 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);  in dwc3_gadget_set_ep_config()
512 		dep->interval = 1 << (desc->bInterval - 1);  in dwc3_gadget_set_ep_config()
515 	return dwc3_send_gadget_ep_cmd(dwc, dep->number,  in dwc3_gadget_set_ep_config()
519 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)  in dwc3_gadget_set_xfer_resource()  argument
527 	return dwc3_send_gadget_ep_cmd(dwc, dep->number,  in dwc3_gadget_set_xfer_resource()
538 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,  in __dwc3_gadget_ep_enable()  argument
543 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_enable()
547 	dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);  in __dwc3_gadget_ep_enable()
549 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in __dwc3_gadget_ep_enable()
550 		ret = dwc3_gadget_start_config(dwc, dep);  in __dwc3_gadget_ep_enable()
555 	ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,  in __dwc3_gadget_ep_enable()
560 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in __dwc3_gadget_ep_enable()
564 		dep->endpoint.desc = desc;  in __dwc3_gadget_ep_enable()
565 		dep->comp_desc = comp_desc;  in __dwc3_gadget_ep_enable()
566 		dep->type = usb_endpoint_type(desc);  in __dwc3_gadget_ep_enable()
567 		dep->flags |= DWC3_EP_ENABLED;  in __dwc3_gadget_ep_enable()
570 		reg |= DWC3_DALEPENA_EP(dep->number);  in __dwc3_gadget_ep_enable()
577 		trb_st_hw = &dep->trb_pool[0];  in __dwc3_gadget_ep_enable()
579 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];  in __dwc3_gadget_ep_enable()
582 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));  in __dwc3_gadget_ep_enable()
583 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));  in __dwc3_gadget_ep_enable()
590 		strlcat(dep->name, "-control", sizeof(dep->name));  in __dwc3_gadget_ep_enable()
593 		strlcat(dep->name, "-isoc", sizeof(dep->name));  in __dwc3_gadget_ep_enable()
596 		strlcat(dep->name, "-bulk", sizeof(dep->name));  in __dwc3_gadget_ep_enable()
599 		strlcat(dep->name, "-int", sizeof(dep->name));  in __dwc3_gadget_ep_enable()
609 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)  in dwc3_remove_requests()  argument
613 	if (!list_empty(&dep->req_queued)) {  in dwc3_remove_requests()
614 		dwc3_stop_active_transfer(dwc, dep->number, true);  in dwc3_remove_requests()
617 		while (!list_empty(&dep->req_queued)) {  in dwc3_remove_requests()
618 			req = next_request(&dep->req_queued);  in dwc3_remove_requests()
620 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);  in dwc3_remove_requests()
624 	while (!list_empty(&dep->request_list)) {  in dwc3_remove_requests()
625 		req = next_request(&dep->request_list);  in dwc3_remove_requests()
627 		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);  in dwc3_remove_requests()
639 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)  in __dwc3_gadget_ep_disable()  argument
641 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_disable()
644 	dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);  in __dwc3_gadget_ep_disable()
646 	dwc3_remove_requests(dwc, dep);  in __dwc3_gadget_ep_disable()
649 	if (dep->flags & DWC3_EP_STALL)  in __dwc3_gadget_ep_disable()
650 		__dwc3_gadget_ep_set_halt(dep, 0, false);  in __dwc3_gadget_ep_disable()
653 	reg &= ~DWC3_DALEPENA_EP(dep->number);  in __dwc3_gadget_ep_disable()
656 	dep->stream_capable = false;  in __dwc3_gadget_ep_disable()
657 	dep->endpoint.desc = NULL;  in __dwc3_gadget_ep_disable()
658 	dep->comp_desc = NULL;  in __dwc3_gadget_ep_disable()
659 	dep->type = 0;  in __dwc3_gadget_ep_disable()
660 	dep->flags = 0;  in __dwc3_gadget_ep_disable()
662 	snprintf(dep->name, sizeof(dep->name), "ep%d%s",  in __dwc3_gadget_ep_disable()
663 			dep->number >> 1,  in __dwc3_gadget_ep_disable()
664 			(dep->number & 1) ? "in" : "out");  in __dwc3_gadget_ep_disable()
687 	struct dwc3_ep			*dep;  in dwc3_gadget_ep_enable()  local
702 	dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_enable()
703 	dwc = dep->dwc;  in dwc3_gadget_ep_enable()
705 	if (dep->flags & DWC3_EP_ENABLED) {  in dwc3_gadget_ep_enable()
707 				dep->name);  in dwc3_gadget_ep_enable()
712 	ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);  in dwc3_gadget_ep_enable()
720 	struct dwc3_ep			*dep;  in dwc3_gadget_ep_disable()  local
730 	dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_disable()
731 	dwc = dep->dwc;  in dwc3_gadget_ep_disable()
733 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in dwc3_gadget_ep_disable()
735 				dep->name);  in dwc3_gadget_ep_disable()
740 	ret = __dwc3_gadget_ep_disable(dep);  in dwc3_gadget_ep_disable()
750 	struct dwc3_ep			*dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_alloc_request()  local
756 	req->epnum	= dep->number;  in dwc3_gadget_ep_alloc_request()
757 	req->dep	= dep;  in dwc3_gadget_ep_alloc_request()
778 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,  in dwc3_prepare_one_trb()  argument
785 			dep->name, req, (unsigned long long) dma,  in dwc3_prepare_one_trb()
790 	trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];  in dwc3_prepare_one_trb()
795 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);  in dwc3_prepare_one_trb()
796 		req->start_slot = dep->free_slot & DWC3_TRB_MASK;  in dwc3_prepare_one_trb()
799 	dep->free_slot++;  in dwc3_prepare_one_trb()
801 	if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&  in dwc3_prepare_one_trb()
802 			usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_prepare_one_trb()
803 		dep->free_slot++;  in dwc3_prepare_one_trb()
809 	switch (usb_endpoint_type(dep->endpoint.desc)) {  in dwc3_prepare_one_trb()
836 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_prepare_one_trb()
846 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)  in dwc3_prepare_one_trb()
851 	trace_dwc3_prepare_trb(dep, trb);  in dwc3_prepare_one_trb()
863 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)  in dwc3_prepare_trbs()  argument
873 	trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;  in dwc3_prepare_trbs()
876 	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_prepare_trbs()
877 		max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);  in dwc3_prepare_trbs()
902 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_prepare_trbs()
903 			dep->busy_slot = 1;  in dwc3_prepare_trbs()
904 			dep->free_slot = 1;  in dwc3_prepare_trbs()
906 			dep->busy_slot = 0;  in dwc3_prepare_trbs()
907 			dep->free_slot = 0;  in dwc3_prepare_trbs()
912 	if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_prepare_trbs()
915 	list_for_each_entry_safe(req, n, &dep->request_list, list) {  in dwc3_prepare_trbs()
934 					if (list_empty(&dep->request_list))  in dwc3_prepare_trbs()
946 				dwc3_prepare_one_trb(dep, req, dma, length,  in dwc3_prepare_trbs()
964 			if (list_is_last(&req->list, &dep->request_list))  in dwc3_prepare_trbs()
967 			dwc3_prepare_one_trb(dep, req, dma, length,  in dwc3_prepare_trbs()
976 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,  in __dwc3_gadget_kick_transfer()  argument
981 	struct dwc3			*dwc = dep->dwc;  in __dwc3_gadget_kick_transfer()
985 	if (start_new && (dep->flags & DWC3_EP_BUSY)) {  in __dwc3_gadget_kick_transfer()
986 		dwc3_trace(trace_dwc3_gadget, "%s: endpoint busy", dep->name);  in __dwc3_gadget_kick_transfer()
995 		if (list_empty(&dep->req_queued))  in __dwc3_gadget_kick_transfer()
996 			dwc3_prepare_trbs(dep, start_new);  in __dwc3_gadget_kick_transfer()
999 		req = next_request(&dep->req_queued);  in __dwc3_gadget_kick_transfer()
1001 		dwc3_prepare_trbs(dep, start_new);  in __dwc3_gadget_kick_transfer()
1006 		req = next_request(&dep->req_queued);  in __dwc3_gadget_kick_transfer()
1009 		dep->flags |= DWC3_EP_PENDING_REQUEST;  in __dwc3_gadget_kick_transfer()
1024 	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);  in __dwc3_gadget_kick_transfer()
1039 	dep->flags |= DWC3_EP_BUSY;  in __dwc3_gadget_kick_transfer()
1042 		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,  in __dwc3_gadget_kick_transfer()
1043 				dep->number);  in __dwc3_gadget_kick_transfer()
1044 		WARN_ON_ONCE(!dep->resource_index);  in __dwc3_gadget_kick_transfer()
1051 		struct dwc3_ep *dep, u32 cur_uf)  in __dwc3_gadget_start_isoc()  argument
1055 	if (list_empty(&dep->request_list)) {  in __dwc3_gadget_start_isoc()
1058 				dep->name);  in __dwc3_gadget_start_isoc()
1059 		dep->flags |= DWC3_EP_PENDING_REQUEST;  in __dwc3_gadget_start_isoc()
1064 	uf = cur_uf + dep->interval * 4;  in __dwc3_gadget_start_isoc()
1066 	__dwc3_gadget_kick_transfer(dep, uf, 1);  in __dwc3_gadget_start_isoc()
1070 		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)  in dwc3_gadget_start_isoc()  argument
1074 	mask = ~(dep->interval - 1);  in dwc3_gadget_start_isoc()
1077 	__dwc3_gadget_start_isoc(dwc, dep, cur_uf);  in dwc3_gadget_start_isoc()
1080 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)  in __dwc3_gadget_ep_queue()  argument
1082 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_queue()
1087 	req->direction		= dep->direction;  in __dwc3_gadget_ep_queue()
1088 	req->epnum		= dep->number;  in __dwc3_gadget_ep_queue()
1105 			dep->direction);  in __dwc3_gadget_ep_queue()
1109 	list_add_tail(&req->list, &dep->request_list);  in __dwc3_gadget_ep_queue()
1118 	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&  in __dwc3_gadget_ep_queue()
1119 			!usb_endpoint_xfer_int(dep->endpoint.desc) &&  in __dwc3_gadget_ep_queue()
1120 			!(dep->flags & DWC3_EP_BUSY)) {  in __dwc3_gadget_ep_queue()
1121 		ret = __dwc3_gadget_kick_transfer(dep, 0, true);  in __dwc3_gadget_ep_queue()
1136 	if (dep->flags & DWC3_EP_PENDING_REQUEST) {  in __dwc3_gadget_ep_queue()
1143 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in __dwc3_gadget_ep_queue()
1144 			if (list_empty(&dep->req_queued)) {  in __dwc3_gadget_ep_queue()
1145 				dwc3_stop_active_transfer(dwc, dep->number, true);  in __dwc3_gadget_ep_queue()
1146 				dep->flags = DWC3_EP_ENABLED;  in __dwc3_gadget_ep_queue()
1151 		ret = __dwc3_gadget_kick_transfer(dep, 0, true);  in __dwc3_gadget_ep_queue()
1153 			dep->flags &= ~DWC3_EP_PENDING_REQUEST;  in __dwc3_gadget_ep_queue()
1163 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&  in __dwc3_gadget_ep_queue()
1164 			(dep->flags & DWC3_EP_BUSY) &&  in __dwc3_gadget_ep_queue()
1165 			!(dep->flags & DWC3_EP_MISSED_ISOC)) {  in __dwc3_gadget_ep_queue()
1166 		WARN_ON_ONCE(!dep->resource_index);  in __dwc3_gadget_ep_queue()
1167 		ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,  in __dwc3_gadget_ep_queue()
1177 	if (dep->stream_capable)  in __dwc3_gadget_ep_queue()
1178 		ret = __dwc3_gadget_kick_transfer(dep, 0, true);  in __dwc3_gadget_ep_queue()
1183 				dep->name);  in __dwc3_gadget_ep_queue()
1194 	struct dwc3_ep			*dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_queue()  local
1195 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_queue()
1202 	if (!dep->endpoint.desc) {  in dwc3_gadget_ep_queue()
1209 	if (WARN(req->dep != dep, "request %p belongs to '%s'\n",  in dwc3_gadget_ep_queue()
1210 				request, req->dep->name)) {  in dwc3_gadget_ep_queue()
1215 	ret = __dwc3_gadget_ep_queue(dep, req);  in dwc3_gadget_ep_queue()
1229 	struct dwc3_ep			*dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_dequeue()  local
1230 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_dequeue()
1239 	list_for_each_entry(r, &dep->request_list, list) {  in dwc3_gadget_ep_dequeue()
1245 		list_for_each_entry(r, &dep->req_queued, list) {  in dwc3_gadget_ep_dequeue()
1251 			dwc3_stop_active_transfer(dwc, dep->number, true);  in dwc3_gadget_ep_dequeue()
1262 	dwc3_gadget_giveback(dep, req, -ECONNRESET);  in dwc3_gadget_ep_dequeue()
1270 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)  in __dwc3_gadget_ep_set_halt()  argument
1273 	struct dwc3				*dwc = dep->dwc;  in __dwc3_gadget_ep_set_halt()
1276 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in __dwc3_gadget_ep_set_halt()
1277 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);  in __dwc3_gadget_ep_set_halt()
1284 		if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||  in __dwc3_gadget_ep_set_halt()
1285 				(!list_empty(&dep->req_queued) ||  in __dwc3_gadget_ep_set_halt()
1286 				 !list_empty(&dep->request_list)))) {  in __dwc3_gadget_ep_set_halt()
1288 					dep->name);  in __dwc3_gadget_ep_set_halt()
1292 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,  in __dwc3_gadget_ep_set_halt()
1296 					dep->name);  in __dwc3_gadget_ep_set_halt()
1298 			dep->flags |= DWC3_EP_STALL;  in __dwc3_gadget_ep_set_halt()
1300 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,  in __dwc3_gadget_ep_set_halt()
1304 					dep->name);  in __dwc3_gadget_ep_set_halt()
1306 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);  in __dwc3_gadget_ep_set_halt()
1314 	struct dwc3_ep			*dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_set_halt()  local
1315 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_set_halt()
1322 	ret = __dwc3_gadget_ep_set_halt(dep, value, false);  in dwc3_gadget_ep_set_halt()
1330 	struct dwc3_ep			*dep = to_dwc3_ep(ep);  in dwc3_gadget_ep_set_wedge()  local
1331 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_set_wedge()
1336 	dep->flags |= DWC3_EP_WEDGE;  in dwc3_gadget_ep_set_wedge()
1338 	if (dep->number == 0 || dep->number == 1)  in dwc3_gadget_ep_set_wedge()
1341 		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);  in dwc3_gadget_ep_set_wedge()
1581 	struct dwc3_ep		*dep;  in dwc3_gadget_start()  local
1648 	dep = dwc->eps[0];  in dwc3_gadget_start()
1649 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,  in dwc3_gadget_start()
1652 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_start()
1656 	dep = dwc->eps[1];  in dwc3_gadget_start()
1657 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,  in dwc3_gadget_start()
1660 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_start()
1725 	struct dwc3_ep			*dep;  in dwc3_gadget_init_hw_endpoints()  local
1731 		dep = kzalloc(sizeof(*dep), GFP_KERNEL);  in dwc3_gadget_init_hw_endpoints()
1732 		if (!dep)  in dwc3_gadget_init_hw_endpoints()
1735 		dep->dwc = dwc;  in dwc3_gadget_init_hw_endpoints()
1736 		dep->number = epnum;  in dwc3_gadget_init_hw_endpoints()
1737 		dep->direction = !!direction;  in dwc3_gadget_init_hw_endpoints()
1738 		dwc->eps[epnum] = dep;  in dwc3_gadget_init_hw_endpoints()
1740 		snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,  in dwc3_gadget_init_hw_endpoints()
1743 		dep->endpoint.name = dep->name;  in dwc3_gadget_init_hw_endpoints()
1745 		dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);  in dwc3_gadget_init_hw_endpoints()
1748 			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);  in dwc3_gadget_init_hw_endpoints()
1749 			dep->endpoint.maxburst = 1;  in dwc3_gadget_init_hw_endpoints()
1750 			dep->endpoint.ops = &dwc3_gadget_ep0_ops;  in dwc3_gadget_init_hw_endpoints()
1752 				dwc->gadget.ep0 = &dep->endpoint;  in dwc3_gadget_init_hw_endpoints()
1756 			usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);  in dwc3_gadget_init_hw_endpoints()
1757 			dep->endpoint.max_streams = 15;  in dwc3_gadget_init_hw_endpoints()
1758 			dep->endpoint.ops = &dwc3_gadget_ep_ops;  in dwc3_gadget_init_hw_endpoints()
1759 			list_add_tail(&dep->endpoint.ep_list,  in dwc3_gadget_init_hw_endpoints()
1762 			ret = dwc3_alloc_trb_pool(dep);  in dwc3_gadget_init_hw_endpoints()
1768 			dep->endpoint.caps.type_control = true;  in dwc3_gadget_init_hw_endpoints()
1770 			dep->endpoint.caps.type_iso = true;  in dwc3_gadget_init_hw_endpoints()
1771 			dep->endpoint.caps.type_bulk = true;  in dwc3_gadget_init_hw_endpoints()
1772 			dep->endpoint.caps.type_int = true;  in dwc3_gadget_init_hw_endpoints()
1775 		dep->endpoint.caps.dir_in = !!direction;  in dwc3_gadget_init_hw_endpoints()
1776 		dep->endpoint.caps.dir_out = !direction;  in dwc3_gadget_init_hw_endpoints()
1778 		INIT_LIST_HEAD(&dep->request_list);  in dwc3_gadget_init_hw_endpoints()
1779 		INIT_LIST_HEAD(&dep->req_queued);  in dwc3_gadget_init_hw_endpoints()
1810 	struct dwc3_ep			*dep;  in dwc3_gadget_free_endpoints()  local
1814 		dep = dwc->eps[epnum];  in dwc3_gadget_free_endpoints()
1815 		if (!dep)  in dwc3_gadget_free_endpoints()
1827 			dwc3_free_trb_pool(dep);  in dwc3_gadget_free_endpoints()
1828 			list_del(&dep->endpoint.ep_list);  in dwc3_gadget_free_endpoints()
1831 		kfree(dep);  in dwc3_gadget_free_endpoints()
1837 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,  in __dwc3_cleanup_done_trbs()  argument
1845 	trace_dwc3_complete_trb(dep, trb);  in __dwc3_cleanup_done_trbs()
1857 				dep->name, trb);  in __dwc3_cleanup_done_trbs()
1860 	if (dep->direction) {  in __dwc3_cleanup_done_trbs()
1865 						dep->name);  in __dwc3_cleanup_done_trbs()
1881 				dep->flags |= DWC3_EP_MISSED_ISOC;  in __dwc3_cleanup_done_trbs()
1884 						dep->name);  in __dwc3_cleanup_done_trbs()
1888 			dep->flags &= ~DWC3_EP_MISSED_ISOC;  in __dwc3_cleanup_done_trbs()
1915 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,  in dwc3_cleanup_done_reqs()  argument
1925 		req = next_request(&dep->req_queued);  in dwc3_cleanup_done_reqs()
1934 				usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_cleanup_done_reqs()
1937 			trb = &dep->trb_pool[slot];  in dwc3_cleanup_done_reqs()
1939 			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,  in dwc3_cleanup_done_reqs()
1945 		dwc3_gadget_giveback(dep, req, status);  in dwc3_cleanup_done_reqs()
1951 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&  in dwc3_cleanup_done_reqs()
1952 			list_empty(&dep->req_queued)) {  in dwc3_cleanup_done_reqs()
1953 		if (list_empty(&dep->request_list)) {  in dwc3_cleanup_done_reqs()
1960 			dep->flags = DWC3_EP_PENDING_REQUEST;  in dwc3_cleanup_done_reqs()
1962 			dwc3_stop_active_transfer(dwc, dep->number, true);  in dwc3_cleanup_done_reqs()
1963 			dep->flags = DWC3_EP_ENABLED;  in dwc3_cleanup_done_reqs()
1972 		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)  in dwc3_endpoint_transfer_complete()  argument
1983 	clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);  in dwc3_endpoint_transfer_complete()
1985 				usb_endpoint_xfer_isoc(dep->endpoint.desc)))  in dwc3_endpoint_transfer_complete()
1986 		dep->flags &= ~DWC3_EP_BUSY;  in dwc3_endpoint_transfer_complete()
1997 			dep = dwc->eps[i];  in dwc3_endpoint_transfer_complete()
1999 			if (!(dep->flags & DWC3_EP_ENABLED))  in dwc3_endpoint_transfer_complete()
2002 			if (!list_empty(&dep->req_queued))  in dwc3_endpoint_transfer_complete()
2013 	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_endpoint_transfer_complete()
2016 		ret = __dwc3_gadget_kick_transfer(dep, 0, is_xfer_complete);  in dwc3_endpoint_transfer_complete()
2025 	struct dwc3_ep		*dep;  in dwc3_endpoint_interrupt()  local
2028 	dep = dwc->eps[epnum];  in dwc3_endpoint_interrupt()
2030 	if (!(dep->flags & DWC3_EP_ENABLED))  in dwc3_endpoint_interrupt()
2040 		dep->resource_index = 0;  in dwc3_endpoint_interrupt()
2042 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_endpoint_interrupt()
2044 					dep->name);  in dwc3_endpoint_interrupt()
2048 		dwc3_endpoint_transfer_complete(dwc, dep, event);  in dwc3_endpoint_interrupt()
2051 		dwc3_endpoint_transfer_complete(dwc, dep, event);  in dwc3_endpoint_interrupt()
2054 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in dwc3_endpoint_interrupt()
2055 			dwc3_gadget_start_isoc(dwc, dep, event);  in dwc3_endpoint_interrupt()
2063 					dep->name, active ? "Transfer Active"  in dwc3_endpoint_interrupt()
2066 			ret = __dwc3_gadget_kick_transfer(dep, 0, !active);  in dwc3_endpoint_interrupt()
2071 					dep->name);  in dwc3_endpoint_interrupt()
2076 		if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {  in dwc3_endpoint_interrupt()
2078 					dep->name);  in dwc3_endpoint_interrupt()
2096 		dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);  in dwc3_endpoint_interrupt()
2145 	struct dwc3_ep *dep;  in dwc3_stop_active_transfer()  local
2150 	dep = dwc->eps[epnum];  in dwc3_stop_active_transfer()
2152 	if (!dep->resource_index)  in dwc3_stop_active_transfer()
2177 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);  in dwc3_stop_active_transfer()
2179 	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);  in dwc3_stop_active_transfer()
2181 	dep->resource_index = 0;  in dwc3_stop_active_transfer()
2182 	dep->flags &= ~DWC3_EP_BUSY;  in dwc3_stop_active_transfer()
2191 		struct dwc3_ep *dep;  in dwc3_stop_active_transfers()  local
2193 		dep = dwc->eps[epnum];  in dwc3_stop_active_transfers()
2194 		if (!dep)  in dwc3_stop_active_transfers()
2197 		if (!(dep->flags & DWC3_EP_ENABLED))  in dwc3_stop_active_transfers()
2200 		dwc3_remove_requests(dwc, dep);  in dwc3_stop_active_transfers()
2209 		struct dwc3_ep *dep;  in dwc3_clear_stall_all_ep()  local
2213 		dep = dwc->eps[epnum];  in dwc3_clear_stall_all_ep()
2214 		if (!dep)  in dwc3_clear_stall_all_ep()
2217 		if (!(dep->flags & DWC3_EP_STALL))  in dwc3_clear_stall_all_ep()
2220 		dep->flags &= ~DWC3_EP_STALL;  in dwc3_clear_stall_all_ep()
2223 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,  in dwc3_clear_stall_all_ep()
2325 	struct dwc3_ep		*dep;  in dwc3_gadget_conndone_interrupt()  local
2409 	dep = dwc->eps[0];  in dwc3_gadget_conndone_interrupt()
2410 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,  in dwc3_gadget_conndone_interrupt()
2413 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_conndone_interrupt()
2417 	dep = dwc->eps[1];  in dwc3_gadget_conndone_interrupt()
2418 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,  in dwc3_gadget_conndone_interrupt()
2421 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_conndone_interrupt()
2889 	struct dwc3_ep		*dep;  in dwc3_gadget_resume()  local
2895 	dep = dwc->eps[0];  in dwc3_gadget_resume()
2896 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,  in dwc3_gadget_resume()
2901 	dep = dwc->eps[1];  in dwc3_gadget_resume()
2902 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,  in dwc3_gadget_resume()