96 				struct gr_request *req)  in gr_dbgprint_request()  argument
98 	int buflen = ep->is_in ? req->req.length : req->req.actual;  in gr_dbgprint_request()
102 	dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,  in gr_dbgprint_request()
105 			     rowlen, 4, req->req.buf, plen, false);  in gr_dbgprint_request()
117 				struct gr_request *req) {}  in gr_dbgprint_request()  argument
134 	struct gr_request *req;  in gr_seq_ep_show()  local
167 	list_for_each_entry(req, &ep->queue, queue) {  in gr_seq_ep_show()
171 		seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,  in gr_seq_ep_show()
172 			   &req->req.buf, req->req.actual, req->req.length);  in gr_seq_ep_show()
174 		next = req->first_desc;  in gr_seq_ep_show()
179 				   desc == req->curr_desc ? 'c' : ' ',  in gr_seq_ep_show()
181 		} while (desc != req->last_desc);  in gr_seq_ep_show()
275 static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)  in gr_free_dma_desc_chain()  argument
280 	next = req->first_desc;  in gr_free_dma_desc_chain()
288 	} while (desc != req->last_desc);  in gr_free_dma_desc_chain()
290 	req->first_desc = NULL;  in gr_free_dma_desc_chain()
291 	req->curr_desc = NULL;  in gr_free_dma_desc_chain()
292 	req->last_desc = NULL;  in gr_free_dma_desc_chain()
295 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
303 static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,  in gr_finish_request()  argument
310 	list_del_init(&req->queue);  in gr_finish_request()
312 	if (likely(req->req.status == -EINPROGRESS))  in gr_finish_request()
313 		req->req.status = status;  in gr_finish_request()
315 		status = req->req.status;  in gr_finish_request()
318 	usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);  in gr_finish_request()
319 	gr_free_dma_desc_chain(dev, req);  in gr_finish_request()
322 		req->req.actual = req->req.length;  in gr_finish_request()
323 	} else if (req->oddlen && req->req.actual > req->evenlen) {  in gr_finish_request()
329 		char *buftail = ((char *)req->req.buf + req->evenlen);  in gr_finish_request()
331 		memcpy(buftail, ep->tailbuf, req->oddlen);  in gr_finish_request()
333 		if (req->req.actual > req->req.length) {  in gr_finish_request()
337 			gr_dbgprint_request("OVFL", ep, req);  in gr_finish_request()
338 			req->req.status = -EOVERFLOW;  in gr_finish_request()
344 			gr_dbgprint_request("SENT", ep, req);  in gr_finish_request()
346 			gr_dbgprint_request("RECV", ep, req);  in gr_finish_request()
351 	if (req == dev->ep0reqo && !status) {  in gr_finish_request()
352 		if (req->setup)  in gr_finish_request()
353 			gr_ep0_setup(dev, req);  in gr_finish_request()
357 	} else if (req->req.complete) {  in gr_finish_request()
360 		usb_gadget_giveback_request(&ep->ep, &req->req);  in gr_finish_request()
369 	struct gr_request *req;  in gr_alloc_request()  local
371 	req = kzalloc(sizeof(*req), gfp_flags);  in gr_alloc_request()
372 	if (!req)  in gr_alloc_request()
375 	INIT_LIST_HEAD(&req->queue);  in gr_alloc_request()
377 	return &req->req;  in gr_alloc_request()
387 	struct gr_request *req;  in gr_start_dma()  local
395 	req = list_first_entry(&ep->queue, struct gr_request, queue);  in gr_start_dma()
398 	BUG_ON(!req->curr_desc);  in gr_start_dma()
406 	if (!ep->is_in && req->oddlen)  in gr_start_dma()
407 		req->last_desc->data = ep->tailbuf_paddr;  in gr_start_dma()
412 	gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);  in gr_start_dma()
429 	struct gr_request *req;  in gr_dma_advance()  local
431 	req = list_first_entry(&ep->queue, struct gr_request, queue);  in gr_dma_advance()
432 	gr_finish_request(ep, req, status);  in gr_dma_advance()
457 static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,  in gr_add_dma_desc()  argument
473 	if (!req->first_desc) {  in gr_add_dma_desc()
474 		req->first_desc = desc;  in gr_add_dma_desc()
475 		req->curr_desc = desc;  in gr_add_dma_desc()
477 		req->last_desc->next_desc = desc;  in gr_add_dma_desc()
478 		req->last_desc->next = desc->paddr;  in gr_add_dma_desc()
479 		req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;  in gr_add_dma_desc()
481 	req->last_desc = desc;  in gr_add_dma_desc()
496 static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,  in gr_setup_out_desc_list()  argument
503 	req->first_desc = NULL; /* Signals that no allocation is done yet */  in gr_setup_out_desc_list()
504 	bytes_left = req->req.length;  in gr_setup_out_desc_list()
507 		dma_addr_t start = req->req.dma + bytes_used;  in gr_setup_out_desc_list()
512 			req->evenlen = req->req.length - bytes_left;  in gr_setup_out_desc_list()
513 			req->oddlen = size;  in gr_setup_out_desc_list()
516 		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);  in gr_setup_out_desc_list()
524 	req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;  in gr_setup_out_desc_list()
529 	gr_free_dma_desc_chain(ep->dev, req);  in gr_setup_out_desc_list()
549 static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,  in gr_setup_in_desc_list()  argument
556 	req->first_desc = NULL; /* Signals that no allocation is done yet */  in gr_setup_in_desc_list()
557 	bytes_left = req->req.length;  in gr_setup_in_desc_list()
560 		dma_addr_t start = req->req.dma + bytes_used;  in gr_setup_in_desc_list()
563 		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);  in gr_setup_in_desc_list()
576 	if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {  in gr_setup_in_desc_list()
577 		ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);  in gr_setup_in_desc_list()
586 	req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;  in gr_setup_in_desc_list()
591 	gr_free_dma_desc_chain(ep->dev, req);  in gr_setup_in_desc_list()
597 static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)  in gr_queue()  argument
607 	if (unlikely(!req->req.buf || !list_empty(&req->queue))) {  in gr_queue()
610 			ep->ep.name, req->req.buf, list_empty(&req->queue));  in gr_queue()
626 	ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);  in gr_queue()
633 		ret = gr_setup_in_desc_list(ep, req, gfp_flags);  in gr_queue()
635 		ret = gr_setup_out_desc_list(ep, req, gfp_flags);  in gr_queue()
639 	req->req.status = -EINPROGRESS;  in gr_queue()
640 	req->req.actual = 0;  in gr_queue()
641 	list_add_tail(&req->queue, &ep->queue);  in gr_queue()
655 static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,  in gr_queue_int()  argument
659 		gr_dbgprint_request("RESP", ep, req);  in gr_queue_int()
661 	return gr_queue(ep, req, gfp_flags);  in gr_queue_int()
674 	struct gr_request *req;  in gr_ep_nuke()  local
681 		req = list_first_entry(&ep->queue, struct gr_request, queue);  in gr_ep_nuke()
682 		gr_finish_request(ep, req, -ESHUTDOWN);  in gr_ep_nuke()
842 					   struct usb_request *req))  in gr_ep0_respond()  argument
844 	u8 *reqbuf = dev->ep0reqi->req.buf;  in gr_ep0_respond()
850 	dev->ep0reqi->req.length = length;  in gr_ep0_respond()
851 	dev->ep0reqi->req.complete = complete;  in gr_ep0_respond()
1062 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)  in gr_ep0_setup()  argument
1082 		if (!req->req.actual)  in gr_ep0_setup()
1088 		if (req->req.actual > 0)  in gr_ep0_setup()
1101 	} else if (!req->req.actual) {  in gr_ep0_setup()
1108 	for (i = 0; i < req->req.actual; i++)  in gr_ep0_setup()
1109 		u.raw[i] = ((u8 *)req->req.buf)[i];  in gr_ep0_setup()
1259 	struct gr_request *req;  in gr_handle_in_ep()  local
1261 	req = list_first_entry(&ep->queue, struct gr_request, queue);  in gr_handle_in_ep()
1262 	if (!req->last_desc)  in gr_handle_in_ep()
1265 	if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)  in gr_handle_in_ep()
1287 	struct gr_request *req;  in gr_handle_out_ep()  local
1290 	req = list_first_entry(&ep->queue, struct gr_request, queue);  in gr_handle_out_ep()
1291 	if (!req->curr_desc)  in gr_handle_out_ep()
1294 	ctrl = ACCESS_ONCE(req->curr_desc->ctrl);  in gr_handle_out_ep()
1300 	req->req.actual += len;  in gr_handle_out_ep()
1302 		req->setup = 1;  in gr_handle_out_ep()
1304 	if (len < ep->ep.maxpacket || req->req.actual >= req->req.length) {  in gr_handle_out_ep()
1321 		req->curr_desc = req->curr_desc->next_desc;  in gr_handle_out_ep()
1322 		req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;  in gr_handle_out_ep()
1658 	struct gr_request *req;  in gr_free_request()  local
1662 	req = container_of(_req, struct gr_request, req);  in gr_free_request()
1665 	WARN(!list_empty(&req->queue),  in gr_free_request()
1668 	kfree(req);  in gr_free_request()
1676 	struct gr_request *req;  in gr_queue_ext()  local
1684 	req = container_of(_req, struct gr_request, req);  in gr_queue_ext()
1701 		gr_dbgprint_request("EXTERN", ep, req);  in gr_queue_ext()
1703 	ret = gr_queue(ep, req, GFP_ATOMIC);  in gr_queue_ext()
1713 	struct gr_request *req;  in gr_dequeue()  local
1733 	list_for_each_entry(req, &ep->queue, queue) {  in gr_dequeue()
1734 		if (&req->req == _req)  in gr_dequeue()
1737 	if (&req->req != _req) {  in gr_dequeue()
1742 	if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {  in gr_dequeue()
1746 			gr_finish_request(ep, req, -ECONNRESET);  in gr_dequeue()
1749 	} else if (!list_empty(&req->queue)) {  in gr_dequeue()
1751 		gr_finish_request(ep, req, -ECONNRESET);  in gr_dequeue()
1981 	struct gr_request *req;  in gr_ep_init()  local
2010 		req = container_of(_req, struct gr_request, req);  in gr_ep_init()
2011 		req->req.buf = buf;  in gr_ep_init()
2012 		req->req.length = MAX_CTRL_PL_SIZE;  in gr_ep_init()
2015 			dev->ep0reqi = req; /* Complete gets set as used */  in gr_ep_init()
2017 			dev->ep0reqo = req; /* Completion treated separately */  in gr_ep_init()
2113 	gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);  in gr_remove()
2114 	gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);  in gr_remove()