1/*
2 * USB Peripheral Controller driver for Aeroflex Gaisler GRUSBDC.
3 *
4 * 2013 (c) Aeroflex Gaisler AB
5 *
6 * This driver supports GRUSBDC USB Device Controller cores available in the
7 * GRLIB VHDL IP core library.
8 *
9 * Full documentation of the GRUSBDC core can be found here:
10 * http://www.gaisler.com/products/grlib/grip.pdf
11 *
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms of the GNU General Public License as published by the
14 * Free Software Foundation; either version 2 of the License, or (at your
15 * option) any later version.
16 *
17 * Contributors:
18 * - Andreas Larsson <andreas@gaisler.com>
19 * - Marko Isomaki
20 */
21
22/*
23 * A GRUSBDC core can have up to 16 IN endpoints and 16 OUT endpoints each
24 * individually configurable to any of the four USB transfer types. This driver
25 * only supports cores in DMA mode.
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/slab.h>
31#include <linux/spinlock.h>
32#include <linux/errno.h>
33#include <linux/list.h>
34#include <linux/interrupt.h>
35#include <linux/device.h>
36#include <linux/usb/ch9.h>
37#include <linux/usb/gadget.h>
38#include <linux/dma-mapping.h>
39#include <linux/dmapool.h>
40#include <linux/debugfs.h>
41#include <linux/seq_file.h>
42#include <linux/of_platform.h>
43#include <linux/of_irq.h>
44#include <linux/of_address.h>
45
46#include <asm/byteorder.h>
47
48#include "gr_udc.h"
49
50#define	DRIVER_NAME	"gr_udc"
51#define	DRIVER_DESC	"Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
52
53static const char driver_name[] = DRIVER_NAME;
54static const char driver_desc[] = DRIVER_DESC;
55
56#define gr_read32(x) (ioread32be((x)))
57#define gr_write32(x, v) (iowrite32be((v), (x)))
58
59/* USB speed and corresponding string calculated from status register value */
60#define GR_SPEED(status) \
61	((status & GR_STATUS_SP) ? USB_SPEED_FULL : USB_SPEED_HIGH)
62#define GR_SPEED_STR(status) usb_speed_string(GR_SPEED(status))
63
64/* Size of hardware buffer calculated from epctrl register value */
65#define GR_BUFFER_SIZE(epctrl)					      \
66	((((epctrl) & GR_EPCTRL_BUFSZ_MASK) >> GR_EPCTRL_BUFSZ_POS) * \
67	 GR_EPCTRL_BUFSZ_SCALER)
68
69/* ---------------------------------------------------------------------- */
70/* Debug printout functionality */
71
72static const char * const gr_modestring[] = {"control", "iso", "bulk", "int"};
73
74static const char *gr_ep0state_string(enum gr_ep0state state)
75{
76	static const char *const names[] = {
77		[GR_EP0_DISCONNECT] = "disconnect",
78		[GR_EP0_SETUP] = "setup",
79		[GR_EP0_IDATA] = "idata",
80		[GR_EP0_ODATA] = "odata",
81		[GR_EP0_ISTATUS] = "istatus",
82		[GR_EP0_OSTATUS] = "ostatus",
83		[GR_EP0_STALL] = "stall",
84		[GR_EP0_SUSPEND] = "suspend",
85	};
86
87	if (state < 0 || state >= ARRAY_SIZE(names))
88		return "UNKNOWN";
89
90	return names[state];
91}
92
93#ifdef VERBOSE_DEBUG
94
95static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
96				struct gr_request *req)
97{
98	int buflen = ep->is_in ? req->req.length : req->req.actual;
99	int rowlen = 32;
100	int plen = min(rowlen, buflen);
101
102	dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,
103		(buflen > plen ? " (truncated)" : ""));
104	print_hex_dump_debug("   ", DUMP_PREFIX_NONE,
105			     rowlen, 4, req->req.buf, plen, false);
106}
107
108static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
109			       u16 value, u16 index, u16 length)
110{
111	dev_vdbg(dev->dev, "REQ: %02x.%02x v%04x i%04x l%04x\n",
112		 type, request, value, index, length);
113}
114#else /* !VERBOSE_DEBUG */
115
116static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
117				struct gr_request *req) {}
118
119static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
120			       u16 value, u16 index, u16 length) {}
121
122#endif /* VERBOSE_DEBUG */
123
124/* ---------------------------------------------------------------------- */
125/* Debugfs functionality */
126
127#ifdef CONFIG_USB_GADGET_DEBUG_FS
128
129static void gr_seq_ep_show(struct seq_file *seq, struct gr_ep *ep)
130{
131	u32 epctrl = gr_read32(&ep->regs->epctrl);
132	u32 epstat = gr_read32(&ep->regs->epstat);
133	int mode = (epctrl & GR_EPCTRL_TT_MASK) >> GR_EPCTRL_TT_POS;
134	struct gr_request *req;
135
136	seq_printf(seq, "%s:\n", ep->ep.name);
137	seq_printf(seq, "  mode = %s\n", gr_modestring[mode]);
138	seq_printf(seq, "  halted: %d\n", !!(epctrl & GR_EPCTRL_EH));
139	seq_printf(seq, "  disabled: %d\n", !!(epctrl & GR_EPCTRL_ED));
140	seq_printf(seq, "  valid: %d\n", !!(epctrl & GR_EPCTRL_EV));
141	seq_printf(seq, "  dma_start = %d\n", ep->dma_start);
142	seq_printf(seq, "  stopped = %d\n", ep->stopped);
143	seq_printf(seq, "  wedged = %d\n", ep->wedged);
144	seq_printf(seq, "  callback = %d\n", ep->callback);
145	seq_printf(seq, "  maxpacket = %d\n", ep->ep.maxpacket);
146	seq_printf(seq, "  maxpacket_limit = %d\n", ep->ep.maxpacket_limit);
147	seq_printf(seq, "  bytes_per_buffer = %d\n", ep->bytes_per_buffer);
148	if (mode == 1 || mode == 3)
149		seq_printf(seq, "  nt = %d\n",
150			   (epctrl & GR_EPCTRL_NT_MASK) >> GR_EPCTRL_NT_POS);
151
152	seq_printf(seq, "  Buffer 0: %s %s%d\n",
153		   epstat & GR_EPSTAT_B0 ? "valid" : "invalid",
154		   epstat & GR_EPSTAT_BS ? " " : "selected ",
155		   (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS);
156	seq_printf(seq, "  Buffer 1: %s %s%d\n",
157		   epstat & GR_EPSTAT_B1 ? "valid" : "invalid",
158		   epstat & GR_EPSTAT_BS ? "selected " : " ",
159		   (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS);
160
161	if (list_empty(&ep->queue)) {
162		seq_puts(seq, "  Queue: empty\n\n");
163		return;
164	}
165
166	seq_puts(seq, "  Queue:\n");
167	list_for_each_entry(req, &ep->queue, queue) {
168		struct gr_dma_desc *desc;
169		struct gr_dma_desc *next;
170
171		seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,
172			   &req->req.buf, req->req.actual, req->req.length);
173
174		next = req->first_desc;
175		do {
176			desc = next;
177			next = desc->next_desc;
178			seq_printf(seq, "    %c 0x%p (0x%08x): 0x%05x 0x%08x\n",
179				   desc == req->curr_desc ? 'c' : ' ',
180				   desc, desc->paddr, desc->ctrl, desc->data);
181		} while (desc != req->last_desc);
182	}
183	seq_puts(seq, "\n");
184}
185
186
187static int gr_seq_show(struct seq_file *seq, void *v)
188{
189	struct gr_udc *dev = seq->private;
190	u32 control = gr_read32(&dev->regs->control);
191	u32 status = gr_read32(&dev->regs->status);
192	struct gr_ep *ep;
193
194	seq_printf(seq, "usb state = %s\n",
195		   usb_state_string(dev->gadget.state));
196	seq_printf(seq, "address = %d\n",
197		   (control & GR_CONTROL_UA_MASK) >> GR_CONTROL_UA_POS);
198	seq_printf(seq, "speed = %s\n", GR_SPEED_STR(status));
199	seq_printf(seq, "ep0state = %s\n", gr_ep0state_string(dev->ep0state));
200	seq_printf(seq, "irq_enabled = %d\n", dev->irq_enabled);
201	seq_printf(seq, "remote_wakeup = %d\n", dev->remote_wakeup);
202	seq_printf(seq, "test_mode = %d\n", dev->test_mode);
203	seq_puts(seq, "\n");
204
205	list_for_each_entry(ep, &dev->ep_list, ep_list)
206		gr_seq_ep_show(seq, ep);
207
208	return 0;
209}
210
211static int gr_dfs_open(struct inode *inode, struct file *file)
212{
213	return single_open(file, gr_seq_show, inode->i_private);
214}
215
216static const struct file_operations gr_dfs_fops = {
217	.owner		= THIS_MODULE,
218	.open		= gr_dfs_open,
219	.read		= seq_read,
220	.llseek		= seq_lseek,
221	.release	= single_release,
222};
223
224static void gr_dfs_create(struct gr_udc *dev)
225{
226	const char *name = "gr_udc_state";
227
228	dev->dfs_root = debugfs_create_dir(dev_name(dev->dev), NULL);
229	dev->dfs_state = debugfs_create_file(name, 0444, dev->dfs_root, dev,
230					     &gr_dfs_fops);
231}
232
233static void gr_dfs_delete(struct gr_udc *dev)
234{
235	/* Handles NULL and ERR pointers internally */
236	debugfs_remove(dev->dfs_state);
237	debugfs_remove(dev->dfs_root);
238}
239
240#else /* !CONFIG_USB_GADGET_DEBUG_FS */
241
242static void gr_dfs_create(struct gr_udc *dev) {}
243static void gr_dfs_delete(struct gr_udc *dev) {}
244
245#endif /* CONFIG_USB_GADGET_DEBUG_FS */
246
247/* ---------------------------------------------------------------------- */
248/* DMA and request handling */
249
250/* Allocates a new struct gr_dma_desc, sets paddr and zeroes the rest */
251static struct gr_dma_desc *gr_alloc_dma_desc(struct gr_ep *ep, gfp_t gfp_flags)
252{
253	dma_addr_t paddr;
254	struct gr_dma_desc *dma_desc;
255
256	dma_desc = dma_pool_alloc(ep->dev->desc_pool, gfp_flags, &paddr);
257	if (!dma_desc) {
258		dev_err(ep->dev->dev, "Could not allocate from DMA pool\n");
259		return NULL;
260	}
261
262	memset(dma_desc, 0, sizeof(*dma_desc));
263	dma_desc->paddr = paddr;
264
265	return dma_desc;
266}
267
268static inline void gr_free_dma_desc(struct gr_udc *dev,
269				    struct gr_dma_desc *desc)
270{
271	dma_pool_free(dev->desc_pool, desc, (dma_addr_t)desc->paddr);
272}
273
274/* Frees the chain of struct gr_dma_desc for the given request */
275static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)
276{
277	struct gr_dma_desc *desc;
278	struct gr_dma_desc *next;
279
280	next = req->first_desc;
281	if (!next)
282		return;
283
284	do {
285		desc = next;
286		next = desc->next_desc;
287		gr_free_dma_desc(dev, desc);
288	} while (desc != req->last_desc);
289
290	req->first_desc = NULL;
291	req->curr_desc = NULL;
292	req->last_desc = NULL;
293}
294
295static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
296
297/*
298 * Frees allocated resources and calls the appropriate completion function/setup
299 * package handler for a finished request.
300 *
301 * Must be called with dev->lock held and irqs disabled.
302 */
303static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,
304			      int status)
305	__releases(&dev->lock)
306	__acquires(&dev->lock)
307{
308	struct gr_udc *dev;
309
310	list_del_init(&req->queue);
311
312	if (likely(req->req.status == -EINPROGRESS))
313		req->req.status = status;
314	else
315		status = req->req.status;
316
317	dev = ep->dev;
318	usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
319	gr_free_dma_desc_chain(dev, req);
320
321	if (ep->is_in) { /* For OUT, req->req.actual gets updated bit by bit */
322		req->req.actual = req->req.length;
323	} else if (req->oddlen && req->req.actual > req->evenlen) {
324		/*
325		 * Copy to user buffer in this case where length was not evenly
326		 * divisible by ep->ep.maxpacket and the last descriptor was
327		 * actually used.
328		 */
329		char *buftail = ((char *)req->req.buf + req->evenlen);
330
331		memcpy(buftail, ep->tailbuf, req->oddlen);
332
333		if (req->req.actual > req->req.length) {
334			/* We got more data than was requested */
335			dev_dbg(ep->dev->dev, "Overflow for ep %s\n",
336				ep->ep.name);
337			gr_dbgprint_request("OVFL", ep, req);
338			req->req.status = -EOVERFLOW;
339		}
340	}
341
342	if (!status) {
343		if (ep->is_in)
344			gr_dbgprint_request("SENT", ep, req);
345		else
346			gr_dbgprint_request("RECV", ep, req);
347	}
348
349	/* Prevent changes to ep->queue during callback */
350	ep->callback = 1;
351	if (req == dev->ep0reqo && !status) {
352		if (req->setup)
353			gr_ep0_setup(dev, req);
354		else
355			dev_err(dev->dev,
356				"Unexpected non setup packet on ep0in\n");
357	} else if (req->req.complete) {
358		spin_unlock(&dev->lock);
359
360		usb_gadget_giveback_request(&ep->ep, &req->req);
361
362		spin_lock(&dev->lock);
363	}
364	ep->callback = 0;
365}
366
367static struct usb_request *gr_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
368{
369	struct gr_request *req;
370
371	req = kzalloc(sizeof(*req), gfp_flags);
372	if (!req)
373		return NULL;
374
375	INIT_LIST_HEAD(&req->queue);
376
377	return &req->req;
378}
379
380/*
381 * Starts DMA for endpoint ep if there are requests in the queue.
382 *
383 * Must be called with dev->lock held and with !ep->stopped.
384 */
385static void gr_start_dma(struct gr_ep *ep)
386{
387	struct gr_request *req;
388	u32 dmactrl;
389
390	if (list_empty(&ep->queue)) {
391		ep->dma_start = 0;
392		return;
393	}
394
395	req = list_first_entry(&ep->queue, struct gr_request, queue);
396
397	/* A descriptor should already have been allocated */
398	BUG_ON(!req->curr_desc);
399
400	/*
401	 * The DMA controller can not handle smaller OUT buffers than
402	 * ep->ep.maxpacket. It could lead to buffer overruns if an unexpectedly
403	 * long packet are received. Therefore an internal bounce buffer gets
404	 * used when such a request gets enabled.
405	 */
406	if (!ep->is_in && req->oddlen)
407		req->last_desc->data = ep->tailbuf_paddr;
408
409	wmb(); /* Make sure all is settled before handing it over to DMA */
410
411	/* Set the descriptor pointer in the hardware */
412	gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);
413
414	/* Announce available descriptors */
415	dmactrl = gr_read32(&ep->regs->dmactrl);
416	gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_DA);
417
418	ep->dma_start = 1;
419}
420
421/*
422 * Finishes the first request in the ep's queue and, if available, starts the
423 * next request in queue.
424 *
425 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
426 */
427static void gr_dma_advance(struct gr_ep *ep, int status)
428{
429	struct gr_request *req;
430
431	req = list_first_entry(&ep->queue, struct gr_request, queue);
432	gr_finish_request(ep, req, status);
433	gr_start_dma(ep); /* Regardless of ep->dma_start */
434}
435
436/*
437 * Abort DMA for an endpoint. Sets the abort DMA bit which causes an ongoing DMA
438 * transfer to be canceled and clears GR_DMACTRL_DA.
439 *
440 * Must be called with dev->lock held.
441 */
442static void gr_abort_dma(struct gr_ep *ep)
443{
444	u32 dmactrl;
445
446	dmactrl = gr_read32(&ep->regs->dmactrl);
447	gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_AD);
448}
449
450/*
451 * Allocates and sets up a struct gr_dma_desc and putting it on the descriptor
452 * chain.
453 *
454 * Size is not used for OUT endpoints. Hardware can not be instructed to handle
455 * smaller buffer than MAXPL in the OUT direction.
456 */
457static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,
458			   dma_addr_t data, unsigned size, gfp_t gfp_flags)
459{
460	struct gr_dma_desc *desc;
461
462	desc = gr_alloc_dma_desc(ep, gfp_flags);
463	if (!desc)
464		return -ENOMEM;
465
466	desc->data = data;
467	if (ep->is_in)
468		desc->ctrl =
469			(GR_DESC_IN_CTRL_LEN_MASK & size) | GR_DESC_IN_CTRL_EN;
470	else
471		desc->ctrl = GR_DESC_OUT_CTRL_IE;
472
473	if (!req->first_desc) {
474		req->first_desc = desc;
475		req->curr_desc = desc;
476	} else {
477		req->last_desc->next_desc = desc;
478		req->last_desc->next = desc->paddr;
479		req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;
480	}
481	req->last_desc = desc;
482
483	return 0;
484}
485
486/*
487 * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
488 * together covers req->req.length bytes of the buffer at DMA address
489 * req->req.dma for the OUT direction.
490 *
491 * The first descriptor in the chain is enabled, the rest disabled. The
492 * interrupt handler will later enable them one by one when needed so we can
493 * find out when the transfer is finished. For OUT endpoints, all descriptors
494 * therefore generate interrutps.
495 */
496static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,
497				  gfp_t gfp_flags)
498{
499	u16 bytes_left; /* Bytes left to provide descriptors for */
500	u16 bytes_used; /* Bytes accommodated for */
501	int ret = 0;
502
503	req->first_desc = NULL; /* Signals that no allocation is done yet */
504	bytes_left = req->req.length;
505	bytes_used = 0;
506	while (bytes_left > 0) {
507		dma_addr_t start = req->req.dma + bytes_used;
508		u16 size = min(bytes_left, ep->bytes_per_buffer);
509
510		if (size < ep->bytes_per_buffer) {
511			/* Prepare using bounce buffer */
512			req->evenlen = req->req.length - bytes_left;
513			req->oddlen = size;
514		}
515
516		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
517		if (ret)
518			goto alloc_err;
519
520		bytes_left -= size;
521		bytes_used += size;
522	}
523
524	req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
525
526	return 0;
527
528alloc_err:
529	gr_free_dma_desc_chain(ep->dev, req);
530
531	return ret;
532}
533
534/*
535 * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
536 * together covers req->req.length bytes of the buffer at DMA address
537 * req->req.dma for the IN direction.
538 *
539 * When more data is provided than the maximum payload size, the hardware splits
540 * this up into several payloads automatically. Moreover, ep->bytes_per_buffer
541 * is always set to a multiple of the maximum payload (restricted to the valid
542 * number of maximum payloads during high bandwidth isochronous or interrupt
543 * transfers)
544 *
545 * All descriptors are enabled from the beginning and we only generate an
546 * interrupt for the last one indicating that the entire request has been pushed
547 * to hardware.
548 */
549static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,
550				 gfp_t gfp_flags)
551{
552	u16 bytes_left; /* Bytes left in req to provide descriptors for */
553	u16 bytes_used; /* Bytes in req accommodated for */
554	int ret = 0;
555
556	req->first_desc = NULL; /* Signals that no allocation is done yet */
557	bytes_left = req->req.length;
558	bytes_used = 0;
559	do { /* Allow for zero length packets */
560		dma_addr_t start = req->req.dma + bytes_used;
561		u16 size = min(bytes_left, ep->bytes_per_buffer);
562
563		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
564		if (ret)
565			goto alloc_err;
566
567		bytes_left -= size;
568		bytes_used += size;
569	} while (bytes_left > 0);
570
571	/*
572	 * Send an extra zero length packet to indicate that no more data is
573	 * available when req->req.zero is set and the data length is even
574	 * multiples of ep->ep.maxpacket.
575	 */
576	if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {
577		ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);
578		if (ret)
579			goto alloc_err;
580	}
581
582	/*
583	 * For IN packets we only want to know when the last packet has been
584	 * transmitted (not just put into internal buffers).
585	 */
586	req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;
587
588	return 0;
589
590alloc_err:
591	gr_free_dma_desc_chain(ep->dev, req);
592
593	return ret;
594}
595
596/* Must be called with dev->lock held */
597static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)
598{
599	struct gr_udc *dev = ep->dev;
600	int ret;
601
602	if (unlikely(!ep->ep.desc && ep->num != 0)) {
603		dev_err(dev->dev, "No ep descriptor for %s\n", ep->ep.name);
604		return -EINVAL;
605	}
606
607	if (unlikely(!req->req.buf || !list_empty(&req->queue))) {
608		dev_err(dev->dev,
609			"Invalid request for %s: buf=%p list_empty=%d\n",
610			ep->ep.name, req->req.buf, list_empty(&req->queue));
611		return -EINVAL;
612	}
613
614	if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
615		dev_err(dev->dev, "-ESHUTDOWN");
616		return -ESHUTDOWN;
617	}
618
619	/* Can't touch registers when suspended */
620	if (dev->ep0state == GR_EP0_SUSPEND) {
621		dev_err(dev->dev, "-EBUSY");
622		return -EBUSY;
623	}
624
625	/* Set up DMA mapping in case the caller didn't */
626	ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);
627	if (ret) {
628		dev_err(dev->dev, "usb_gadget_map_request");
629		return ret;
630	}
631
632	if (ep->is_in)
633		ret = gr_setup_in_desc_list(ep, req, gfp_flags);
634	else
635		ret = gr_setup_out_desc_list(ep, req, gfp_flags);
636	if (ret)
637		return ret;
638
639	req->req.status = -EINPROGRESS;
640	req->req.actual = 0;
641	list_add_tail(&req->queue, &ep->queue);
642
643	/* Start DMA if not started, otherwise interrupt handler handles it */
644	if (!ep->dma_start && likely(!ep->stopped))
645		gr_start_dma(ep);
646
647	return 0;
648}
649
650/*
651 * Queue a request from within the driver.
652 *
653 * Must be called with dev->lock held.
654 */
655static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,
656			       gfp_t gfp_flags)
657{
658	if (ep->is_in)
659		gr_dbgprint_request("RESP", ep, req);
660
661	return gr_queue(ep, req, gfp_flags);
662}
663
664/* ---------------------------------------------------------------------- */
665/* General helper functions */
666
667/*
668 * Dequeue ALL requests.
669 *
670 * Must be called with dev->lock held and irqs disabled.
671 */
672static void gr_ep_nuke(struct gr_ep *ep)
673{
674	struct gr_request *req;
675
676	ep->stopped = 1;
677	ep->dma_start = 0;
678	gr_abort_dma(ep);
679
680	while (!list_empty(&ep->queue)) {
681		req = list_first_entry(&ep->queue, struct gr_request, queue);
682		gr_finish_request(ep, req, -ESHUTDOWN);
683	}
684}
685
686/*
687 * Reset the hardware state of this endpoint.
688 *
689 * Must be called with dev->lock held.
690 */
691static void gr_ep_reset(struct gr_ep *ep)
692{
693	gr_write32(&ep->regs->epctrl, 0);
694	gr_write32(&ep->regs->dmactrl, 0);
695
696	ep->ep.maxpacket = MAX_CTRL_PL_SIZE;
697	ep->ep.desc = NULL;
698	ep->stopped = 1;
699	ep->dma_start = 0;
700}
701
702/*
703 * Generate STALL on ep0in/out.
704 *
705 * Must be called with dev->lock held.
706 */
707static void gr_control_stall(struct gr_udc *dev)
708{
709	u32 epctrl;
710
711	epctrl = gr_read32(&dev->epo[0].regs->epctrl);
712	gr_write32(&dev->epo[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
713	epctrl = gr_read32(&dev->epi[0].regs->epctrl);
714	gr_write32(&dev->epi[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
715
716	dev->ep0state = GR_EP0_STALL;
717}
718
719/*
720 * Halts, halts and wedges, or clears halt for an endpoint.
721 *
722 * Must be called with dev->lock held.
723 */
724static int gr_ep_halt_wedge(struct gr_ep *ep, int halt, int wedge, int fromhost)
725{
726	u32 epctrl;
727	int retval = 0;
728
729	if (ep->num && !ep->ep.desc)
730		return -EINVAL;
731
732	if (ep->num && ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC)
733		return -EOPNOTSUPP;
734
735	/* Never actually halt ep0, and therefore never clear halt for ep0 */
736	if (!ep->num) {
737		if (halt && !fromhost) {
738			/* ep0 halt from gadget - generate protocol stall */
739			gr_control_stall(ep->dev);
740			dev_dbg(ep->dev->dev, "EP: stall ep0\n");
741			return 0;
742		}
743		return -EINVAL;
744	}
745
746	dev_dbg(ep->dev->dev, "EP: %s halt %s\n",
747		(halt ? (wedge ? "wedge" : "set") : "clear"), ep->ep.name);
748
749	epctrl = gr_read32(&ep->regs->epctrl);
750	if (halt) {
751		/* Set HALT */
752		gr_write32(&ep->regs->epctrl, epctrl | GR_EPCTRL_EH);
753		ep->stopped = 1;
754		if (wedge)
755			ep->wedged = 1;
756	} else {
757		gr_write32(&ep->regs->epctrl, epctrl & ~GR_EPCTRL_EH);
758		ep->stopped = 0;
759		ep->wedged = 0;
760
761		/* Things might have been queued up in the meantime */
762		if (!ep->dma_start)
763			gr_start_dma(ep);
764	}
765
766	return retval;
767}
768
769/* Must be called with dev->lock held */
770static inline void gr_set_ep0state(struct gr_udc *dev, enum gr_ep0state value)
771{
772	if (dev->ep0state != value)
773		dev_vdbg(dev->dev, "STATE:  ep0state=%s\n",
774			 gr_ep0state_string(value));
775	dev->ep0state = value;
776}
777
778/*
779 * Should only be called when endpoints can not generate interrupts.
780 *
781 * Must be called with dev->lock held.
782 */
783static void gr_disable_interrupts_and_pullup(struct gr_udc *dev)
784{
785	gr_write32(&dev->regs->control, 0);
786	wmb(); /* Make sure that we do not deny one of our interrupts */
787	dev->irq_enabled = 0;
788}
789
790/*
791 * Stop all device activity and disable data line pullup.
792 *
793 * Must be called with dev->lock held and irqs disabled.
794 */
795static void gr_stop_activity(struct gr_udc *dev)
796{
797	struct gr_ep *ep;
798
799	list_for_each_entry(ep, &dev->ep_list, ep_list)
800		gr_ep_nuke(ep);
801
802	gr_disable_interrupts_and_pullup(dev);
803
804	gr_set_ep0state(dev, GR_EP0_DISCONNECT);
805	usb_gadget_set_state(&dev->gadget, USB_STATE_NOTATTACHED);
806}
807
808/* ---------------------------------------------------------------------- */
809/* ep0 setup packet handling */
810
811static void gr_ep0_testmode_complete(struct usb_ep *_ep,
812				     struct usb_request *_req)
813{
814	struct gr_ep *ep;
815	struct gr_udc *dev;
816	u32 control;
817
818	ep = container_of(_ep, struct gr_ep, ep);
819	dev = ep->dev;
820
821	spin_lock(&dev->lock);
822
823	control = gr_read32(&dev->regs->control);
824	control |= GR_CONTROL_TM | (dev->test_mode << GR_CONTROL_TS_POS);
825	gr_write32(&dev->regs->control, control);
826
827	spin_unlock(&dev->lock);
828}
829
830static void gr_ep0_dummy_complete(struct usb_ep *_ep, struct usb_request *_req)
831{
832	/* Nothing needs to be done here */
833}
834
835/*
836 * Queue a response on ep0in.
837 *
838 * Must be called with dev->lock held.
839 */
840static int gr_ep0_respond(struct gr_udc *dev, u8 *buf, int length,
841			  void (*complete)(struct usb_ep *ep,
842					   struct usb_request *req))
843{
844	u8 *reqbuf = dev->ep0reqi->req.buf;
845	int status;
846	int i;
847
848	for (i = 0; i < length; i++)
849		reqbuf[i] = buf[i];
850	dev->ep0reqi->req.length = length;
851	dev->ep0reqi->req.complete = complete;
852
853	status = gr_queue_int(&dev->epi[0], dev->ep0reqi, GFP_ATOMIC);
854	if (status < 0)
855		dev_err(dev->dev,
856			"Could not queue ep0in setup response: %d\n", status);
857
858	return status;
859}
860
861/*
862 * Queue a 2 byte response on ep0in.
863 *
864 * Must be called with dev->lock held.
865 */
866static inline int gr_ep0_respond_u16(struct gr_udc *dev, u16 response)
867{
868	__le16 le_response = cpu_to_le16(response);
869
870	return gr_ep0_respond(dev, (u8 *)&le_response, 2,
871			      gr_ep0_dummy_complete);
872}
873
874/*
875 * Queue a ZLP response on ep0in.
876 *
877 * Must be called with dev->lock held.
878 */
879static inline int gr_ep0_respond_empty(struct gr_udc *dev)
880{
881	return gr_ep0_respond(dev, NULL, 0, gr_ep0_dummy_complete);
882}
883
884/*
885 * This is run when a SET_ADDRESS request is received. First writes
886 * the new address to the control register which is updated internally
887 * when the next IN packet is ACKED.
888 *
889 * Must be called with dev->lock held.
890 */
891static void gr_set_address(struct gr_udc *dev, u8 address)
892{
893	u32 control;
894
895	control = gr_read32(&dev->regs->control) & ~GR_CONTROL_UA_MASK;
896	control |= (address << GR_CONTROL_UA_POS) & GR_CONTROL_UA_MASK;
897	control |= GR_CONTROL_SU;
898	gr_write32(&dev->regs->control, control);
899}
900
901/*
902 * Returns negative for STALL, 0 for successful handling and positive for
903 * delegation.
904 *
905 * Must be called with dev->lock held.
906 */
907static int gr_device_request(struct gr_udc *dev, u8 type, u8 request,
908			     u16 value, u16 index)
909{
910	u16 response;
911	u8 test;
912
913	switch (request) {
914	case USB_REQ_SET_ADDRESS:
915		dev_dbg(dev->dev, "STATUS: address %d\n", value & 0xff);
916		gr_set_address(dev, value & 0xff);
917		if (value)
918			usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
919		else
920			usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
921		return gr_ep0_respond_empty(dev);
922
923	case USB_REQ_GET_STATUS:
924		/* Self powered | remote wakeup */
925		response = 0x0001 | (dev->remote_wakeup ? 0x0002 : 0);
926		return gr_ep0_respond_u16(dev, response);
927
928	case USB_REQ_SET_FEATURE:
929		switch (value) {
930		case USB_DEVICE_REMOTE_WAKEUP:
931			/* Allow remote wakeup */
932			dev->remote_wakeup = 1;
933			return gr_ep0_respond_empty(dev);
934
935		case USB_DEVICE_TEST_MODE:
936			/* The hardware does not support TEST_FORCE_EN */
937			test = index >> 8;
938			if (test >= TEST_J && test <= TEST_PACKET) {
939				dev->test_mode = test;
940				return gr_ep0_respond(dev, NULL, 0,
941						      gr_ep0_testmode_complete);
942			}
943		}
944		break;
945
946	case USB_REQ_CLEAR_FEATURE:
947		switch (value) {
948		case USB_DEVICE_REMOTE_WAKEUP:
949			/* Disallow remote wakeup */
950			dev->remote_wakeup = 0;
951			return gr_ep0_respond_empty(dev);
952		}
953		break;
954	}
955
956	return 1; /* Delegate the rest */
957}
958
959/*
960 * Returns negative for STALL, 0 for successful handling and positive for
961 * delegation.
962 *
963 * Must be called with dev->lock held.
964 */
965static int gr_interface_request(struct gr_udc *dev, u8 type, u8 request,
966				u16 value, u16 index)
967{
968	if (dev->gadget.state != USB_STATE_CONFIGURED)
969		return -1;
970
971	/*
972	 * Should return STALL for invalid interfaces, but udc driver does not
973	 * know anything about that. However, many gadget drivers do not handle
974	 * GET_STATUS so we need to take care of that.
975	 */
976
977	switch (request) {
978	case USB_REQ_GET_STATUS:
979		return gr_ep0_respond_u16(dev, 0x0000);
980
981	case USB_REQ_SET_FEATURE:
982	case USB_REQ_CLEAR_FEATURE:
983		/*
984		 * No possible valid standard requests. Still let gadget drivers
985		 * have a go at it.
986		 */
987		break;
988	}
989
990	return 1; /* Delegate the rest */
991}
992
993/*
994 * Returns negative for STALL, 0 for successful handling and positive for
995 * delegation.
996 *
997 * Must be called with dev->lock held.
998 */
999static int gr_endpoint_request(struct gr_udc *dev, u8 type, u8 request,
1000			       u16 value, u16 index)
1001{
1002	struct gr_ep *ep;
1003	int status;
1004	int halted;
1005	u8 epnum = index & USB_ENDPOINT_NUMBER_MASK;
1006	u8 is_in = index & USB_ENDPOINT_DIR_MASK;
1007
1008	if ((is_in && epnum >= dev->nepi) || (!is_in && epnum >= dev->nepo))
1009		return -1;
1010
1011	if (dev->gadget.state != USB_STATE_CONFIGURED && epnum != 0)
1012		return -1;
1013
1014	ep = (is_in ? &dev->epi[epnum] : &dev->epo[epnum]);
1015
1016	switch (request) {
1017	case USB_REQ_GET_STATUS:
1018		halted = gr_read32(&ep->regs->epctrl) & GR_EPCTRL_EH;
1019		return gr_ep0_respond_u16(dev, halted ? 0x0001 : 0);
1020
1021	case USB_REQ_SET_FEATURE:
1022		switch (value) {
1023		case USB_ENDPOINT_HALT:
1024			status = gr_ep_halt_wedge(ep, 1, 0, 1);
1025			if (status >= 0)
1026				status = gr_ep0_respond_empty(dev);
1027			return status;
1028		}
1029		break;
1030
1031	case USB_REQ_CLEAR_FEATURE:
1032		switch (value) {
1033		case USB_ENDPOINT_HALT:
1034			if (ep->wedged)
1035				return -1;
1036			status = gr_ep_halt_wedge(ep, 0, 0, 1);
1037			if (status >= 0)
1038				status = gr_ep0_respond_empty(dev);
1039			return status;
1040		}
1041		break;
1042	}
1043
1044	return 1; /* Delegate the rest */
1045}
1046
1047/* Must be called with dev->lock held */
1048static void gr_ep0out_requeue(struct gr_udc *dev)
1049{
1050	int ret = gr_queue_int(&dev->epo[0], dev->ep0reqo, GFP_ATOMIC);
1051
1052	if (ret)
1053		dev_err(dev->dev, "Could not queue ep0out setup request: %d\n",
1054			ret);
1055}
1056
1057/*
1058 * The main function dealing with setup requests on ep0.
1059 *
1060 * Must be called with dev->lock held and irqs disabled
1061 */
1062static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)
1063	__releases(&dev->lock)
1064	__acquires(&dev->lock)
1065{
1066	union {
1067		struct usb_ctrlrequest ctrl;
1068		u8 raw[8];
1069		u32 word[2];
1070	} u;
1071	u8 type;
1072	u8 request;
1073	u16 value;
1074	u16 index;
1075	u16 length;
1076	int i;
1077	int status;
1078
1079	/* Restore from ep0 halt */
1080	if (dev->ep0state == GR_EP0_STALL) {
1081		gr_set_ep0state(dev, GR_EP0_SETUP);
1082		if (!req->req.actual)
1083			goto out;
1084	}
1085
1086	if (dev->ep0state == GR_EP0_ISTATUS) {
1087		gr_set_ep0state(dev, GR_EP0_SETUP);
1088		if (req->req.actual > 0)
1089			dev_dbg(dev->dev,
1090				"Unexpected setup packet at state %s\n",
1091				gr_ep0state_string(GR_EP0_ISTATUS));
1092		else
1093			goto out; /* Got expected ZLP */
1094	} else if (dev->ep0state != GR_EP0_SETUP) {
1095		dev_info(dev->dev,
1096			 "Unexpected ep0out request at state %s - stalling\n",
1097			 gr_ep0state_string(dev->ep0state));
1098		gr_control_stall(dev);
1099		gr_set_ep0state(dev, GR_EP0_SETUP);
1100		goto out;
1101	} else if (!req->req.actual) {
1102		dev_dbg(dev->dev, "Unexpected ZLP at state %s\n",
1103			gr_ep0state_string(dev->ep0state));
1104		goto out;
1105	}
1106
1107	/* Handle SETUP packet */
1108	for (i = 0; i < req->req.actual; i++)
1109		u.raw[i] = ((u8 *)req->req.buf)[i];
1110
1111	type = u.ctrl.bRequestType;
1112	request = u.ctrl.bRequest;
1113	value = le16_to_cpu(u.ctrl.wValue);
1114	index = le16_to_cpu(u.ctrl.wIndex);
1115	length = le16_to_cpu(u.ctrl.wLength);
1116
1117	gr_dbgprint_devreq(dev, type, request, value, index, length);
1118
1119	/* Check for data stage */
1120	if (length) {
1121		if (type & USB_DIR_IN)
1122			gr_set_ep0state(dev, GR_EP0_IDATA);
1123		else
1124			gr_set_ep0state(dev, GR_EP0_ODATA);
1125	}
1126
1127	status = 1; /* Positive status flags delegation */
1128	if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1129		switch (type & USB_RECIP_MASK) {
1130		case USB_RECIP_DEVICE:
1131			status = gr_device_request(dev, type, request,
1132						   value, index);
1133			break;
1134		case USB_RECIP_ENDPOINT:
1135			status =  gr_endpoint_request(dev, type, request,
1136						      value, index);
1137			break;
1138		case USB_RECIP_INTERFACE:
1139			status = gr_interface_request(dev, type, request,
1140						      value, index);
1141			break;
1142		}
1143	}
1144
1145	if (status > 0) {
1146		spin_unlock(&dev->lock);
1147
1148		dev_vdbg(dev->dev, "DELEGATE\n");
1149		status = dev->driver->setup(&dev->gadget, &u.ctrl);
1150
1151		spin_lock(&dev->lock);
1152	}
1153
1154	/* Generate STALL on both ep0out and ep0in if requested */
1155	if (unlikely(status < 0)) {
1156		dev_vdbg(dev->dev, "STALL\n");
1157		gr_control_stall(dev);
1158	}
1159
1160	if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD &&
1161	    request == USB_REQ_SET_CONFIGURATION) {
1162		if (!value) {
1163			dev_dbg(dev->dev, "STATUS: deconfigured\n");
1164			usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
1165		} else if (status >= 0) {
1166			/* Not configured unless gadget OK:s it */
1167			dev_dbg(dev->dev, "STATUS: configured: %d\n", value);
1168			usb_gadget_set_state(&dev->gadget,
1169					     USB_STATE_CONFIGURED);
1170		}
1171	}
1172
1173	/* Get ready for next stage */
1174	if (dev->ep0state == GR_EP0_ODATA)
1175		gr_set_ep0state(dev, GR_EP0_OSTATUS);
1176	else if (dev->ep0state == GR_EP0_IDATA)
1177		gr_set_ep0state(dev, GR_EP0_ISTATUS);
1178	else
1179		gr_set_ep0state(dev, GR_EP0_SETUP);
1180
1181out:
1182	gr_ep0out_requeue(dev);
1183}
1184
1185/* ---------------------------------------------------------------------- */
1186/* VBUS and USB reset handling */
1187
1188/* Must be called with dev->lock held and irqs disabled  */
1189static void gr_vbus_connected(struct gr_udc *dev, u32 status)
1190{
1191	u32 control;
1192
1193	dev->gadget.speed = GR_SPEED(status);
1194	usb_gadget_set_state(&dev->gadget, USB_STATE_POWERED);
1195
1196	/* Turn on full interrupts and pullup */
1197	control = (GR_CONTROL_SI | GR_CONTROL_UI | GR_CONTROL_VI |
1198		   GR_CONTROL_SP | GR_CONTROL_EP);
1199	gr_write32(&dev->regs->control, control);
1200}
1201
1202/* Must be called with dev->lock held */
1203static void gr_enable_vbus_detect(struct gr_udc *dev)
1204{
1205	u32 status;
1206
1207	dev->irq_enabled = 1;
1208	wmb(); /* Make sure we do not ignore an interrupt */
1209	gr_write32(&dev->regs->control, GR_CONTROL_VI);
1210
1211	/* Take care of the case we are already plugged in at this point */
1212	status = gr_read32(&dev->regs->status);
1213	if (status & GR_STATUS_VB)
1214		gr_vbus_connected(dev, status);
1215}
1216
1217/* Must be called with dev->lock held and irqs disabled */
1218static void gr_vbus_disconnected(struct gr_udc *dev)
1219{
1220	gr_stop_activity(dev);
1221
1222	/* Report disconnect */
1223	if (dev->driver && dev->driver->disconnect) {
1224		spin_unlock(&dev->lock);
1225
1226		dev->driver->disconnect(&dev->gadget);
1227
1228		spin_lock(&dev->lock);
1229	}
1230
1231	gr_enable_vbus_detect(dev);
1232}
1233
1234/* Must be called with dev->lock held and irqs disabled */
1235static void gr_udc_usbreset(struct gr_udc *dev, u32 status)
1236{
1237	gr_set_address(dev, 0);
1238	gr_set_ep0state(dev, GR_EP0_SETUP);
1239	usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
1240	dev->gadget.speed = GR_SPEED(status);
1241
1242	gr_ep_nuke(&dev->epo[0]);
1243	gr_ep_nuke(&dev->epi[0]);
1244	dev->epo[0].stopped = 0;
1245	dev->epi[0].stopped = 0;
1246	gr_ep0out_requeue(dev);
1247}
1248
1249/* ---------------------------------------------------------------------- */
1250/* Irq handling */
1251
1252/*
1253 * Handles interrupts from in endpoints. Returns whether something was handled.
1254 *
1255 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1256 */
1257static int gr_handle_in_ep(struct gr_ep *ep)
1258{
1259	struct gr_request *req;
1260
1261	req = list_first_entry(&ep->queue, struct gr_request, queue);
1262	if (!req->last_desc)
1263		return 0;
1264
1265	if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)
1266		return 0; /* Not put in hardware buffers yet */
1267
1268	if (gr_read32(&ep->regs->epstat) & (GR_EPSTAT_B1 | GR_EPSTAT_B0))
1269		return 0; /* Not transmitted yet, still in hardware buffers */
1270
1271	/* Write complete */
1272	gr_dma_advance(ep, 0);
1273
1274	return 1;
1275}
1276
1277/*
1278 * Handles interrupts from out endpoints. Returns whether something was handled.
1279 *
1280 * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1281 */
1282static int gr_handle_out_ep(struct gr_ep *ep)
1283{
1284	u32 ep_dmactrl;
1285	u32 ctrl;
1286	u16 len;
1287	struct gr_request *req;
1288	struct gr_udc *dev = ep->dev;
1289
1290	req = list_first_entry(&ep->queue, struct gr_request, queue);
1291	if (!req->curr_desc)
1292		return 0;
1293
1294	ctrl = ACCESS_ONCE(req->curr_desc->ctrl);
1295	if (ctrl & GR_DESC_OUT_CTRL_EN)
1296		return 0; /* Not received yet */
1297
1298	/* Read complete */
1299	len = ctrl & GR_DESC_OUT_CTRL_LEN_MASK;
1300	req->req.actual += len;
1301	if (ctrl & GR_DESC_OUT_CTRL_SE)
1302		req->setup = 1;
1303
1304	if (len < ep->ep.maxpacket || req->req.actual >= req->req.length) {
1305		/* Short packet or >= expected size - we are done */
1306
1307		if ((ep == &dev->epo[0]) && (dev->ep0state == GR_EP0_OSTATUS)) {
1308			/*
1309			 * Send a status stage ZLP to ack the DATA stage in the
1310			 * OUT direction. This needs to be done before
1311			 * gr_dma_advance as that can lead to a call to
1312			 * ep0_setup that can change dev->ep0state.
1313			 */
1314			gr_ep0_respond_empty(dev);
1315			gr_set_ep0state(dev, GR_EP0_SETUP);
1316		}
1317
1318		gr_dma_advance(ep, 0);
1319	} else {
1320		/* Not done yet. Enable the next descriptor to receive more. */
1321		req->curr_desc = req->curr_desc->next_desc;
1322		req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
1323
1324		ep_dmactrl = gr_read32(&ep->regs->dmactrl);
1325		gr_write32(&ep->regs->dmactrl, ep_dmactrl | GR_DMACTRL_DA);
1326	}
1327
1328	return 1;
1329}
1330
1331/*
1332 * Handle state changes. Returns whether something was handled.
1333 *
1334 * Must be called with dev->lock held and irqs disabled.
1335 */
1336static int gr_handle_state_changes(struct gr_udc *dev)
1337{
1338	u32 status = gr_read32(&dev->regs->status);
1339	int handled = 0;
1340	int powstate = !(dev->gadget.state == USB_STATE_NOTATTACHED ||
1341			 dev->gadget.state == USB_STATE_ATTACHED);
1342
1343	/* VBUS valid detected */
1344	if (!powstate && (status & GR_STATUS_VB)) {
1345		dev_dbg(dev->dev, "STATUS: vbus valid detected\n");
1346		gr_vbus_connected(dev, status);
1347		handled = 1;
1348	}
1349
1350	/* Disconnect */
1351	if (powstate && !(status & GR_STATUS_VB)) {
1352		dev_dbg(dev->dev, "STATUS: vbus invalid detected\n");
1353		gr_vbus_disconnected(dev);
1354		handled = 1;
1355	}
1356
1357	/* USB reset detected */
1358	if (status & GR_STATUS_UR) {
1359		dev_dbg(dev->dev, "STATUS: USB reset - speed is %s\n",
1360			GR_SPEED_STR(status));
1361		gr_write32(&dev->regs->status, GR_STATUS_UR);
1362		gr_udc_usbreset(dev, status);
1363		handled = 1;
1364	}
1365
1366	/* Speed change */
1367	if (dev->gadget.speed != GR_SPEED(status)) {
1368		dev_dbg(dev->dev, "STATUS: USB Speed change to %s\n",
1369			GR_SPEED_STR(status));
1370		dev->gadget.speed = GR_SPEED(status);
1371		handled = 1;
1372	}
1373
1374	/* Going into suspend */
1375	if ((dev->ep0state != GR_EP0_SUSPEND) && !(status & GR_STATUS_SU)) {
1376		dev_dbg(dev->dev, "STATUS: USB suspend\n");
1377		gr_set_ep0state(dev, GR_EP0_SUSPEND);
1378		dev->suspended_from = dev->gadget.state;
1379		usb_gadget_set_state(&dev->gadget, USB_STATE_SUSPENDED);
1380
1381		if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1382		    dev->driver && dev->driver->suspend) {
1383			spin_unlock(&dev->lock);
1384
1385			dev->driver->suspend(&dev->gadget);
1386
1387			spin_lock(&dev->lock);
1388		}
1389		handled = 1;
1390	}
1391
1392	/* Coming out of suspend */
1393	if ((dev->ep0state == GR_EP0_SUSPEND) && (status & GR_STATUS_SU)) {
1394		dev_dbg(dev->dev, "STATUS: USB resume\n");
1395		if (dev->suspended_from == USB_STATE_POWERED)
1396			gr_set_ep0state(dev, GR_EP0_DISCONNECT);
1397		else
1398			gr_set_ep0state(dev, GR_EP0_SETUP);
1399		usb_gadget_set_state(&dev->gadget, dev->suspended_from);
1400
1401		if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1402		    dev->driver && dev->driver->resume) {
1403			spin_unlock(&dev->lock);
1404
1405			dev->driver->resume(&dev->gadget);
1406
1407			spin_lock(&dev->lock);
1408		}
1409		handled = 1;
1410	}
1411
1412	return handled;
1413}
1414
1415/* Non-interrupt context irq handler */
1416static irqreturn_t gr_irq_handler(int irq, void *_dev)
1417{
1418	struct gr_udc *dev = _dev;
1419	struct gr_ep *ep;
1420	int handled = 0;
1421	int i;
1422	unsigned long flags;
1423
1424	spin_lock_irqsave(&dev->lock, flags);
1425
1426	if (!dev->irq_enabled)
1427		goto out;
1428
1429	/*
1430	 * Check IN ep interrupts. We check these before the OUT eps because
1431	 * some gadgets reuse the request that might already be currently
1432	 * outstanding and needs to be completed (mainly setup requests).
1433	 */
1434	for (i = 0; i < dev->nepi; i++) {
1435		ep = &dev->epi[i];
1436		if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1437			handled = gr_handle_in_ep(ep) || handled;
1438	}
1439
1440	/* Check OUT ep interrupts */
1441	for (i = 0; i < dev->nepo; i++) {
1442		ep = &dev->epo[i];
1443		if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1444			handled = gr_handle_out_ep(ep) || handled;
1445	}
1446
1447	/* Check status interrupts */
1448	handled = gr_handle_state_changes(dev) || handled;
1449
1450	/*
1451	 * Check AMBA DMA errors. Only check if we didn't find anything else to
1452	 * handle because this shouldn't happen if we did everything right.
1453	 */
1454	if (!handled) {
1455		list_for_each_entry(ep, &dev->ep_list, ep_list) {
1456			if (gr_read32(&ep->regs->dmactrl) & GR_DMACTRL_AE) {
1457				dev_err(dev->dev,
1458					"AMBA Error occurred for %s\n",
1459					ep->ep.name);
1460				handled = 1;
1461			}
1462		}
1463	}
1464
1465out:
1466	spin_unlock_irqrestore(&dev->lock, flags);
1467
1468	return handled ? IRQ_HANDLED : IRQ_NONE;
1469}
1470
1471/* Interrupt context irq handler */
1472static irqreturn_t gr_irq(int irq, void *_dev)
1473{
1474	struct gr_udc *dev = _dev;
1475
1476	if (!dev->irq_enabled)
1477		return IRQ_NONE;
1478
1479	return IRQ_WAKE_THREAD;
1480}
1481
1482/* ---------------------------------------------------------------------- */
1483/* USB ep ops */
1484
1485/* Enable endpoint. Not for ep0in and ep0out that are handled separately. */
1486static int gr_ep_enable(struct usb_ep *_ep,
1487			const struct usb_endpoint_descriptor *desc)
1488{
1489	struct gr_udc *dev;
1490	struct gr_ep *ep;
1491	u8 mode;
1492	u8 nt;
1493	u16 max;
1494	u16 buffer_size = 0;
1495	u32 epctrl;
1496
1497	ep = container_of(_ep, struct gr_ep, ep);
1498	if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT)
1499		return -EINVAL;
1500
1501	dev = ep->dev;
1502
1503	/* 'ep0' IN and OUT are reserved */
1504	if (ep == &dev->epo[0] || ep == &dev->epi[0])
1505		return -EINVAL;
1506
1507	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1508		return -ESHUTDOWN;
1509
1510	/* Make sure we are clear for enabling */
1511	epctrl = gr_read32(&ep->regs->epctrl);
1512	if (epctrl & GR_EPCTRL_EV)
1513		return -EBUSY;
1514
1515	/* Check that directions match */
1516	if (!ep->is_in != !usb_endpoint_dir_in(desc))
1517		return -EINVAL;
1518
1519	/* Check ep num */
1520	if ((!ep->is_in && ep->num >= dev->nepo) ||
1521	    (ep->is_in && ep->num >= dev->nepi))
1522		return -EINVAL;
1523
1524	if (usb_endpoint_xfer_control(desc)) {
1525		mode = 0;
1526	} else if (usb_endpoint_xfer_isoc(desc)) {
1527		mode = 1;
1528	} else if (usb_endpoint_xfer_bulk(desc)) {
1529		mode = 2;
1530	} else if (usb_endpoint_xfer_int(desc)) {
1531		mode = 3;
1532	} else {
1533		dev_err(dev->dev, "Unknown transfer type for %s\n",
1534			ep->ep.name);
1535		return -EINVAL;
1536	}
1537
1538	/*
1539	 * Bits 10-0 set the max payload. 12-11 set the number of
1540	 * additional transactions.
1541	 */
1542	max = 0x7ff & usb_endpoint_maxp(desc);
1543	nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
1544	buffer_size = GR_BUFFER_SIZE(epctrl);
1545	if (nt && (mode == 0 || mode == 2)) {
1546		dev_err(dev->dev,
1547			"%s mode: multiple trans./microframe not valid\n",
1548			(mode == 2 ? "Bulk" : "Control"));
1549		return -EINVAL;
1550	} else if (nt == 0x3) {
1551		dev_err(dev->dev,
1552			"Invalid value 0x3 for additional trans./microframe\n");
1553		return -EINVAL;
1554	} else if ((nt + 1) * max > buffer_size) {
1555		dev_err(dev->dev, "Hw buffer size %d < max payload %d * %d\n",
1556			buffer_size, (nt + 1), max);
1557		return -EINVAL;
1558	} else if (max == 0) {
1559		dev_err(dev->dev, "Max payload cannot be set to 0\n");
1560		return -EINVAL;
1561	} else if (max > ep->ep.maxpacket_limit) {
1562		dev_err(dev->dev, "Requested max payload %d > limit %d\n",
1563			max, ep->ep.maxpacket_limit);
1564		return -EINVAL;
1565	}
1566
1567	spin_lock(&ep->dev->lock);
1568
1569	if (!ep->stopped) {
1570		spin_unlock(&ep->dev->lock);
1571		return -EBUSY;
1572	}
1573
1574	ep->stopped = 0;
1575	ep->wedged = 0;
1576	ep->ep.desc = desc;
1577	ep->ep.maxpacket = max;
1578	ep->dma_start = 0;
1579
1580
1581	if (nt) {
1582		/*
1583		 * Maximum possible size of all payloads in one microframe
1584		 * regardless of direction when using high-bandwidth mode.
1585		 */
1586		ep->bytes_per_buffer = (nt + 1) * max;
1587	} else if (ep->is_in) {
1588		/*
1589		 * The biggest multiple of maximum packet size that fits into
1590		 * the buffer. The hardware will split up into many packets in
1591		 * the IN direction.
1592		 */
1593		ep->bytes_per_buffer = (buffer_size / max) * max;
1594	} else {
1595		/*
1596		 * Only single packets will be placed the buffers in the OUT
1597		 * direction.
1598		 */
1599		ep->bytes_per_buffer = max;
1600	}
1601
1602	epctrl = (max << GR_EPCTRL_MAXPL_POS)
1603		| (nt << GR_EPCTRL_NT_POS)
1604		| (mode << GR_EPCTRL_TT_POS)
1605		| GR_EPCTRL_EV;
1606	if (ep->is_in)
1607		epctrl |= GR_EPCTRL_PI;
1608	gr_write32(&ep->regs->epctrl, epctrl);
1609
1610	gr_write32(&ep->regs->dmactrl, GR_DMACTRL_IE | GR_DMACTRL_AI);
1611
1612	spin_unlock(&ep->dev->lock);
1613
1614	dev_dbg(ep->dev->dev, "EP: %s enabled - %s with %d bytes/buffer\n",
1615		ep->ep.name, gr_modestring[mode], ep->bytes_per_buffer);
1616	return 0;
1617}
1618
1619/* Disable endpoint. Not for ep0in and ep0out that are handled separately. */
1620static int gr_ep_disable(struct usb_ep *_ep)
1621{
1622	struct gr_ep *ep;
1623	struct gr_udc *dev;
1624	unsigned long flags;
1625
1626	ep = container_of(_ep, struct gr_ep, ep);
1627	if (!_ep || !ep->ep.desc)
1628		return -ENODEV;
1629
1630	dev = ep->dev;
1631
1632	/* 'ep0' IN and OUT are reserved */
1633	if (ep == &dev->epo[0] || ep == &dev->epi[0])
1634		return -EINVAL;
1635
1636	if (dev->ep0state == GR_EP0_SUSPEND)
1637		return -EBUSY;
1638
1639	dev_dbg(ep->dev->dev, "EP: disable %s\n", ep->ep.name);
1640
1641	spin_lock_irqsave(&dev->lock, flags);
1642
1643	gr_ep_nuke(ep);
1644	gr_ep_reset(ep);
1645	ep->ep.desc = NULL;
1646
1647	spin_unlock_irqrestore(&dev->lock, flags);
1648
1649	return 0;
1650}
1651
1652/*
1653 * Frees a request, but not any DMA buffers associated with it
1654 * (gr_finish_request should already have taken care of that).
1655 */
1656static void gr_free_request(struct usb_ep *_ep, struct usb_request *_req)
1657{
1658	struct gr_request *req;
1659
1660	if (!_ep || !_req)
1661		return;
1662	req = container_of(_req, struct gr_request, req);
1663
1664	/* Leads to memory leak */
1665	WARN(!list_empty(&req->queue),
1666	     "request not dequeued properly before freeing\n");
1667
1668	kfree(req);
1669}
1670
1671/* Queue a request from the gadget */
1672static int gr_queue_ext(struct usb_ep *_ep, struct usb_request *_req,
1673			gfp_t gfp_flags)
1674{
1675	struct gr_ep *ep;
1676	struct gr_request *req;
1677	struct gr_udc *dev;
1678	int ret;
1679
1680	if (unlikely(!_ep || !_req))
1681		return -EINVAL;
1682
1683	ep = container_of(_ep, struct gr_ep, ep);
1684	req = container_of(_req, struct gr_request, req);
1685	dev = ep->dev;
1686
1687	spin_lock(&ep->dev->lock);
1688
1689	/*
1690	 * The ep0 pointer in the gadget struct is used both for ep0in and
1691	 * ep0out. In a data stage in the out direction ep0out needs to be used
1692	 * instead of the default ep0in. Completion functions might use
1693	 * driver_data, so that needs to be copied as well.
1694	 */
1695	if ((ep == &dev->epi[0]) && (dev->ep0state == GR_EP0_ODATA)) {
1696		ep = &dev->epo[0];
1697		ep->ep.driver_data = dev->epi[0].ep.driver_data;
1698	}
1699
1700	if (ep->is_in)
1701		gr_dbgprint_request("EXTERN", ep, req);
1702
1703	ret = gr_queue(ep, req, GFP_ATOMIC);
1704
1705	spin_unlock(&ep->dev->lock);
1706
1707	return ret;
1708}
1709
1710/* Dequeue JUST ONE request */
1711static int gr_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1712{
1713	struct gr_request *req;
1714	struct gr_ep *ep;
1715	struct gr_udc *dev;
1716	int ret = 0;
1717	unsigned long flags;
1718
1719	ep = container_of(_ep, struct gr_ep, ep);
1720	if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
1721		return -EINVAL;
1722	dev = ep->dev;
1723	if (!dev->driver)
1724		return -ESHUTDOWN;
1725
1726	/* We can't touch (DMA) registers when suspended */
1727	if (dev->ep0state == GR_EP0_SUSPEND)
1728		return -EBUSY;
1729
1730	spin_lock_irqsave(&dev->lock, flags);
1731
1732	/* Make sure it's actually queued on this endpoint */
1733	list_for_each_entry(req, &ep->queue, queue) {
1734		if (&req->req == _req)
1735			break;
1736	}
1737	if (&req->req != _req) {
1738		ret = -EINVAL;
1739		goto out;
1740	}
1741
1742	if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {
1743		/* This request is currently being processed */
1744		gr_abort_dma(ep);
1745		if (ep->stopped)
1746			gr_finish_request(ep, req, -ECONNRESET);
1747		else
1748			gr_dma_advance(ep, -ECONNRESET);
1749	} else if (!list_empty(&req->queue)) {
1750		/* Not being processed - gr_finish_request dequeues it */
1751		gr_finish_request(ep, req, -ECONNRESET);
1752	} else {
1753		ret = -EOPNOTSUPP;
1754	}
1755
1756out:
1757	spin_unlock_irqrestore(&dev->lock, flags);
1758
1759	return ret;
1760}
1761
1762/* Helper for gr_set_halt and gr_set_wedge */
1763static int gr_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
1764{
1765	int ret;
1766	struct gr_ep *ep;
1767
1768	if (!_ep)
1769		return -ENODEV;
1770	ep = container_of(_ep, struct gr_ep, ep);
1771
1772	spin_lock(&ep->dev->lock);
1773
1774	/* Halting an IN endpoint should fail if queue is not empty */
1775	if (halt && ep->is_in && !list_empty(&ep->queue)) {
1776		ret = -EAGAIN;
1777		goto out;
1778	}
1779
1780	ret = gr_ep_halt_wedge(ep, halt, wedge, 0);
1781
1782out:
1783	spin_unlock(&ep->dev->lock);
1784
1785	return ret;
1786}
1787
1788/* Halt endpoint */
1789static int gr_set_halt(struct usb_ep *_ep, int halt)
1790{
1791	return gr_set_halt_wedge(_ep, halt, 0);
1792}
1793
1794/* Halt and wedge endpoint */
1795static int gr_set_wedge(struct usb_ep *_ep)
1796{
1797	return gr_set_halt_wedge(_ep, 1, 1);
1798}
1799
1800/*
1801 * Return the total number of bytes currently stored in the internal buffers of
1802 * the endpoint.
1803 */
1804static int gr_fifo_status(struct usb_ep *_ep)
1805{
1806	struct gr_ep *ep;
1807	u32 epstat;
1808	u32 bytes = 0;
1809
1810	if (!_ep)
1811		return -ENODEV;
1812	ep = container_of(_ep, struct gr_ep, ep);
1813
1814	epstat = gr_read32(&ep->regs->epstat);
1815
1816	if (epstat & GR_EPSTAT_B0)
1817		bytes += (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS;
1818	if (epstat & GR_EPSTAT_B1)
1819		bytes += (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS;
1820
1821	return bytes;
1822}
1823
1824
1825/* Empty data from internal buffers of an endpoint. */
1826static void gr_fifo_flush(struct usb_ep *_ep)
1827{
1828	struct gr_ep *ep;
1829	u32 epctrl;
1830
1831	if (!_ep)
1832		return;
1833	ep = container_of(_ep, struct gr_ep, ep);
1834	dev_vdbg(ep->dev->dev, "EP: flush fifo %s\n", ep->ep.name);
1835
1836	spin_lock(&ep->dev->lock);
1837
1838	epctrl = gr_read32(&ep->regs->epctrl);
1839	epctrl |= GR_EPCTRL_CB;
1840	gr_write32(&ep->regs->epctrl, epctrl);
1841
1842	spin_unlock(&ep->dev->lock);
1843}
1844
1845static struct usb_ep_ops gr_ep_ops = {
1846	.enable		= gr_ep_enable,
1847	.disable	= gr_ep_disable,
1848
1849	.alloc_request	= gr_alloc_request,
1850	.free_request	= gr_free_request,
1851
1852	.queue		= gr_queue_ext,
1853	.dequeue	= gr_dequeue,
1854
1855	.set_halt	= gr_set_halt,
1856	.set_wedge	= gr_set_wedge,
1857	.fifo_status	= gr_fifo_status,
1858	.fifo_flush	= gr_fifo_flush,
1859};
1860
1861/* ---------------------------------------------------------------------- */
1862/* USB Gadget ops */
1863
1864static int gr_get_frame(struct usb_gadget *_gadget)
1865{
1866	struct gr_udc *dev;
1867
1868	if (!_gadget)
1869		return -ENODEV;
1870	dev = container_of(_gadget, struct gr_udc, gadget);
1871	return gr_read32(&dev->regs->status) & GR_STATUS_FN_MASK;
1872}
1873
1874static int gr_wakeup(struct usb_gadget *_gadget)
1875{
1876	struct gr_udc *dev;
1877
1878	if (!_gadget)
1879		return -ENODEV;
1880	dev = container_of(_gadget, struct gr_udc, gadget);
1881
1882	/* Remote wakeup feature not enabled by host*/
1883	if (!dev->remote_wakeup)
1884		return -EINVAL;
1885
1886	spin_lock(&dev->lock);
1887
1888	gr_write32(&dev->regs->control,
1889		   gr_read32(&dev->regs->control) | GR_CONTROL_RW);
1890
1891	spin_unlock(&dev->lock);
1892
1893	return 0;
1894}
1895
1896static int gr_pullup(struct usb_gadget *_gadget, int is_on)
1897{
1898	struct gr_udc *dev;
1899	u32 control;
1900
1901	if (!_gadget)
1902		return -ENODEV;
1903	dev = container_of(_gadget, struct gr_udc, gadget);
1904
1905	spin_lock(&dev->lock);
1906
1907	control = gr_read32(&dev->regs->control);
1908	if (is_on)
1909		control |= GR_CONTROL_EP;
1910	else
1911		control &= ~GR_CONTROL_EP;
1912	gr_write32(&dev->regs->control, control);
1913
1914	spin_unlock(&dev->lock);
1915
1916	return 0;
1917}
1918
1919static int gr_udc_start(struct usb_gadget *gadget,
1920			struct usb_gadget_driver *driver)
1921{
1922	struct gr_udc *dev = to_gr_udc(gadget);
1923
1924	spin_lock(&dev->lock);
1925
1926	/* Hook up the driver */
1927	driver->driver.bus = NULL;
1928	dev->driver = driver;
1929
1930	/* Get ready for host detection */
1931	gr_enable_vbus_detect(dev);
1932
1933	spin_unlock(&dev->lock);
1934
1935	return 0;
1936}
1937
1938static int gr_udc_stop(struct usb_gadget *gadget)
1939{
1940	struct gr_udc *dev = to_gr_udc(gadget);
1941	unsigned long flags;
1942
1943	spin_lock_irqsave(&dev->lock, flags);
1944
1945	dev->driver = NULL;
1946	gr_stop_activity(dev);
1947
1948	spin_unlock_irqrestore(&dev->lock, flags);
1949
1950	return 0;
1951}
1952
1953static const struct usb_gadget_ops gr_ops = {
1954	.get_frame	= gr_get_frame,
1955	.wakeup         = gr_wakeup,
1956	.pullup         = gr_pullup,
1957	.udc_start	= gr_udc_start,
1958	.udc_stop	= gr_udc_stop,
1959	/* Other operations not supported */
1960};
1961
1962/* ---------------------------------------------------------------------- */
1963/* Module probe, removal and of-matching */
1964
1965static const char * const onames[] = {
1966	"ep0out", "ep1out", "ep2out", "ep3out", "ep4out", "ep5out",
1967	"ep6out", "ep7out", "ep8out", "ep9out", "ep10out", "ep11out",
1968	"ep12out", "ep13out", "ep14out", "ep15out"
1969};
1970
1971static const char * const inames[] = {
1972	"ep0in", "ep1in", "ep2in", "ep3in", "ep4in", "ep5in",
1973	"ep6in", "ep7in", "ep8in", "ep9in", "ep10in", "ep11in",
1974	"ep12in", "ep13in", "ep14in", "ep15in"
1975};
1976
1977/* Must be called with dev->lock held */
1978static int gr_ep_init(struct gr_udc *dev, int num, int is_in, u32 maxplimit)
1979{
1980	struct gr_ep *ep;
1981	struct gr_request *req;
1982	struct usb_request *_req;
1983	void *buf;
1984
1985	if (is_in) {
1986		ep = &dev->epi[num];
1987		ep->ep.name = inames[num];
1988		ep->regs = &dev->regs->epi[num];
1989	} else {
1990		ep = &dev->epo[num];
1991		ep->ep.name = onames[num];
1992		ep->regs = &dev->regs->epo[num];
1993	}
1994
1995	gr_ep_reset(ep);
1996	ep->num = num;
1997	ep->is_in = is_in;
1998	ep->dev = dev;
1999	ep->ep.ops = &gr_ep_ops;
2000	INIT_LIST_HEAD(&ep->queue);
2001
2002	if (num == 0) {
2003		_req = gr_alloc_request(&ep->ep, GFP_ATOMIC);
2004		buf = devm_kzalloc(dev->dev, PAGE_SIZE, GFP_DMA | GFP_ATOMIC);
2005		if (!_req || !buf) {
2006			/* possible _req freed by gr_probe via gr_remove */
2007			return -ENOMEM;
2008		}
2009
2010		req = container_of(_req, struct gr_request, req);
2011		req->req.buf = buf;
2012		req->req.length = MAX_CTRL_PL_SIZE;
2013
2014		if (is_in)
2015			dev->ep0reqi = req; /* Complete gets set as used */
2016		else
2017			dev->ep0reqo = req; /* Completion treated separately */
2018
2019		usb_ep_set_maxpacket_limit(&ep->ep, MAX_CTRL_PL_SIZE);
2020		ep->bytes_per_buffer = MAX_CTRL_PL_SIZE;
2021	} else {
2022		usb_ep_set_maxpacket_limit(&ep->ep, (u16)maxplimit);
2023		list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
2024	}
2025	list_add_tail(&ep->ep_list, &dev->ep_list);
2026
2027	ep->tailbuf = dma_alloc_coherent(dev->dev, ep->ep.maxpacket_limit,
2028					 &ep->tailbuf_paddr, GFP_ATOMIC);
2029	if (!ep->tailbuf)
2030		return -ENOMEM;
2031
2032	return 0;
2033}
2034
2035/* Must be called with dev->lock held */
2036static int gr_udc_init(struct gr_udc *dev)
2037{
2038	struct device_node *np = dev->dev->of_node;
2039	u32 epctrl_val;
2040	u32 dmactrl_val;
2041	int i;
2042	int ret = 0;
2043	u32 bufsize;
2044
2045	gr_set_address(dev, 0);
2046
2047	INIT_LIST_HEAD(&dev->gadget.ep_list);
2048	dev->gadget.speed = USB_SPEED_UNKNOWN;
2049	dev->gadget.ep0 = &dev->epi[0].ep;
2050
2051	INIT_LIST_HEAD(&dev->ep_list);
2052	gr_set_ep0state(dev, GR_EP0_DISCONNECT);
2053
2054	for (i = 0; i < dev->nepo; i++) {
2055		if (of_property_read_u32_index(np, "epobufsizes", i, &bufsize))
2056			bufsize = 1024;
2057		ret = gr_ep_init(dev, i, 0, bufsize);
2058		if (ret)
2059			return ret;
2060	}
2061
2062	for (i = 0; i < dev->nepi; i++) {
2063		if (of_property_read_u32_index(np, "epibufsizes", i, &bufsize))
2064			bufsize = 1024;
2065		ret = gr_ep_init(dev, i, 1, bufsize);
2066		if (ret)
2067			return ret;
2068	}
2069
2070	/* Must be disabled by default */
2071	dev->remote_wakeup = 0;
2072
2073	/* Enable ep0out and ep0in */
2074	epctrl_val = (MAX_CTRL_PL_SIZE << GR_EPCTRL_MAXPL_POS) | GR_EPCTRL_EV;
2075	dmactrl_val = GR_DMACTRL_IE | GR_DMACTRL_AI;
2076	gr_write32(&dev->epo[0].regs->epctrl, epctrl_val);
2077	gr_write32(&dev->epi[0].regs->epctrl, epctrl_val | GR_EPCTRL_PI);
2078	gr_write32(&dev->epo[0].regs->dmactrl, dmactrl_val);
2079	gr_write32(&dev->epi[0].regs->dmactrl, dmactrl_val);
2080
2081	return 0;
2082}
2083
2084static void gr_ep_remove(struct gr_udc *dev, int num, int is_in)
2085{
2086	struct gr_ep *ep;
2087
2088	if (is_in)
2089		ep = &dev->epi[num];
2090	else
2091		ep = &dev->epo[num];
2092
2093	if (ep->tailbuf)
2094		dma_free_coherent(dev->dev, ep->ep.maxpacket_limit,
2095				  ep->tailbuf, ep->tailbuf_paddr);
2096}
2097
2098static int gr_remove(struct platform_device *pdev)
2099{
2100	struct gr_udc *dev = platform_get_drvdata(pdev);
2101	int i;
2102
2103	if (dev->added)
2104		usb_del_gadget_udc(&dev->gadget); /* Shuts everything down */
2105	if (dev->driver)
2106		return -EBUSY;
2107
2108	gr_dfs_delete(dev);
2109	if (dev->desc_pool)
2110		dma_pool_destroy(dev->desc_pool);
2111	platform_set_drvdata(pdev, NULL);
2112
2113	gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
2114	gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);
2115
2116	for (i = 0; i < dev->nepo; i++)
2117		gr_ep_remove(dev, i, 0);
2118	for (i = 0; i < dev->nepi; i++)
2119		gr_ep_remove(dev, i, 1);
2120
2121	return 0;
2122}
2123static int gr_request_irq(struct gr_udc *dev, int irq)
2124{
2125	return devm_request_threaded_irq(dev->dev, irq, gr_irq, gr_irq_handler,
2126					 IRQF_SHARED, driver_name, dev);
2127}
2128
2129static int gr_probe(struct platform_device *pdev)
2130{
2131	struct gr_udc *dev;
2132	struct resource *res;
2133	struct gr_regs __iomem *regs;
2134	int retval;
2135	u32 status;
2136
2137	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
2138	if (!dev)
2139		return -ENOMEM;
2140	dev->dev = &pdev->dev;
2141
2142	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2143	regs = devm_ioremap_resource(dev->dev, res);
2144	if (IS_ERR(regs))
2145		return PTR_ERR(regs);
2146
2147	dev->irq = platform_get_irq(pdev, 0);
2148	if (dev->irq <= 0) {
2149		dev_err(dev->dev, "No irq found\n");
2150		return -ENODEV;
2151	}
2152
2153	/* Some core configurations has separate irqs for IN and OUT events */
2154	dev->irqi = platform_get_irq(pdev, 1);
2155	if (dev->irqi > 0) {
2156		dev->irqo = platform_get_irq(pdev, 2);
2157		if (dev->irqo <= 0) {
2158			dev_err(dev->dev, "Found irqi but not irqo\n");
2159			return -ENODEV;
2160		}
2161	} else {
2162		dev->irqi = 0;
2163	}
2164
2165	dev->gadget.name = driver_name;
2166	dev->gadget.max_speed = USB_SPEED_HIGH;
2167	dev->gadget.ops = &gr_ops;
2168
2169	spin_lock_init(&dev->lock);
2170	dev->regs = regs;
2171
2172	platform_set_drvdata(pdev, dev);
2173
2174	/* Determine number of endpoints and data interface mode */
2175	status = gr_read32(&dev->regs->status);
2176	dev->nepi = ((status & GR_STATUS_NEPI_MASK) >> GR_STATUS_NEPI_POS) + 1;
2177	dev->nepo = ((status & GR_STATUS_NEPO_MASK) >> GR_STATUS_NEPO_POS) + 1;
2178
2179	if (!(status & GR_STATUS_DM)) {
2180		dev_err(dev->dev, "Slave mode cores are not supported\n");
2181		return -ENODEV;
2182	}
2183
2184	/* --- Effects of the following calls might need explicit cleanup --- */
2185
2186	/* Create DMA pool for descriptors */
2187	dev->desc_pool = dma_pool_create("desc_pool", dev->dev,
2188					 sizeof(struct gr_dma_desc), 4, 0);
2189	if (!dev->desc_pool) {
2190		dev_err(dev->dev, "Could not allocate DMA pool");
2191		return -ENOMEM;
2192	}
2193
2194	spin_lock(&dev->lock);
2195
2196	/* Inside lock so that no gadget can use this udc until probe is done */
2197	retval = usb_add_gadget_udc(dev->dev, &dev->gadget);
2198	if (retval) {
2199		dev_err(dev->dev, "Could not add gadget udc");
2200		goto out;
2201	}
2202	dev->added = 1;
2203
2204	retval = gr_udc_init(dev);
2205	if (retval)
2206		goto out;
2207
2208	gr_dfs_create(dev);
2209
2210	/* Clear all interrupt enables that might be left on since last boot */
2211	gr_disable_interrupts_and_pullup(dev);
2212
2213	retval = gr_request_irq(dev, dev->irq);
2214	if (retval) {
2215		dev_err(dev->dev, "Failed to request irq %d\n", dev->irq);
2216		goto out;
2217	}
2218
2219	if (dev->irqi) {
2220		retval = gr_request_irq(dev, dev->irqi);
2221		if (retval) {
2222			dev_err(dev->dev, "Failed to request irqi %d\n",
2223				dev->irqi);
2224			goto out;
2225		}
2226		retval = gr_request_irq(dev, dev->irqo);
2227		if (retval) {
2228			dev_err(dev->dev, "Failed to request irqo %d\n",
2229				dev->irqo);
2230			goto out;
2231		}
2232	}
2233
2234	if (dev->irqi)
2235		dev_info(dev->dev, "regs: %p, irqs %d, %d, %d\n", dev->regs,
2236			 dev->irq, dev->irqi, dev->irqo);
2237	else
2238		dev_info(dev->dev, "regs: %p, irq %d\n", dev->regs, dev->irq);
2239
2240out:
2241	spin_unlock(&dev->lock);
2242
2243	if (retval)
2244		gr_remove(pdev);
2245
2246	return retval;
2247}
2248
2249static const struct of_device_id gr_match[] = {
2250	{.name = "GAISLER_USBDC"},
2251	{.name = "01_021"},
2252	{},
2253};
2254MODULE_DEVICE_TABLE(of, gr_match);
2255
2256static struct platform_driver gr_driver = {
2257	.driver = {
2258		.name = DRIVER_NAME,
2259		.of_match_table = gr_match,
2260	},
2261	.probe = gr_probe,
2262	.remove = gr_remove,
2263};
2264module_platform_driver(gr_driver);
2265
2266MODULE_AUTHOR("Aeroflex Gaisler AB.");
2267MODULE_DESCRIPTION(DRIVER_DESC);
2268MODULE_LICENSE("GPL");
2269