1/*
2 * FCC driver for Motorola MPC82xx (PQ2).
3 *
4 * Copyright (c) 2003 Intracom S.A.
5 *  by Pantelis Antoniou <panto@intracom.gr>
6 *
7 * 2005 (c) MontaVista Software, Inc.
8 * Vitaly Bordug <vbordug@ru.mvista.com>
9 *
10 * This file is licensed under the terms of the GNU General Public License
11 * version 2. This program is licensed "as is" without any warranty of any
12 * kind, whether express or implied.
13 */
14
15#include <linux/module.h>
16#include <linux/kernel.h>
17#include <linux/types.h>
18#include <linux/string.h>
19#include <linux/ptrace.h>
20#include <linux/errno.h>
21#include <linux/ioport.h>
22#include <linux/interrupt.h>
23#include <linux/delay.h>
24#include <linux/netdevice.h>
25#include <linux/etherdevice.h>
26#include <linux/skbuff.h>
27#include <linux/spinlock.h>
28#include <linux/mii.h>
29#include <linux/ethtool.h>
30#include <linux/bitops.h>
31#include <linux/fs.h>
32#include <linux/platform_device.h>
33#include <linux/phy.h>
34#include <linux/of_address.h>
35#include <linux/of_device.h>
36#include <linux/of_irq.h>
37#include <linux/gfp.h>
38
39#include <asm/immap_cpm2.h>
40#include <asm/mpc8260.h>
41#include <asm/cpm2.h>
42
43#include <asm/pgtable.h>
44#include <asm/irq.h>
45#include <asm/uaccess.h>
46
47#include "fs_enet.h"
48
49/*************************************************/
50
51/* FCC access macros */
52
53/* write, read, set bits, clear bits */
54#define W32(_p, _m, _v)	out_be32(&(_p)->_m, (_v))
55#define R32(_p, _m)	in_be32(&(_p)->_m)
56#define S32(_p, _m, _v)	W32(_p, _m, R32(_p, _m) | (_v))
57#define C32(_p, _m, _v)	W32(_p, _m, R32(_p, _m) & ~(_v))
58
59#define W16(_p, _m, _v)	out_be16(&(_p)->_m, (_v))
60#define R16(_p, _m)	in_be16(&(_p)->_m)
61#define S16(_p, _m, _v)	W16(_p, _m, R16(_p, _m) | (_v))
62#define C16(_p, _m, _v)	W16(_p, _m, R16(_p, _m) & ~(_v))
63
64#define W8(_p, _m, _v)	out_8(&(_p)->_m, (_v))
65#define R8(_p, _m)	in_8(&(_p)->_m)
66#define S8(_p, _m, _v)	W8(_p, _m, R8(_p, _m) | (_v))
67#define C8(_p, _m, _v)	W8(_p, _m, R8(_p, _m) & ~(_v))
68
69/*************************************************/
70
71#define FCC_MAX_MULTICAST_ADDRS	64
72
73#define mk_mii_read(REG)	(0x60020000 | ((REG & 0x1f) << 18))
74#define mk_mii_write(REG, VAL)	(0x50020000 | ((REG & 0x1f) << 18) | (VAL & 0xffff))
75#define mk_mii_end		0
76
77#define MAX_CR_CMD_LOOPS	10000
78
79static inline int fcc_cr_cmd(struct fs_enet_private *fep, u32 op)
80{
81	const struct fs_platform_info *fpi = fep->fpi;
82
83	return cpm_command(fpi->cp_command, op);
84}
85
86static int do_pd_setup(struct fs_enet_private *fep)
87{
88	struct platform_device *ofdev = to_platform_device(fep->dev);
89	struct fs_platform_info *fpi = fep->fpi;
90	int ret = -EINVAL;
91
92	fep->interrupt = irq_of_parse_and_map(ofdev->dev.of_node, 0);
93	if (fep->interrupt == NO_IRQ)
94		goto out;
95
96	fep->fcc.fccp = of_iomap(ofdev->dev.of_node, 0);
97	if (!fep->fcc.fccp)
98		goto out;
99
100	fep->fcc.ep = of_iomap(ofdev->dev.of_node, 1);
101	if (!fep->fcc.ep)
102		goto out_fccp;
103
104	fep->fcc.fcccp = of_iomap(ofdev->dev.of_node, 2);
105	if (!fep->fcc.fcccp)
106		goto out_ep;
107
108	fep->fcc.mem = (void __iomem *)cpm2_immr;
109	fpi->dpram_offset = cpm_dpalloc(128, 32);
110	if (IS_ERR_VALUE(fpi->dpram_offset)) {
111		ret = fpi->dpram_offset;
112		goto out_fcccp;
113	}
114
115	return 0;
116
117out_fcccp:
118	iounmap(fep->fcc.fcccp);
119out_ep:
120	iounmap(fep->fcc.ep);
121out_fccp:
122	iounmap(fep->fcc.fccp);
123out:
124	return ret;
125}
126
127#define FCC_NAPI_RX_EVENT_MSK	(FCC_ENET_RXF | FCC_ENET_RXB)
128#define FCC_NAPI_TX_EVENT_MSK	(FCC_ENET_TXB)
129#define FCC_RX_EVENT		(FCC_ENET_RXF)
130#define FCC_TX_EVENT		(FCC_ENET_TXB)
131#define FCC_ERR_EVENT_MSK	(FCC_ENET_TXE)
132
133static int setup_data(struct net_device *dev)
134{
135	struct fs_enet_private *fep = netdev_priv(dev);
136
137	if (do_pd_setup(fep) != 0)
138		return -EINVAL;
139
140	fep->ev_napi_rx = FCC_NAPI_RX_EVENT_MSK;
141	fep->ev_napi_tx = FCC_NAPI_TX_EVENT_MSK;
142	fep->ev_rx = FCC_RX_EVENT;
143	fep->ev_tx = FCC_TX_EVENT;
144	fep->ev_err = FCC_ERR_EVENT_MSK;
145
146	return 0;
147}
148
149static int allocate_bd(struct net_device *dev)
150{
151	struct fs_enet_private *fep = netdev_priv(dev);
152	const struct fs_platform_info *fpi = fep->fpi;
153
154	fep->ring_base = (void __iomem __force *)dma_alloc_coherent(fep->dev,
155					    (fpi->tx_ring + fpi->rx_ring) *
156					    sizeof(cbd_t), &fep->ring_mem_addr,
157					    GFP_KERNEL);
158	if (fep->ring_base == NULL)
159		return -ENOMEM;
160
161	return 0;
162}
163
164static void free_bd(struct net_device *dev)
165{
166	struct fs_enet_private *fep = netdev_priv(dev);
167	const struct fs_platform_info *fpi = fep->fpi;
168
169	if (fep->ring_base)
170		dma_free_coherent(fep->dev,
171			(fpi->tx_ring + fpi->rx_ring) * sizeof(cbd_t),
172			(void __force *)fep->ring_base, fep->ring_mem_addr);
173}
174
175static void cleanup_data(struct net_device *dev)
176{
177	/* nothing */
178}
179
180static void set_promiscuous_mode(struct net_device *dev)
181{
182	struct fs_enet_private *fep = netdev_priv(dev);
183	fcc_t __iomem *fccp = fep->fcc.fccp;
184
185	S32(fccp, fcc_fpsmr, FCC_PSMR_PRO);
186}
187
188static void set_multicast_start(struct net_device *dev)
189{
190	struct fs_enet_private *fep = netdev_priv(dev);
191	fcc_enet_t __iomem *ep = fep->fcc.ep;
192
193	W32(ep, fen_gaddrh, 0);
194	W32(ep, fen_gaddrl, 0);
195}
196
197static void set_multicast_one(struct net_device *dev, const u8 *mac)
198{
199	struct fs_enet_private *fep = netdev_priv(dev);
200	fcc_enet_t __iomem *ep = fep->fcc.ep;
201	u16 taddrh, taddrm, taddrl;
202
203	taddrh = ((u16)mac[5] << 8) | mac[4];
204	taddrm = ((u16)mac[3] << 8) | mac[2];
205	taddrl = ((u16)mac[1] << 8) | mac[0];
206
207	W16(ep, fen_taddrh, taddrh);
208	W16(ep, fen_taddrm, taddrm);
209	W16(ep, fen_taddrl, taddrl);
210	fcc_cr_cmd(fep, CPM_CR_SET_GADDR);
211}
212
213static void set_multicast_finish(struct net_device *dev)
214{
215	struct fs_enet_private *fep = netdev_priv(dev);
216	fcc_t __iomem *fccp = fep->fcc.fccp;
217	fcc_enet_t __iomem *ep = fep->fcc.ep;
218
219	/* clear promiscuous always */
220	C32(fccp, fcc_fpsmr, FCC_PSMR_PRO);
221
222	/* if all multi or too many multicasts; just enable all */
223	if ((dev->flags & IFF_ALLMULTI) != 0 ||
224	    netdev_mc_count(dev) > FCC_MAX_MULTICAST_ADDRS) {
225
226		W32(ep, fen_gaddrh, 0xffffffff);
227		W32(ep, fen_gaddrl, 0xffffffff);
228	}
229
230	/* read back */
231	fep->fcc.gaddrh = R32(ep, fen_gaddrh);
232	fep->fcc.gaddrl = R32(ep, fen_gaddrl);
233}
234
235static void set_multicast_list(struct net_device *dev)
236{
237	struct netdev_hw_addr *ha;
238
239	if ((dev->flags & IFF_PROMISC) == 0) {
240		set_multicast_start(dev);
241		netdev_for_each_mc_addr(ha, dev)
242			set_multicast_one(dev, ha->addr);
243		set_multicast_finish(dev);
244	} else
245		set_promiscuous_mode(dev);
246}
247
248static void restart(struct net_device *dev)
249{
250	struct fs_enet_private *fep = netdev_priv(dev);
251	const struct fs_platform_info *fpi = fep->fpi;
252	fcc_t __iomem *fccp = fep->fcc.fccp;
253	fcc_c_t __iomem *fcccp = fep->fcc.fcccp;
254	fcc_enet_t __iomem *ep = fep->fcc.ep;
255	dma_addr_t rx_bd_base_phys, tx_bd_base_phys;
256	u16 paddrh, paddrm, paddrl;
257	const unsigned char *mac;
258	int i;
259
260	C32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
261
262	/* clear everything (slow & steady does it) */
263	for (i = 0; i < sizeof(*ep); i++)
264		out_8((u8 __iomem *)ep + i, 0);
265
266	/* get physical address */
267	rx_bd_base_phys = fep->ring_mem_addr;
268	tx_bd_base_phys = rx_bd_base_phys + sizeof(cbd_t) * fpi->rx_ring;
269
270	/* point to bds */
271	W32(ep, fen_genfcc.fcc_rbase, rx_bd_base_phys);
272	W32(ep, fen_genfcc.fcc_tbase, tx_bd_base_phys);
273
274	/* Set maximum bytes per receive buffer.
275	 * It must be a multiple of 32.
276	 */
277	W16(ep, fen_genfcc.fcc_mrblr, PKT_MAXBLR_SIZE);
278
279	W32(ep, fen_genfcc.fcc_rstate, (CPMFCR_GBL | CPMFCR_EB) << 24);
280	W32(ep, fen_genfcc.fcc_tstate, (CPMFCR_GBL | CPMFCR_EB) << 24);
281
282	/* Allocate space in the reserved FCC area of DPRAM for the
283	 * internal buffers.  No one uses this space (yet), so we
284	 * can do this.  Later, we will add resource management for
285	 * this area.
286	 */
287
288	W16(ep, fen_genfcc.fcc_riptr, fpi->dpram_offset);
289	W16(ep, fen_genfcc.fcc_tiptr, fpi->dpram_offset + 32);
290
291	W16(ep, fen_padptr, fpi->dpram_offset + 64);
292
293	/* fill with special symbol...  */
294	memset_io(fep->fcc.mem + fpi->dpram_offset + 64, 0x88, 32);
295
296	W32(ep, fen_genfcc.fcc_rbptr, 0);
297	W32(ep, fen_genfcc.fcc_tbptr, 0);
298	W32(ep, fen_genfcc.fcc_rcrc, 0);
299	W32(ep, fen_genfcc.fcc_tcrc, 0);
300	W16(ep, fen_genfcc.fcc_res1, 0);
301	W32(ep, fen_genfcc.fcc_res2, 0);
302
303	/* no CAM */
304	W32(ep, fen_camptr, 0);
305
306	/* Set CRC preset and mask */
307	W32(ep, fen_cmask, 0xdebb20e3);
308	W32(ep, fen_cpres, 0xffffffff);
309
310	W32(ep, fen_crcec, 0);		/* CRC Error counter       */
311	W32(ep, fen_alec, 0);		/* alignment error counter */
312	W32(ep, fen_disfc, 0);		/* discard frame counter   */
313	W16(ep, fen_retlim, 15);	/* Retry limit threshold   */
314	W16(ep, fen_pper, 0);		/* Normal persistence      */
315
316	/* set group address */
317	W32(ep, fen_gaddrh, fep->fcc.gaddrh);
318	W32(ep, fen_gaddrl, fep->fcc.gaddrh);
319
320	/* Clear hash filter tables */
321	W32(ep, fen_iaddrh, 0);
322	W32(ep, fen_iaddrl, 0);
323
324	/* Clear the Out-of-sequence TxBD  */
325	W16(ep, fen_tfcstat, 0);
326	W16(ep, fen_tfclen, 0);
327	W32(ep, fen_tfcptr, 0);
328
329	W16(ep, fen_mflr, PKT_MAXBUF_SIZE);	/* maximum frame length register */
330	W16(ep, fen_minflr, PKT_MINBUF_SIZE);	/* minimum frame length register */
331
332	/* set address */
333	mac = dev->dev_addr;
334	paddrh = ((u16)mac[5] << 8) | mac[4];
335	paddrm = ((u16)mac[3] << 8) | mac[2];
336	paddrl = ((u16)mac[1] << 8) | mac[0];
337
338	W16(ep, fen_paddrh, paddrh);
339	W16(ep, fen_paddrm, paddrm);
340	W16(ep, fen_paddrl, paddrl);
341
342	W16(ep, fen_taddrh, 0);
343	W16(ep, fen_taddrm, 0);
344	W16(ep, fen_taddrl, 0);
345
346	W16(ep, fen_maxd1, 1520);	/* maximum DMA1 length */
347	W16(ep, fen_maxd2, 1520);	/* maximum DMA2 length */
348
349	/* Clear stat counters, in case we ever enable RMON */
350	W32(ep, fen_octc, 0);
351	W32(ep, fen_colc, 0);
352	W32(ep, fen_broc, 0);
353	W32(ep, fen_mulc, 0);
354	W32(ep, fen_uspc, 0);
355	W32(ep, fen_frgc, 0);
356	W32(ep, fen_ospc, 0);
357	W32(ep, fen_jbrc, 0);
358	W32(ep, fen_p64c, 0);
359	W32(ep, fen_p65c, 0);
360	W32(ep, fen_p128c, 0);
361	W32(ep, fen_p256c, 0);
362	W32(ep, fen_p512c, 0);
363	W32(ep, fen_p1024c, 0);
364
365	W16(ep, fen_rfthr, 0);	/* Suggested by manual */
366	W16(ep, fen_rfcnt, 0);
367	W16(ep, fen_cftype, 0);
368
369	fs_init_bds(dev);
370
371	/* adjust to speed (for RMII mode) */
372	if (fpi->use_rmii) {
373		if (fep->phydev->speed == 100)
374			C8(fcccp, fcc_gfemr, 0x20);
375		else
376			S8(fcccp, fcc_gfemr, 0x20);
377	}
378
379	fcc_cr_cmd(fep, CPM_CR_INIT_TRX);
380
381	/* clear events */
382	W16(fccp, fcc_fcce, 0xffff);
383
384	/* Enable interrupts we wish to service */
385	W16(fccp, fcc_fccm, FCC_ENET_TXE | FCC_ENET_RXF | FCC_ENET_TXB);
386
387	/* Set GFMR to enable Ethernet operating mode */
388	W32(fccp, fcc_gfmr, FCC_GFMR_TCI | FCC_GFMR_MODE_ENET);
389
390	/* set sync/delimiters */
391	W16(fccp, fcc_fdsr, 0xd555);
392
393	W32(fccp, fcc_fpsmr, FCC_PSMR_ENCRC);
394
395	if (fpi->use_rmii)
396		S32(fccp, fcc_fpsmr, FCC_PSMR_RMII);
397
398	/* adjust to duplex mode */
399	if (fep->phydev->duplex)
400		S32(fccp, fcc_fpsmr, FCC_PSMR_FDE | FCC_PSMR_LPB);
401	else
402		C32(fccp, fcc_fpsmr, FCC_PSMR_FDE | FCC_PSMR_LPB);
403
404	/* Restore multicast and promiscuous settings */
405	set_multicast_list(dev);
406
407	S32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
408}
409
410static void stop(struct net_device *dev)
411{
412	struct fs_enet_private *fep = netdev_priv(dev);
413	fcc_t __iomem *fccp = fep->fcc.fccp;
414
415	/* stop ethernet */
416	C32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
417
418	/* clear events */
419	W16(fccp, fcc_fcce, 0xffff);
420
421	/* clear interrupt mask */
422	W16(fccp, fcc_fccm, 0);
423
424	fs_cleanup_bds(dev);
425}
426
427static void napi_clear_rx_event(struct net_device *dev)
428{
429	struct fs_enet_private *fep = netdev_priv(dev);
430	fcc_t __iomem *fccp = fep->fcc.fccp;
431
432	W16(fccp, fcc_fcce, FCC_NAPI_RX_EVENT_MSK);
433}
434
435static void napi_enable_rx(struct net_device *dev)
436{
437	struct fs_enet_private *fep = netdev_priv(dev);
438	fcc_t __iomem *fccp = fep->fcc.fccp;
439
440	S16(fccp, fcc_fccm, FCC_NAPI_RX_EVENT_MSK);
441}
442
443static void napi_disable_rx(struct net_device *dev)
444{
445	struct fs_enet_private *fep = netdev_priv(dev);
446	fcc_t __iomem *fccp = fep->fcc.fccp;
447
448	C16(fccp, fcc_fccm, FCC_NAPI_RX_EVENT_MSK);
449}
450
451static void napi_clear_tx_event(struct net_device *dev)
452{
453	struct fs_enet_private *fep = netdev_priv(dev);
454	fcc_t __iomem *fccp = fep->fcc.fccp;
455
456	W16(fccp, fcc_fcce, FCC_NAPI_TX_EVENT_MSK);
457}
458
459static void napi_enable_tx(struct net_device *dev)
460{
461	struct fs_enet_private *fep = netdev_priv(dev);
462	fcc_t __iomem *fccp = fep->fcc.fccp;
463
464	S16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
465}
466
467static void napi_disable_tx(struct net_device *dev)
468{
469	struct fs_enet_private *fep = netdev_priv(dev);
470	fcc_t __iomem *fccp = fep->fcc.fccp;
471
472	C16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
473}
474
475static void rx_bd_done(struct net_device *dev)
476{
477	/* nothing */
478}
479
480static void tx_kickstart(struct net_device *dev)
481{
482	struct fs_enet_private *fep = netdev_priv(dev);
483	fcc_t __iomem *fccp = fep->fcc.fccp;
484
485	S16(fccp, fcc_ftodr, 0x8000);
486}
487
488static u32 get_int_events(struct net_device *dev)
489{
490	struct fs_enet_private *fep = netdev_priv(dev);
491	fcc_t __iomem *fccp = fep->fcc.fccp;
492
493	return (u32)R16(fccp, fcc_fcce);
494}
495
496static void clear_int_events(struct net_device *dev, u32 int_events)
497{
498	struct fs_enet_private *fep = netdev_priv(dev);
499	fcc_t __iomem *fccp = fep->fcc.fccp;
500
501	W16(fccp, fcc_fcce, int_events & 0xffff);
502}
503
504static void ev_error(struct net_device *dev, u32 int_events)
505{
506	struct fs_enet_private *fep = netdev_priv(dev);
507
508	dev_warn(fep->dev, "FS_ENET ERROR(s) 0x%x\n", int_events);
509}
510
511static int get_regs(struct net_device *dev, void *p, int *sizep)
512{
513	struct fs_enet_private *fep = netdev_priv(dev);
514
515	if (*sizep < sizeof(fcc_t) + sizeof(fcc_enet_t) + 1)
516		return -EINVAL;
517
518	memcpy_fromio(p, fep->fcc.fccp, sizeof(fcc_t));
519	p = (char *)p + sizeof(fcc_t);
520
521	memcpy_fromio(p, fep->fcc.ep, sizeof(fcc_enet_t));
522	p = (char *)p + sizeof(fcc_enet_t);
523
524	memcpy_fromio(p, fep->fcc.fcccp, 1);
525	return 0;
526}
527
528static int get_regs_len(struct net_device *dev)
529{
530	return sizeof(fcc_t) + sizeof(fcc_enet_t) + 1;
531}
532
533/* Some transmit errors cause the transmitter to shut
534 * down.  We now issue a restart transmit.
535 * Also, to workaround 8260 device erratum CPM37, we must
536 * disable and then re-enable the transmitterfollowing a
537 * Late Collision, Underrun, or Retry Limit error.
538 * In addition, tbptr may point beyond BDs beyond still marked
539 * as ready due to internal pipelining, so we need to look back
540 * through the BDs and adjust tbptr to point to the last BD
541 * marked as ready.  This may result in some buffers being
542 * retransmitted.
543 */
544static void tx_restart(struct net_device *dev)
545{
546	struct fs_enet_private *fep = netdev_priv(dev);
547	fcc_t __iomem *fccp = fep->fcc.fccp;
548	const struct fs_platform_info *fpi = fep->fpi;
549	fcc_enet_t __iomem *ep = fep->fcc.ep;
550	cbd_t __iomem *curr_tbptr;
551	cbd_t __iomem *recheck_bd;
552	cbd_t __iomem *prev_bd;
553	cbd_t __iomem *last_tx_bd;
554
555	last_tx_bd = fep->tx_bd_base + ((fpi->tx_ring - 1) * sizeof(cbd_t));
556
557	/* get the current bd held in TBPTR  and scan back from this point */
558	recheck_bd = curr_tbptr = (cbd_t __iomem *)
559		((R32(ep, fen_genfcc.fcc_tbptr) - fep->ring_mem_addr) +
560		fep->ring_base);
561
562	prev_bd = (recheck_bd == fep->tx_bd_base) ? last_tx_bd : recheck_bd - 1;
563
564	/* Move through the bds in reverse, look for the earliest buffer
565	 * that is not ready.  Adjust TBPTR to the following buffer */
566	while ((CBDR_SC(prev_bd) & BD_ENET_TX_READY) != 0) {
567		/* Go back one buffer */
568		recheck_bd = prev_bd;
569
570		/* update the previous buffer */
571		prev_bd = (prev_bd == fep->tx_bd_base) ? last_tx_bd : prev_bd - 1;
572
573		/* We should never see all bds marked as ready, check anyway */
574		if (recheck_bd == curr_tbptr)
575			break;
576	}
577	/* Now update the TBPTR and dirty flag to the current buffer */
578	W32(ep, fen_genfcc.fcc_tbptr,
579		(uint) (((void *)recheck_bd - fep->ring_base) +
580		fep->ring_mem_addr));
581	fep->dirty_tx = recheck_bd;
582
583	C32(fccp, fcc_gfmr, FCC_GFMR_ENT);
584	udelay(10);
585	S32(fccp, fcc_gfmr, FCC_GFMR_ENT);
586
587	fcc_cr_cmd(fep, CPM_CR_RESTART_TX);
588}
589
590/*************************************************************************/
591
592const struct fs_ops fs_fcc_ops = {
593	.setup_data		= setup_data,
594	.cleanup_data		= cleanup_data,
595	.set_multicast_list	= set_multicast_list,
596	.restart		= restart,
597	.stop			= stop,
598	.napi_clear_rx_event	= napi_clear_rx_event,
599	.napi_enable_rx		= napi_enable_rx,
600	.napi_disable_rx	= napi_disable_rx,
601	.napi_clear_tx_event	= napi_clear_tx_event,
602	.napi_enable_tx		= napi_enable_tx,
603	.napi_disable_tx	= napi_disable_tx,
604	.rx_bd_done		= rx_bd_done,
605	.tx_kickstart		= tx_kickstart,
606	.get_int_events		= get_int_events,
607	.clear_int_events	= clear_int_events,
608	.ev_error		= ev_error,
609	.get_regs		= get_regs,
610	.get_regs_len		= get_regs_len,
611	.tx_restart		= tx_restart,
612	.allocate_bd		= allocate_bd,
613	.free_bd		= free_bd,
614};
615