This source file includes following definitions.
- dchan2dev
- chan2dev
- vd2dw_edma_desc
- dw_edma_alloc_burst
- dw_edma_alloc_chunk
- dw_edma_alloc_desc
- dw_edma_free_burst
- dw_edma_free_chunk
- dw_edma_free_desc
- vchan_free_desc
- dw_edma_start_transfer
- dw_edma_device_config
- dw_edma_device_pause
- dw_edma_device_resume
- dw_edma_device_terminate_all
- dw_edma_device_issue_pending
- dw_edma_device_tx_status
- dw_edma_device_transfer
- dw_edma_device_prep_slave_sg
- dw_edma_device_prep_dma_cyclic
- dw_edma_done_interrupt
- dw_edma_abort_interrupt
- dw_edma_interrupt
- dw_edma_interrupt_write
- dw_edma_interrupt_read
- dw_edma_interrupt_common
- dw_edma_alloc_chan_resources
- dw_edma_free_chan_resources
- dw_edma_channel_setup
- dw_edma_dec_irq_alloc
- dw_edma_add_irq_mask
- dw_edma_irq_request
- dw_edma_probe
- dw_edma_remove
1
2
3
4
5
6
7
8
9 #include <linux/module.h>
10 #include <linux/device.h>
11 #include <linux/kernel.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/dmaengine.h>
14 #include <linux/err.h>
15 #include <linux/interrupt.h>
16 #include <linux/dma/edma.h>
17 #include <linux/pci.h>
18
19 #include "dw-edma-core.h"
20 #include "dw-edma-v0-core.h"
21 #include "../dmaengine.h"
22 #include "../virt-dma.h"
23
24 static inline
25 struct device *dchan2dev(struct dma_chan *dchan)
26 {
27 return &dchan->dev->device;
28 }
29
30 static inline
31 struct device *chan2dev(struct dw_edma_chan *chan)
32 {
33 return &chan->vc.chan.dev->device;
34 }
35
36 static inline
37 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
38 {
39 return container_of(vd, struct dw_edma_desc, vd);
40 }
41
42 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
43 {
44 struct dw_edma_burst *burst;
45
46 burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
47 if (unlikely(!burst))
48 return NULL;
49
50 INIT_LIST_HEAD(&burst->list);
51 if (chunk->burst) {
52
53 chunk->bursts_alloc++;
54 list_add_tail(&burst->list, &chunk->burst->list);
55 } else {
56
57 chunk->bursts_alloc = 0;
58 chunk->burst = burst;
59 }
60
61 return burst;
62 }
63
64 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
65 {
66 struct dw_edma_chan *chan = desc->chan;
67 struct dw_edma *dw = chan->chip->dw;
68 struct dw_edma_chunk *chunk;
69
70 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
71 if (unlikely(!chunk))
72 return NULL;
73
74 INIT_LIST_HEAD(&chunk->list);
75 chunk->chan = chan;
76
77
78
79
80
81
82 chunk->cb = !(desc->chunks_alloc % 2);
83 chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off;
84 chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off;
85
86 if (desc->chunk) {
87
88 desc->chunks_alloc++;
89 list_add_tail(&chunk->list, &desc->chunk->list);
90 if (!dw_edma_alloc_burst(chunk)) {
91 kfree(chunk);
92 return NULL;
93 }
94 } else {
95
96 chunk->burst = NULL;
97 desc->chunks_alloc = 0;
98 desc->chunk = chunk;
99 }
100
101 return chunk;
102 }
103
104 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
105 {
106 struct dw_edma_desc *desc;
107
108 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
109 if (unlikely(!desc))
110 return NULL;
111
112 desc->chan = chan;
113 if (!dw_edma_alloc_chunk(desc)) {
114 kfree(desc);
115 return NULL;
116 }
117
118 return desc;
119 }
120
121 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
122 {
123 struct dw_edma_burst *child, *_next;
124
125
126 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
127 list_del(&child->list);
128 kfree(child);
129 chunk->bursts_alloc--;
130 }
131
132
133 kfree(child);
134 chunk->burst = NULL;
135 }
136
137 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
138 {
139 struct dw_edma_chunk *child, *_next;
140
141 if (!desc->chunk)
142 return;
143
144
145 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
146 dw_edma_free_burst(child);
147 list_del(&child->list);
148 kfree(child);
149 desc->chunks_alloc--;
150 }
151
152
153 kfree(child);
154 desc->chunk = NULL;
155 }
156
157 static void dw_edma_free_desc(struct dw_edma_desc *desc)
158 {
159 dw_edma_free_chunk(desc);
160 kfree(desc);
161 }
162
163 static void vchan_free_desc(struct virt_dma_desc *vdesc)
164 {
165 dw_edma_free_desc(vd2dw_edma_desc(vdesc));
166 }
167
168 static void dw_edma_start_transfer(struct dw_edma_chan *chan)
169 {
170 struct dw_edma_chunk *child;
171 struct dw_edma_desc *desc;
172 struct virt_dma_desc *vd;
173
174 vd = vchan_next_desc(&chan->vc);
175 if (!vd)
176 return;
177
178 desc = vd2dw_edma_desc(vd);
179 if (!desc)
180 return;
181
182 child = list_first_entry_or_null(&desc->chunk->list,
183 struct dw_edma_chunk, list);
184 if (!child)
185 return;
186
187 dw_edma_v0_core_start(child, !desc->xfer_sz);
188 desc->xfer_sz += child->ll_region.sz;
189 dw_edma_free_burst(child);
190 list_del(&child->list);
191 kfree(child);
192 desc->chunks_alloc--;
193 }
194
195 static int dw_edma_device_config(struct dma_chan *dchan,
196 struct dma_slave_config *config)
197 {
198 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
199
200 memcpy(&chan->config, config, sizeof(*config));
201 chan->configured = true;
202
203 return 0;
204 }
205
206 static int dw_edma_device_pause(struct dma_chan *dchan)
207 {
208 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
209 int err = 0;
210
211 if (!chan->configured)
212 err = -EPERM;
213 else if (chan->status != EDMA_ST_BUSY)
214 err = -EPERM;
215 else if (chan->request != EDMA_REQ_NONE)
216 err = -EPERM;
217 else
218 chan->request = EDMA_REQ_PAUSE;
219
220 return err;
221 }
222
223 static int dw_edma_device_resume(struct dma_chan *dchan)
224 {
225 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
226 int err = 0;
227
228 if (!chan->configured) {
229 err = -EPERM;
230 } else if (chan->status != EDMA_ST_PAUSE) {
231 err = -EPERM;
232 } else if (chan->request != EDMA_REQ_NONE) {
233 err = -EPERM;
234 } else {
235 chan->status = EDMA_ST_BUSY;
236 dw_edma_start_transfer(chan);
237 }
238
239 return err;
240 }
241
242 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
243 {
244 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
245 int err = 0;
246 LIST_HEAD(head);
247
248 if (!chan->configured) {
249
250 } else if (chan->status == EDMA_ST_PAUSE) {
251 chan->status = EDMA_ST_IDLE;
252 chan->configured = false;
253 } else if (chan->status == EDMA_ST_IDLE) {
254 chan->configured = false;
255 } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) {
256
257
258
259
260 chan->status = EDMA_ST_IDLE;
261 chan->configured = false;
262 } else if (chan->request > EDMA_REQ_PAUSE) {
263 err = -EPERM;
264 } else {
265 chan->request = EDMA_REQ_STOP;
266 }
267
268 return err;
269 }
270
271 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
272 {
273 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
274 unsigned long flags;
275
276 spin_lock_irqsave(&chan->vc.lock, flags);
277 if (chan->configured && chan->request == EDMA_REQ_NONE &&
278 chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) {
279 chan->status = EDMA_ST_BUSY;
280 dw_edma_start_transfer(chan);
281 }
282 spin_unlock_irqrestore(&chan->vc.lock, flags);
283 }
284
285 static enum dma_status
286 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
287 struct dma_tx_state *txstate)
288 {
289 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
290 struct dw_edma_desc *desc;
291 struct virt_dma_desc *vd;
292 unsigned long flags;
293 enum dma_status ret;
294 u32 residue = 0;
295
296 ret = dma_cookie_status(dchan, cookie, txstate);
297 if (ret == DMA_COMPLETE)
298 return ret;
299
300 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
301 ret = DMA_PAUSED;
302
303 if (!txstate)
304 goto ret_residue;
305
306 spin_lock_irqsave(&chan->vc.lock, flags);
307 vd = vchan_find_desc(&chan->vc, cookie);
308 if (vd) {
309 desc = vd2dw_edma_desc(vd);
310 if (desc)
311 residue = desc->alloc_sz - desc->xfer_sz;
312 }
313 spin_unlock_irqrestore(&chan->vc.lock, flags);
314
315 ret_residue:
316 dma_set_residue(txstate, residue);
317
318 return ret;
319 }
320
321 static struct dma_async_tx_descriptor *
322 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
323 {
324 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
325 enum dma_transfer_direction direction = xfer->direction;
326 phys_addr_t src_addr, dst_addr;
327 struct scatterlist *sg = NULL;
328 struct dw_edma_chunk *chunk;
329 struct dw_edma_burst *burst;
330 struct dw_edma_desc *desc;
331 u32 cnt;
332 int i;
333
334 if ((direction == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) ||
335 (direction == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ))
336 return NULL;
337
338 if (xfer->cyclic) {
339 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
340 return NULL;
341 } else {
342 if (xfer->xfer.sg.len < 1)
343 return NULL;
344 }
345
346 if (!chan->configured)
347 return NULL;
348
349 desc = dw_edma_alloc_desc(chan);
350 if (unlikely(!desc))
351 goto err_alloc;
352
353 chunk = dw_edma_alloc_chunk(desc);
354 if (unlikely(!chunk))
355 goto err_alloc;
356
357 src_addr = chan->config.src_addr;
358 dst_addr = chan->config.dst_addr;
359
360 if (xfer->cyclic) {
361 cnt = xfer->xfer.cyclic.cnt;
362 } else {
363 cnt = xfer->xfer.sg.len;
364 sg = xfer->xfer.sg.sgl;
365 }
366
367 for (i = 0; i < cnt; i++) {
368 if (!xfer->cyclic && !sg)
369 break;
370
371 if (chunk->bursts_alloc == chan->ll_max) {
372 chunk = dw_edma_alloc_chunk(desc);
373 if (unlikely(!chunk))
374 goto err_alloc;
375 }
376
377 burst = dw_edma_alloc_burst(chunk);
378 if (unlikely(!burst))
379 goto err_alloc;
380
381 if (xfer->cyclic)
382 burst->sz = xfer->xfer.cyclic.len;
383 else
384 burst->sz = sg_dma_len(sg);
385
386 chunk->ll_region.sz += burst->sz;
387 desc->alloc_sz += burst->sz;
388
389 if (direction == DMA_DEV_TO_MEM) {
390 burst->sar = src_addr;
391 if (xfer->cyclic) {
392 burst->dar = xfer->xfer.cyclic.paddr;
393 } else {
394 burst->dar = sg_dma_address(sg);
395
396
397
398
399
400
401
402 src_addr += sg_dma_len(sg);
403 }
404 } else {
405 burst->dar = dst_addr;
406 if (xfer->cyclic) {
407 burst->sar = xfer->xfer.cyclic.paddr;
408 } else {
409 burst->sar = sg_dma_address(sg);
410
411
412
413
414
415
416
417 dst_addr += sg_dma_len(sg);
418 }
419 }
420
421 if (!xfer->cyclic)
422 sg = sg_next(sg);
423 }
424
425 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
426
427 err_alloc:
428 if (desc)
429 dw_edma_free_desc(desc);
430
431 return NULL;
432 }
433
434 static struct dma_async_tx_descriptor *
435 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
436 unsigned int len,
437 enum dma_transfer_direction direction,
438 unsigned long flags, void *context)
439 {
440 struct dw_edma_transfer xfer;
441
442 xfer.dchan = dchan;
443 xfer.direction = direction;
444 xfer.xfer.sg.sgl = sgl;
445 xfer.xfer.sg.len = len;
446 xfer.flags = flags;
447 xfer.cyclic = false;
448
449 return dw_edma_device_transfer(&xfer);
450 }
451
452 static struct dma_async_tx_descriptor *
453 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
454 size_t len, size_t count,
455 enum dma_transfer_direction direction,
456 unsigned long flags)
457 {
458 struct dw_edma_transfer xfer;
459
460 xfer.dchan = dchan;
461 xfer.direction = direction;
462 xfer.xfer.cyclic.paddr = paddr;
463 xfer.xfer.cyclic.len = len;
464 xfer.xfer.cyclic.cnt = count;
465 xfer.flags = flags;
466 xfer.cyclic = true;
467
468 return dw_edma_device_transfer(&xfer);
469 }
470
471 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
472 {
473 struct dw_edma_desc *desc;
474 struct virt_dma_desc *vd;
475 unsigned long flags;
476
477 dw_edma_v0_core_clear_done_int(chan);
478
479 spin_lock_irqsave(&chan->vc.lock, flags);
480 vd = vchan_next_desc(&chan->vc);
481 if (vd) {
482 switch (chan->request) {
483 case EDMA_REQ_NONE:
484 desc = vd2dw_edma_desc(vd);
485 if (desc->chunks_alloc) {
486 chan->status = EDMA_ST_BUSY;
487 dw_edma_start_transfer(chan);
488 } else {
489 list_del(&vd->node);
490 vchan_cookie_complete(vd);
491 chan->status = EDMA_ST_IDLE;
492 }
493 break;
494
495 case EDMA_REQ_STOP:
496 list_del(&vd->node);
497 vchan_cookie_complete(vd);
498 chan->request = EDMA_REQ_NONE;
499 chan->status = EDMA_ST_IDLE;
500 break;
501
502 case EDMA_REQ_PAUSE:
503 chan->request = EDMA_REQ_NONE;
504 chan->status = EDMA_ST_PAUSE;
505 break;
506
507 default:
508 break;
509 }
510 }
511 spin_unlock_irqrestore(&chan->vc.lock, flags);
512 }
513
514 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
515 {
516 struct virt_dma_desc *vd;
517 unsigned long flags;
518
519 dw_edma_v0_core_clear_abort_int(chan);
520
521 spin_lock_irqsave(&chan->vc.lock, flags);
522 vd = vchan_next_desc(&chan->vc);
523 if (vd) {
524 list_del(&vd->node);
525 vchan_cookie_complete(vd);
526 }
527 spin_unlock_irqrestore(&chan->vc.lock, flags);
528 chan->request = EDMA_REQ_NONE;
529 chan->status = EDMA_ST_IDLE;
530 }
531
532 static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write)
533 {
534 struct dw_edma_irq *dw_irq = data;
535 struct dw_edma *dw = dw_irq->dw;
536 unsigned long total, pos, val;
537 unsigned long off;
538 u32 mask;
539
540 if (write) {
541 total = dw->wr_ch_cnt;
542 off = 0;
543 mask = dw_irq->wr_mask;
544 } else {
545 total = dw->rd_ch_cnt;
546 off = dw->wr_ch_cnt;
547 mask = dw_irq->rd_mask;
548 }
549
550 val = dw_edma_v0_core_status_done_int(dw, write ?
551 EDMA_DIR_WRITE :
552 EDMA_DIR_READ);
553 val &= mask;
554 for_each_set_bit(pos, &val, total) {
555 struct dw_edma_chan *chan = &dw->chan[pos + off];
556
557 dw_edma_done_interrupt(chan);
558 }
559
560 val = dw_edma_v0_core_status_abort_int(dw, write ?
561 EDMA_DIR_WRITE :
562 EDMA_DIR_READ);
563 val &= mask;
564 for_each_set_bit(pos, &val, total) {
565 struct dw_edma_chan *chan = &dw->chan[pos + off];
566
567 dw_edma_abort_interrupt(chan);
568 }
569
570 return IRQ_HANDLED;
571 }
572
573 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
574 {
575 return dw_edma_interrupt(irq, data, true);
576 }
577
578 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
579 {
580 return dw_edma_interrupt(irq, data, false);
581 }
582
583 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
584 {
585 dw_edma_interrupt(irq, data, true);
586 dw_edma_interrupt(irq, data, false);
587
588 return IRQ_HANDLED;
589 }
590
591 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
592 {
593 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
594
595 if (chan->status != EDMA_ST_IDLE)
596 return -EBUSY;
597
598 pm_runtime_get(chan->chip->dev);
599
600 return 0;
601 }
602
603 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
604 {
605 unsigned long timeout = jiffies + msecs_to_jiffies(5000);
606 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
607 int ret;
608
609 while (time_before(jiffies, timeout)) {
610 ret = dw_edma_device_terminate_all(dchan);
611 if (!ret)
612 break;
613
614 if (time_after_eq(jiffies, timeout))
615 return;
616
617 cpu_relax();
618 }
619
620 pm_runtime_put(chan->chip->dev);
621 }
622
623 static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
624 u32 wr_alloc, u32 rd_alloc)
625 {
626 struct dw_edma_region *dt_region;
627 struct device *dev = chip->dev;
628 struct dw_edma *dw = chip->dw;
629 struct dw_edma_chan *chan;
630 size_t ll_chunk, dt_chunk;
631 struct dw_edma_irq *irq;
632 struct dma_device *dma;
633 u32 i, j, cnt, ch_cnt;
634 u32 alloc, off_alloc;
635 int err = 0;
636 u32 pos;
637
638 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
639 ll_chunk = dw->ll_region.sz;
640 dt_chunk = dw->dt_region.sz;
641
642
643 ll_chunk /= roundup_pow_of_two(ch_cnt);
644
645
646 dt_chunk /= roundup_pow_of_two(ch_cnt);
647
648 if (write) {
649 i = 0;
650 cnt = dw->wr_ch_cnt;
651 dma = &dw->wr_edma;
652 alloc = wr_alloc;
653 off_alloc = 0;
654 } else {
655 i = dw->wr_ch_cnt;
656 cnt = dw->rd_ch_cnt;
657 dma = &dw->rd_edma;
658 alloc = rd_alloc;
659 off_alloc = wr_alloc;
660 }
661
662 INIT_LIST_HEAD(&dma->channels);
663 for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) {
664 chan = &dw->chan[i];
665
666 dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL);
667 if (!dt_region)
668 return -ENOMEM;
669
670 chan->vc.chan.private = dt_region;
671
672 chan->chip = chip;
673 chan->id = j;
674 chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ;
675 chan->configured = false;
676 chan->request = EDMA_REQ_NONE;
677 chan->status = EDMA_ST_IDLE;
678
679 chan->ll_off = (ll_chunk * i);
680 chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1;
681
682 chan->dt_off = (dt_chunk * i);
683
684 dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n",
685 write ? "write" : "read", j,
686 chan->ll_off, chan->ll_max);
687
688 if (dw->nr_irqs == 1)
689 pos = 0;
690 else
691 pos = off_alloc + (j % alloc);
692
693 irq = &dw->irq[pos];
694
695 if (write)
696 irq->wr_mask |= BIT(j);
697 else
698 irq->rd_mask |= BIT(j);
699
700 irq->dw = dw;
701 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
702
703 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
704 write ? "write" : "read", j,
705 chan->msi.address_hi, chan->msi.address_lo,
706 chan->msi.data);
707
708 chan->vc.desc_free = vchan_free_desc;
709 vchan_init(&chan->vc, dma);
710
711 dt_region->paddr = dw->dt_region.paddr + chan->dt_off;
712 dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off;
713 dt_region->sz = dt_chunk;
714
715 dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n",
716 write ? "write" : "read", j, chan->dt_off);
717
718 dw_edma_v0_core_device_config(chan);
719 }
720
721
722 dma_cap_zero(dma->cap_mask);
723 dma_cap_set(DMA_SLAVE, dma->cap_mask);
724 dma_cap_set(DMA_CYCLIC, dma->cap_mask);
725 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
726 dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
727 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
728 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
729 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
730 dma->chancnt = cnt;
731
732
733 dma->dev = chip->dev;
734 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
735 dma->device_free_chan_resources = dw_edma_free_chan_resources;
736 dma->device_config = dw_edma_device_config;
737 dma->device_pause = dw_edma_device_pause;
738 dma->device_resume = dw_edma_device_resume;
739 dma->device_terminate_all = dw_edma_device_terminate_all;
740 dma->device_issue_pending = dw_edma_device_issue_pending;
741 dma->device_tx_status = dw_edma_device_tx_status;
742 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
743 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
744
745 dma_set_max_seg_size(dma->dev, U32_MAX);
746
747
748 err = dma_async_device_register(dma);
749
750 return err;
751 }
752
753 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
754 {
755 if (*nr_irqs && *alloc < cnt) {
756 (*alloc)++;
757 (*nr_irqs)--;
758 }
759 }
760
761 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
762 {
763 while (*mask * alloc < cnt)
764 (*mask)++;
765 }
766
767 static int dw_edma_irq_request(struct dw_edma_chip *chip,
768 u32 *wr_alloc, u32 *rd_alloc)
769 {
770 struct device *dev = chip->dev;
771 struct dw_edma *dw = chip->dw;
772 u32 wr_mask = 1;
773 u32 rd_mask = 1;
774 int i, err = 0;
775 u32 ch_cnt;
776
777 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
778
779 if (dw->nr_irqs < 1)
780 return -EINVAL;
781
782 if (dw->nr_irqs == 1) {
783
784 err = request_irq(pci_irq_vector(to_pci_dev(dev), 0),
785 dw_edma_interrupt_common,
786 IRQF_SHARED, dw->name, &dw->irq[0]);
787 if (err) {
788 dw->nr_irqs = 0;
789 return err;
790 }
791
792 get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), 0),
793 &dw->irq[0].msi);
794 } else {
795
796 int tmp = dw->nr_irqs;
797
798 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
799 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
800 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
801 }
802
803 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
804 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
805
806 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
807 err = request_irq(pci_irq_vector(to_pci_dev(dev), i),
808 i < *wr_alloc ?
809 dw_edma_interrupt_write :
810 dw_edma_interrupt_read,
811 IRQF_SHARED, dw->name,
812 &dw->irq[i]);
813 if (err) {
814 dw->nr_irqs = i;
815 return err;
816 }
817
818 get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), i),
819 &dw->irq[i].msi);
820 }
821
822 dw->nr_irqs = i;
823 }
824
825 return err;
826 }
827
828 int dw_edma_probe(struct dw_edma_chip *chip)
829 {
830 struct device *dev = chip->dev;
831 struct dw_edma *dw = chip->dw;
832 u32 wr_alloc = 0;
833 u32 rd_alloc = 0;
834 int i, err;
835
836 raw_spin_lock_init(&dw->lock);
837
838
839 dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE);
840 if (!dw->wr_ch_cnt)
841 return -EINVAL;
842
843
844 dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ);
845 if (!dw->rd_ch_cnt)
846 return -EINVAL;
847
848 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
849 dw->wr_ch_cnt, dw->rd_ch_cnt);
850
851
852 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
853 sizeof(*dw->chan), GFP_KERNEL);
854 if (!dw->chan)
855 return -ENOMEM;
856
857 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id);
858
859
860 dw_edma_v0_core_off(dw);
861
862
863 err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc);
864 if (err)
865 return err;
866
867
868 err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc);
869 if (err)
870 goto err_irq_free;
871
872
873 err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc);
874 if (err)
875 goto err_irq_free;
876
877
878 pm_runtime_enable(dev);
879
880
881 dw_edma_v0_core_debugfs_on(chip);
882
883 return 0;
884
885 err_irq_free:
886 for (i = (dw->nr_irqs - 1); i >= 0; i--)
887 free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
888
889 dw->nr_irqs = 0;
890
891 return err;
892 }
893 EXPORT_SYMBOL_GPL(dw_edma_probe);
894
895 int dw_edma_remove(struct dw_edma_chip *chip)
896 {
897 struct dw_edma_chan *chan, *_chan;
898 struct device *dev = chip->dev;
899 struct dw_edma *dw = chip->dw;
900 int i;
901
902
903 dw_edma_v0_core_off(dw);
904
905
906 for (i = (dw->nr_irqs - 1); i >= 0; i--)
907 free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
908
909
910 pm_runtime_disable(dev);
911
912 list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
913 vc.chan.device_node) {
914 list_del(&chan->vc.chan.device_node);
915 tasklet_kill(&chan->vc.task);
916 }
917
918 list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
919 vc.chan.device_node) {
920 list_del(&chan->vc.chan.device_node);
921 tasklet_kill(&chan->vc.task);
922 }
923
924
925 dma_async_device_unregister(&dw->wr_edma);
926 dma_async_device_unregister(&dw->rd_edma);
927
928
929 dw_edma_v0_core_debugfs_off();
930
931 return 0;
932 }
933 EXPORT_SYMBOL_GPL(dw_edma_remove);
934
935 MODULE_LICENSE("GPL v2");
936 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
937 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");