This source file includes following definitions.
- pchan_update
- pchan_writel
- pchan_readl
- dma_update
- dma_writel
- dma_readl
- to_owl_dma
- chan2dev
- to_owl_vchan
- to_owl_txd
- llc_hw_ctrla
- llc_hw_ctrlb
- owl_dma_free_lli
- owl_dma_alloc_lli
- owl_dma_add_lli
- owl_dma_cfg_lli
- owl_dma_get_pchan
- owl_dma_pchan_busy
- owl_dma_terminate_pchan
- owl_dma_pause_pchan
- owl_dma_resume_pchan
- owl_dma_start_next_txd
- owl_dma_phy_free
- owl_dma_interrupt
- owl_dma_free_txd
- owl_dma_desc_free
- owl_dma_terminate_all
- owl_dma_config
- owl_dma_pause
- owl_dma_resume
- owl_dma_getbytes_chan
- owl_dma_tx_status
- owl_dma_phy_alloc_and_start
- owl_dma_issue_pending
- owl_dma_prep_memcpy
- owl_dma_prep_slave_sg
- owl_prep_dma_cyclic
- owl_dma_free_chan_resources
- owl_dma_free
- owl_dma_of_xlate
- owl_dma_probe
- owl_dma_remove
- owl_dma_init
- owl_dma_exit
1
2
3
4
5
6
7
8
9
10
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/io.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/of_device.h>
24 #include <linux/of_dma.h>
25 #include <linux/slab.h>
26 #include "virt-dma.h"
27
28 #define OWL_DMA_FRAME_MAX_LENGTH 0xfffff
29
30
31 #define OWL_DMA_IRQ_PD0 0x00
32 #define OWL_DMA_IRQ_PD1 0x04
33 #define OWL_DMA_IRQ_PD2 0x08
34 #define OWL_DMA_IRQ_PD3 0x0C
35 #define OWL_DMA_IRQ_EN0 0x10
36 #define OWL_DMA_IRQ_EN1 0x14
37 #define OWL_DMA_IRQ_EN2 0x18
38 #define OWL_DMA_IRQ_EN3 0x1C
39 #define OWL_DMA_SECURE_ACCESS_CTL 0x20
40 #define OWL_DMA_NIC_QOS 0x24
41 #define OWL_DMA_DBGSEL 0x28
42 #define OWL_DMA_IDLE_STAT 0x2C
43
44
45 #define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100)
46 #define OWL_DMAX_MODE 0x00
47 #define OWL_DMAX_SOURCE 0x04
48 #define OWL_DMAX_DESTINATION 0x08
49 #define OWL_DMAX_FRAME_LEN 0x0C
50 #define OWL_DMAX_FRAME_CNT 0x10
51 #define OWL_DMAX_REMAIN_FRAME_CNT 0x14
52 #define OWL_DMAX_REMAIN_CNT 0x18
53 #define OWL_DMAX_SOURCE_STRIDE 0x1C
54 #define OWL_DMAX_DESTINATION_STRIDE 0x20
55 #define OWL_DMAX_START 0x24
56 #define OWL_DMAX_PAUSE 0x28
57 #define OWL_DMAX_CHAINED_CTL 0x2C
58 #define OWL_DMAX_CONSTANT 0x30
59 #define OWL_DMAX_LINKLIST_CTL 0x34
60 #define OWL_DMAX_NEXT_DESCRIPTOR 0x38
61 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C
62 #define OWL_DMAX_INT_CTL 0x40
63 #define OWL_DMAX_INT_STATUS 0x44
64 #define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48
65 #define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C
66
67
68 #define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0)
69 #define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8)
70 #define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0)
71 #define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2)
72 #define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3)
73 #define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10)
74 #define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0)
75 #define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2)
76 #define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3)
77 #define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16)
78 #define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0)
79 #define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1)
80 #define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2)
81 #define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18)
82 #define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0)
83 #define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1)
84 #define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2)
85 #define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20)
86 #define OWL_DMA_MODE_CB BIT(23)
87 #define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28)
88 #define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0)
89 #define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1)
90 #define OWL_DMA_MODE_CFE BIT(29)
91 #define OWL_DMA_MODE_LME BIT(30)
92 #define OWL_DMA_MODE_CME BIT(31)
93
94
95 #define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8)
96 #define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0)
97 #define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1)
98 #define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2)
99 #define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10)
100 #define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0)
101 #define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1)
102 #define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2)
103 #define OWL_DMA_LLC_SUSPEND BIT(16)
104
105
106 #define OWL_DMA_INTCTL_BLOCK BIT(0)
107 #define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1)
108 #define OWL_DMA_INTCTL_FRAME BIT(2)
109 #define OWL_DMA_INTCTL_HALF_FRAME BIT(3)
110 #define OWL_DMA_INTCTL_LAST_FRAME BIT(4)
111
112
113 #define OWL_DMA_INTSTAT_BLOCK BIT(0)
114 #define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1)
115 #define OWL_DMA_INTSTAT_FRAME BIT(2)
116 #define OWL_DMA_INTSTAT_HALF_FRAME BIT(3)
117 #define OWL_DMA_INTSTAT_LAST_FRAME BIT(4)
118
119
120 #define BIT_FIELD(val, width, shift, newshift) \
121 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136 struct owl_dma_lli_hw {
137 u32 next_lli;
138 u32 saddr;
139 u32 daddr;
140 u32 flen:20;
141 u32 fcnt:12;
142 u32 src_stride;
143 u32 dst_stride;
144 u32 ctrla;
145 u32 ctrlb;
146 u32 const_num;
147 };
148
149
150
151
152
153
154
155 struct owl_dma_lli {
156 struct owl_dma_lli_hw hw;
157 dma_addr_t phys;
158 struct list_head node;
159 };
160
161
162
163
164
165
166
167 struct owl_dma_txd {
168 struct virt_dma_desc vd;
169 struct list_head lli_list;
170 bool cyclic;
171 };
172
173
174
175
176
177
178
179 struct owl_dma_pchan {
180 u32 id;
181 void __iomem *base;
182 struct owl_dma_vchan *vchan;
183 };
184
185
186
187
188
189
190
191
192
193 struct owl_dma_vchan {
194 struct virt_dma_chan vc;
195 struct owl_dma_pchan *pchan;
196 struct owl_dma_txd *txd;
197 struct dma_slave_config cfg;
198 u8 drq;
199 };
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214 struct owl_dma {
215 struct dma_device dma;
216 void __iomem *base;
217 struct clk *clk;
218 spinlock_t lock;
219 struct dma_pool *lli_pool;
220 int irq;
221
222 unsigned int nr_pchans;
223 struct owl_dma_pchan *pchans;
224
225 unsigned int nr_vchans;
226 struct owl_dma_vchan *vchans;
227 };
228
229 static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
230 u32 val, bool state)
231 {
232 u32 regval;
233
234 regval = readl(pchan->base + reg);
235
236 if (state)
237 regval |= val;
238 else
239 regval &= ~val;
240
241 writel(val, pchan->base + reg);
242 }
243
244 static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
245 {
246 writel(data, pchan->base + reg);
247 }
248
249 static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
250 {
251 return readl(pchan->base + reg);
252 }
253
254 static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
255 {
256 u32 regval;
257
258 regval = readl(od->base + reg);
259
260 if (state)
261 regval |= val;
262 else
263 regval &= ~val;
264
265 writel(val, od->base + reg);
266 }
267
268 static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
269 {
270 writel(data, od->base + reg);
271 }
272
273 static u32 dma_readl(struct owl_dma *od, u32 reg)
274 {
275 return readl(od->base + reg);
276 }
277
278 static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
279 {
280 return container_of(dd, struct owl_dma, dma);
281 }
282
283 static struct device *chan2dev(struct dma_chan *chan)
284 {
285 return &chan->dev->device;
286 }
287
288 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
289 {
290 return container_of(chan, struct owl_dma_vchan, vc.chan);
291 }
292
293 static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
294 {
295 return container_of(tx, struct owl_dma_txd, vd.tx);
296 }
297
298 static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
299 {
300 u32 ctl;
301
302 ctl = BIT_FIELD(mode, 4, 28, 28) |
303 BIT_FIELD(mode, 8, 16, 20) |
304 BIT_FIELD(mode, 4, 8, 16) |
305 BIT_FIELD(mode, 6, 0, 10) |
306 BIT_FIELD(llc_ctl, 2, 10, 8) |
307 BIT_FIELD(llc_ctl, 2, 8, 6);
308
309 return ctl;
310 }
311
312 static inline u32 llc_hw_ctrlb(u32 int_ctl)
313 {
314 u32 ctl;
315
316 ctl = BIT_FIELD(int_ctl, 7, 0, 18);
317
318 return ctl;
319 }
320
321 static void owl_dma_free_lli(struct owl_dma *od,
322 struct owl_dma_lli *lli)
323 {
324 list_del(&lli->node);
325 dma_pool_free(od->lli_pool, lli, lli->phys);
326 }
327
328 static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
329 {
330 struct owl_dma_lli *lli;
331 dma_addr_t phys;
332
333 lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
334 if (!lli)
335 return NULL;
336
337 INIT_LIST_HEAD(&lli->node);
338 lli->phys = phys;
339
340 return lli;
341 }
342
343 static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
344 struct owl_dma_lli *prev,
345 struct owl_dma_lli *next,
346 bool is_cyclic)
347 {
348 if (!is_cyclic)
349 list_add_tail(&next->node, &txd->lli_list);
350
351 if (prev) {
352 prev->hw.next_lli = next->phys;
353 prev->hw.ctrla |= llc_hw_ctrla(OWL_DMA_MODE_LME, 0);
354 }
355
356 return next;
357 }
358
359 static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
360 struct owl_dma_lli *lli,
361 dma_addr_t src, dma_addr_t dst,
362 u32 len, enum dma_transfer_direction dir,
363 struct dma_slave_config *sconfig,
364 bool is_cyclic)
365 {
366 struct owl_dma_lli_hw *hw = &lli->hw;
367 u32 mode;
368
369 mode = OWL_DMA_MODE_PW(0);
370
371 switch (dir) {
372 case DMA_MEM_TO_MEM:
373 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
374 OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
375 OWL_DMA_MODE_DAM_INC;
376
377 break;
378 case DMA_MEM_TO_DEV:
379 mode |= OWL_DMA_MODE_TS(vchan->drq)
380 | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
381 | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
382
383
384
385
386
387 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
388 mode |= OWL_DMA_MODE_NDDBW_8BIT;
389
390 break;
391 case DMA_DEV_TO_MEM:
392 mode |= OWL_DMA_MODE_TS(vchan->drq)
393 | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
394 | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
395
396
397
398
399
400 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
401 mode |= OWL_DMA_MODE_NDDBW_8BIT;
402
403 break;
404 default:
405 return -EINVAL;
406 }
407
408 hw->next_lli = 0;
409 hw->saddr = src;
410 hw->daddr = dst;
411
412 hw->fcnt = 1;
413 hw->flen = len;
414 hw->src_stride = 0;
415 hw->dst_stride = 0;
416 hw->ctrla = llc_hw_ctrla(mode,
417 OWL_DMA_LLC_SAV_LOAD_NEXT |
418 OWL_DMA_LLC_DAV_LOAD_NEXT);
419
420 if (is_cyclic)
421 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);
422 else
423 hw->ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
424
425 return 0;
426 }
427
428 static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
429 struct owl_dma_vchan *vchan)
430 {
431 struct owl_dma_pchan *pchan = NULL;
432 unsigned long flags;
433 int i;
434
435 for (i = 0; i < od->nr_pchans; i++) {
436 pchan = &od->pchans[i];
437
438 spin_lock_irqsave(&od->lock, flags);
439 if (!pchan->vchan) {
440 pchan->vchan = vchan;
441 spin_unlock_irqrestore(&od->lock, flags);
442 break;
443 }
444
445 spin_unlock_irqrestore(&od->lock, flags);
446 }
447
448 return pchan;
449 }
450
451 static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
452 {
453 unsigned int val;
454
455 val = dma_readl(od, OWL_DMA_IDLE_STAT);
456
457 return !(val & (1 << pchan->id));
458 }
459
460 static void owl_dma_terminate_pchan(struct owl_dma *od,
461 struct owl_dma_pchan *pchan)
462 {
463 unsigned long flags;
464 u32 irq_pd;
465
466 pchan_writel(pchan, OWL_DMAX_START, 0);
467 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
468
469 spin_lock_irqsave(&od->lock, flags);
470 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
471
472 irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
473 if (irq_pd & (1 << pchan->id)) {
474 dev_warn(od->dma.dev,
475 "terminating pchan %d that still has pending irq\n",
476 pchan->id);
477 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
478 }
479
480 pchan->vchan = NULL;
481
482 spin_unlock_irqrestore(&od->lock, flags);
483 }
484
485 static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
486 {
487 pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
488 }
489
490 static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
491 {
492 pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
493 }
494
495 static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
496 {
497 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
498 struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
499 struct owl_dma_pchan *pchan = vchan->pchan;
500 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
501 struct owl_dma_lli *lli;
502 unsigned long flags;
503 u32 int_ctl;
504
505 list_del(&vd->node);
506
507 vchan->txd = txd;
508
509
510 while (owl_dma_pchan_busy(od, pchan))
511 cpu_relax();
512
513 lli = list_first_entry(&txd->lli_list,
514 struct owl_dma_lli, node);
515
516 if (txd->cyclic)
517 int_ctl = OWL_DMA_INTCTL_BLOCK;
518 else
519 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;
520
521 pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
522 pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
523 OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
524 pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
525 pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
526
527
528 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
529
530 spin_lock_irqsave(&od->lock, flags);
531
532 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
533
534 spin_unlock_irqrestore(&od->lock, flags);
535
536 dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
537
538
539 pchan_writel(pchan, OWL_DMAX_START, 0x1);
540
541 return 0;
542 }
543
544 static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
545 {
546
547 owl_dma_terminate_pchan(od, vchan->pchan);
548
549 vchan->pchan = NULL;
550 }
551
552 static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
553 {
554 struct owl_dma *od = dev_id;
555 struct owl_dma_vchan *vchan;
556 struct owl_dma_pchan *pchan;
557 unsigned long pending;
558 int i;
559 unsigned int global_irq_pending, chan_irq_pending;
560
561 spin_lock(&od->lock);
562
563 pending = dma_readl(od, OWL_DMA_IRQ_PD0);
564
565
566 for_each_set_bit(i, &pending, od->nr_pchans) {
567 pchan = &od->pchans[i];
568 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
569 }
570
571
572 dma_writel(od, OWL_DMA_IRQ_PD0, pending);
573
574
575 for (i = 0; i < od->nr_pchans; i++) {
576 pchan = &od->pchans[i];
577 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
578 pchan_readl(pchan, OWL_DMAX_INT_STATUS);
579
580
581 dma_readl(od, OWL_DMA_IRQ_PD0);
582
583 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);
584
585 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {
586 dev_dbg(od->dma.dev,
587 "global and channel IRQ pending match err\n");
588
589
590 pchan_update(pchan, OWL_DMAX_INT_STATUS,
591 0xff, false);
592
593
594 pending |= BIT(i);
595 }
596 }
597
598 spin_unlock(&od->lock);
599
600 for_each_set_bit(i, &pending, od->nr_pchans) {
601 struct owl_dma_txd *txd;
602
603 pchan = &od->pchans[i];
604
605 vchan = pchan->vchan;
606 if (!vchan) {
607 dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",
608 pchan->id);
609 continue;
610 }
611
612 spin_lock(&vchan->vc.lock);
613
614 txd = vchan->txd;
615 if (txd) {
616 vchan->txd = NULL;
617
618 vchan_cookie_complete(&txd->vd);
619
620
621
622
623
624 if (vchan_next_desc(&vchan->vc))
625 owl_dma_start_next_txd(vchan);
626 else
627 owl_dma_phy_free(od, vchan);
628 }
629
630 spin_unlock(&vchan->vc.lock);
631 }
632
633 return IRQ_HANDLED;
634 }
635
636 static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
637 {
638 struct owl_dma_lli *lli, *_lli;
639
640 if (unlikely(!txd))
641 return;
642
643 list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
644 owl_dma_free_lli(od, lli);
645
646 kfree(txd);
647 }
648
649 static void owl_dma_desc_free(struct virt_dma_desc *vd)
650 {
651 struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
652 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
653
654 owl_dma_free_txd(od, txd);
655 }
656
657 static int owl_dma_terminate_all(struct dma_chan *chan)
658 {
659 struct owl_dma *od = to_owl_dma(chan->device);
660 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
661 unsigned long flags;
662 LIST_HEAD(head);
663
664 spin_lock_irqsave(&vchan->vc.lock, flags);
665
666 if (vchan->pchan)
667 owl_dma_phy_free(od, vchan);
668
669 if (vchan->txd) {
670 owl_dma_desc_free(&vchan->txd->vd);
671 vchan->txd = NULL;
672 }
673
674 vchan_get_all_descriptors(&vchan->vc, &head);
675 vchan_dma_desc_free_list(&vchan->vc, &head);
676
677 spin_unlock_irqrestore(&vchan->vc.lock, flags);
678
679 return 0;
680 }
681
682 static int owl_dma_config(struct dma_chan *chan,
683 struct dma_slave_config *config)
684 {
685 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
686
687
688 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
689 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
690 return -EINVAL;
691
692 memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
693
694 return 0;
695 }
696
697 static int owl_dma_pause(struct dma_chan *chan)
698 {
699 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
700 unsigned long flags;
701
702 spin_lock_irqsave(&vchan->vc.lock, flags);
703
704 owl_dma_pause_pchan(vchan->pchan);
705
706 spin_unlock_irqrestore(&vchan->vc.lock, flags);
707
708 return 0;
709 }
710
711 static int owl_dma_resume(struct dma_chan *chan)
712 {
713 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
714 unsigned long flags;
715
716 if (!vchan->pchan && !vchan->txd)
717 return 0;
718
719 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
720
721 spin_lock_irqsave(&vchan->vc.lock, flags);
722
723 owl_dma_resume_pchan(vchan->pchan);
724
725 spin_unlock_irqrestore(&vchan->vc.lock, flags);
726
727 return 0;
728 }
729
730 static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
731 {
732 struct owl_dma_pchan *pchan;
733 struct owl_dma_txd *txd;
734 struct owl_dma_lli *lli;
735 unsigned int next_lli_phy;
736 size_t bytes;
737
738 pchan = vchan->pchan;
739 txd = vchan->txd;
740
741 if (!pchan || !txd)
742 return 0;
743
744
745 bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
746
747
748 if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
749 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
750 list_for_each_entry(lli, &txd->lli_list, node) {
751
752 if (lli->phys == next_lli_phy) {
753 list_for_each_entry(lli, &txd->lli_list, node)
754 bytes += lli->hw.flen;
755 break;
756 }
757 }
758 }
759
760 return bytes;
761 }
762
763 static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
764 dma_cookie_t cookie,
765 struct dma_tx_state *state)
766 {
767 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
768 struct owl_dma_lli *lli;
769 struct virt_dma_desc *vd;
770 struct owl_dma_txd *txd;
771 enum dma_status ret;
772 unsigned long flags;
773 size_t bytes = 0;
774
775 ret = dma_cookie_status(chan, cookie, state);
776 if (ret == DMA_COMPLETE || !state)
777 return ret;
778
779 spin_lock_irqsave(&vchan->vc.lock, flags);
780
781 vd = vchan_find_desc(&vchan->vc, cookie);
782 if (vd) {
783 txd = to_owl_txd(&vd->tx);
784 list_for_each_entry(lli, &txd->lli_list, node)
785 bytes += lli->hw.flen;
786 } else {
787 bytes = owl_dma_getbytes_chan(vchan);
788 }
789
790 spin_unlock_irqrestore(&vchan->vc.lock, flags);
791
792 dma_set_residue(state, bytes);
793
794 return ret;
795 }
796
797 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
798 {
799 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
800 struct owl_dma_pchan *pchan;
801
802 pchan = owl_dma_get_pchan(od, vchan);
803 if (!pchan)
804 return;
805
806 dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);
807
808 vchan->pchan = pchan;
809 owl_dma_start_next_txd(vchan);
810 }
811
812 static void owl_dma_issue_pending(struct dma_chan *chan)
813 {
814 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
815 unsigned long flags;
816
817 spin_lock_irqsave(&vchan->vc.lock, flags);
818 if (vchan_issue_pending(&vchan->vc)) {
819 if (!vchan->pchan)
820 owl_dma_phy_alloc_and_start(vchan);
821 }
822 spin_unlock_irqrestore(&vchan->vc.lock, flags);
823 }
824
825 static struct dma_async_tx_descriptor
826 *owl_dma_prep_memcpy(struct dma_chan *chan,
827 dma_addr_t dst, dma_addr_t src,
828 size_t len, unsigned long flags)
829 {
830 struct owl_dma *od = to_owl_dma(chan->device);
831 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
832 struct owl_dma_txd *txd;
833 struct owl_dma_lli *lli, *prev = NULL;
834 size_t offset, bytes;
835 int ret;
836
837 if (!len)
838 return NULL;
839
840 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
841 if (!txd)
842 return NULL;
843
844 INIT_LIST_HEAD(&txd->lli_list);
845
846
847 for (offset = 0; offset < len; offset += bytes) {
848 lli = owl_dma_alloc_lli(od);
849 if (!lli) {
850 dev_warn(chan2dev(chan), "failed to allocate lli\n");
851 goto err_txd_free;
852 }
853
854 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
855
856 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,
857 bytes, DMA_MEM_TO_MEM,
858 &vchan->cfg, txd->cyclic);
859 if (ret) {
860 dev_warn(chan2dev(chan), "failed to config lli\n");
861 goto err_txd_free;
862 }
863
864 prev = owl_dma_add_lli(txd, prev, lli, false);
865 }
866
867 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
868
869 err_txd_free:
870 owl_dma_free_txd(od, txd);
871 return NULL;
872 }
873
874 static struct dma_async_tx_descriptor
875 *owl_dma_prep_slave_sg(struct dma_chan *chan,
876 struct scatterlist *sgl,
877 unsigned int sg_len,
878 enum dma_transfer_direction dir,
879 unsigned long flags, void *context)
880 {
881 struct owl_dma *od = to_owl_dma(chan->device);
882 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
883 struct dma_slave_config *sconfig = &vchan->cfg;
884 struct owl_dma_txd *txd;
885 struct owl_dma_lli *lli, *prev = NULL;
886 struct scatterlist *sg;
887 dma_addr_t addr, src = 0, dst = 0;
888 size_t len;
889 int ret, i;
890
891 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
892 if (!txd)
893 return NULL;
894
895 INIT_LIST_HEAD(&txd->lli_list);
896
897 for_each_sg(sgl, sg, sg_len, i) {
898 addr = sg_dma_address(sg);
899 len = sg_dma_len(sg);
900
901 if (len > OWL_DMA_FRAME_MAX_LENGTH) {
902 dev_err(od->dma.dev,
903 "frame length exceeds max supported length");
904 goto err_txd_free;
905 }
906
907 lli = owl_dma_alloc_lli(od);
908 if (!lli) {
909 dev_err(chan2dev(chan), "failed to allocate lli");
910 goto err_txd_free;
911 }
912
913 if (dir == DMA_MEM_TO_DEV) {
914 src = addr;
915 dst = sconfig->dst_addr;
916 } else {
917 src = sconfig->src_addr;
918 dst = addr;
919 }
920
921 ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
922 txd->cyclic);
923 if (ret) {
924 dev_warn(chan2dev(chan), "failed to config lli");
925 goto err_txd_free;
926 }
927
928 prev = owl_dma_add_lli(txd, prev, lli, false);
929 }
930
931 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
932
933 err_txd_free:
934 owl_dma_free_txd(od, txd);
935
936 return NULL;
937 }
938
939 static struct dma_async_tx_descriptor
940 *owl_prep_dma_cyclic(struct dma_chan *chan,
941 dma_addr_t buf_addr, size_t buf_len,
942 size_t period_len,
943 enum dma_transfer_direction dir,
944 unsigned long flags)
945 {
946 struct owl_dma *od = to_owl_dma(chan->device);
947 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
948 struct dma_slave_config *sconfig = &vchan->cfg;
949 struct owl_dma_txd *txd;
950 struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
951 dma_addr_t src = 0, dst = 0;
952 unsigned int periods = buf_len / period_len;
953 int ret, i;
954
955 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
956 if (!txd)
957 return NULL;
958
959 INIT_LIST_HEAD(&txd->lli_list);
960 txd->cyclic = true;
961
962 for (i = 0; i < periods; i++) {
963 lli = owl_dma_alloc_lli(od);
964 if (!lli) {
965 dev_warn(chan2dev(chan), "failed to allocate lli");
966 goto err_txd_free;
967 }
968
969 if (dir == DMA_MEM_TO_DEV) {
970 src = buf_addr + (period_len * i);
971 dst = sconfig->dst_addr;
972 } else if (dir == DMA_DEV_TO_MEM) {
973 src = sconfig->src_addr;
974 dst = buf_addr + (period_len * i);
975 }
976
977 ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
978 dir, sconfig, txd->cyclic);
979 if (ret) {
980 dev_warn(chan2dev(chan), "failed to config lli");
981 goto err_txd_free;
982 }
983
984 if (!first)
985 first = lli;
986
987 prev = owl_dma_add_lli(txd, prev, lli, false);
988 }
989
990
991 owl_dma_add_lli(txd, prev, first, true);
992
993 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
994
995 err_txd_free:
996 owl_dma_free_txd(od, txd);
997
998 return NULL;
999 }
1000
1001 static void owl_dma_free_chan_resources(struct dma_chan *chan)
1002 {
1003 struct owl_dma_vchan *vchan = to_owl_vchan(chan);
1004
1005
1006 vchan_free_chan_resources(&vchan->vc);
1007 }
1008
1009 static inline void owl_dma_free(struct owl_dma *od)
1010 {
1011 struct owl_dma_vchan *vchan = NULL;
1012 struct owl_dma_vchan *next;
1013
1014 list_for_each_entry_safe(vchan,
1015 next, &od->dma.channels, vc.chan.device_node) {
1016 list_del(&vchan->vc.chan.device_node);
1017 tasklet_kill(&vchan->vc.task);
1018 }
1019 }
1020
1021 static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
1022 struct of_dma *ofdma)
1023 {
1024 struct owl_dma *od = ofdma->of_dma_data;
1025 struct owl_dma_vchan *vchan;
1026 struct dma_chan *chan;
1027 u8 drq = dma_spec->args[0];
1028
1029 if (drq > od->nr_vchans)
1030 return NULL;
1031
1032 chan = dma_get_any_slave_channel(&od->dma);
1033 if (!chan)
1034 return NULL;
1035
1036 vchan = to_owl_vchan(chan);
1037 vchan->drq = drq;
1038
1039 return chan;
1040 }
1041
1042 static int owl_dma_probe(struct platform_device *pdev)
1043 {
1044 struct device_node *np = pdev->dev.of_node;
1045 struct owl_dma *od;
1046 struct resource *res;
1047 int ret, i, nr_channels, nr_requests;
1048
1049 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1050 if (!od)
1051 return -ENOMEM;
1052
1053 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1054 if (!res)
1055 return -EINVAL;
1056
1057 od->base = devm_ioremap_resource(&pdev->dev, res);
1058 if (IS_ERR(od->base))
1059 return PTR_ERR(od->base);
1060
1061 ret = of_property_read_u32(np, "dma-channels", &nr_channels);
1062 if (ret) {
1063 dev_err(&pdev->dev, "can't get dma-channels\n");
1064 return ret;
1065 }
1066
1067 ret = of_property_read_u32(np, "dma-requests", &nr_requests);
1068 if (ret) {
1069 dev_err(&pdev->dev, "can't get dma-requests\n");
1070 return ret;
1071 }
1072
1073 dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",
1074 nr_channels, nr_requests);
1075
1076 od->nr_pchans = nr_channels;
1077 od->nr_vchans = nr_requests;
1078
1079 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
1080
1081 platform_set_drvdata(pdev, od);
1082 spin_lock_init(&od->lock);
1083
1084 dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);
1085 dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
1086 dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);
1087
1088 od->dma.dev = &pdev->dev;
1089 od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
1090 od->dma.device_tx_status = owl_dma_tx_status;
1091 od->dma.device_issue_pending = owl_dma_issue_pending;
1092 od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;
1093 od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
1094 od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
1095 od->dma.device_config = owl_dma_config;
1096 od->dma.device_pause = owl_dma_pause;
1097 od->dma.device_resume = owl_dma_resume;
1098 od->dma.device_terminate_all = owl_dma_terminate_all;
1099 od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1100 od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1101 od->dma.directions = BIT(DMA_MEM_TO_MEM);
1102 od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1103
1104 INIT_LIST_HEAD(&od->dma.channels);
1105
1106 od->clk = devm_clk_get(&pdev->dev, NULL);
1107 if (IS_ERR(od->clk)) {
1108 dev_err(&pdev->dev, "unable to get clock\n");
1109 return PTR_ERR(od->clk);
1110 }
1111
1112
1113
1114
1115
1116
1117 od->irq = platform_get_irq(pdev, 0);
1118 ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
1119 dev_name(&pdev->dev), od);
1120 if (ret) {
1121 dev_err(&pdev->dev, "unable to request IRQ\n");
1122 return ret;
1123 }
1124
1125
1126 od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
1127 sizeof(struct owl_dma_pchan), GFP_KERNEL);
1128 if (!od->pchans)
1129 return -ENOMEM;
1130
1131 for (i = 0; i < od->nr_pchans; i++) {
1132 struct owl_dma_pchan *pchan = &od->pchans[i];
1133
1134 pchan->id = i;
1135 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
1136 }
1137
1138
1139 od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
1140 sizeof(struct owl_dma_vchan), GFP_KERNEL);
1141 if (!od->vchans)
1142 return -ENOMEM;
1143
1144 for (i = 0; i < od->nr_vchans; i++) {
1145 struct owl_dma_vchan *vchan = &od->vchans[i];
1146
1147 vchan->vc.desc_free = owl_dma_desc_free;
1148 vchan_init(&vchan->vc, &od->dma);
1149 }
1150
1151
1152 od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
1153 sizeof(struct owl_dma_lli),
1154 __alignof__(struct owl_dma_lli),
1155 0);
1156 if (!od->lli_pool) {
1157 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");
1158 return -ENOMEM;
1159 }
1160
1161 clk_prepare_enable(od->clk);
1162
1163 ret = dma_async_device_register(&od->dma);
1164 if (ret) {
1165 dev_err(&pdev->dev, "failed to register DMA engine device\n");
1166 goto err_pool_free;
1167 }
1168
1169
1170 ret = of_dma_controller_register(pdev->dev.of_node,
1171 owl_dma_of_xlate, od);
1172 if (ret) {
1173 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1174 goto err_dma_unregister;
1175 }
1176
1177 return 0;
1178
1179 err_dma_unregister:
1180 dma_async_device_unregister(&od->dma);
1181 err_pool_free:
1182 clk_disable_unprepare(od->clk);
1183 dma_pool_destroy(od->lli_pool);
1184
1185 return ret;
1186 }
1187
1188 static int owl_dma_remove(struct platform_device *pdev)
1189 {
1190 struct owl_dma *od = platform_get_drvdata(pdev);
1191
1192 of_dma_controller_free(pdev->dev.of_node);
1193 dma_async_device_unregister(&od->dma);
1194
1195
1196 dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
1197
1198
1199 devm_free_irq(od->dma.dev, od->irq, od);
1200
1201 owl_dma_free(od);
1202
1203 clk_disable_unprepare(od->clk);
1204
1205 return 0;
1206 }
1207
1208 static const struct of_device_id owl_dma_match[] = {
1209 { .compatible = "actions,s900-dma", },
1210 { }
1211 };
1212 MODULE_DEVICE_TABLE(of, owl_dma_match);
1213
1214 static struct platform_driver owl_dma_driver = {
1215 .probe = owl_dma_probe,
1216 .remove = owl_dma_remove,
1217 .driver = {
1218 .name = "dma-owl",
1219 .of_match_table = of_match_ptr(owl_dma_match),
1220 },
1221 };
1222
1223 static int owl_dma_init(void)
1224 {
1225 return platform_driver_register(&owl_dma_driver);
1226 }
1227 subsys_initcall(owl_dma_init);
1228
1229 static void __exit owl_dma_exit(void)
1230 {
1231 platform_driver_unregister(&owl_dma_driver);
1232 }
1233 module_exit(owl_dma_exit);
1234
1235 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1236 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1237 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1238 MODULE_LICENSE("GPL");