1/* 2 * drivers/dma/fsl-edma.c 3 * 4 * Copyright 2013-2014 Freescale Semiconductor, Inc. 5 * 6 * Driver for the Freescale eDMA engine with flexible channel multiplexing 7 * capability for DMA request sources. The eDMA block can be found on some 8 * Vybrid and Layerscape SoCs. 9 * 10 * This program is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 */ 15 16#include <linux/init.h> 17#include <linux/module.h> 18#include <linux/interrupt.h> 19#include <linux/clk.h> 20#include <linux/dma-mapping.h> 21#include <linux/dmapool.h> 22#include <linux/slab.h> 23#include <linux/spinlock.h> 24#include <linux/of.h> 25#include <linux/of_device.h> 26#include <linux/of_address.h> 27#include <linux/of_irq.h> 28#include <linux/of_dma.h> 29 30#include "virt-dma.h" 31 32#define EDMA_CR 0x00 33#define EDMA_ES 0x04 34#define EDMA_ERQ 0x0C 35#define EDMA_EEI 0x14 36#define EDMA_SERQ 0x1B 37#define EDMA_CERQ 0x1A 38#define EDMA_SEEI 0x19 39#define EDMA_CEEI 0x18 40#define EDMA_CINT 0x1F 41#define EDMA_CERR 0x1E 42#define EDMA_SSRT 0x1D 43#define EDMA_CDNE 0x1C 44#define EDMA_INTR 0x24 45#define EDMA_ERR 0x2C 46 47#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) 48#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) 49#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) 50#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) 51#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) 52#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) 53#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) 54#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) 55#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) 56#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) 57#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) 58#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) 59#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) 60 61#define EDMA_CR_EDBG BIT(1) 62#define EDMA_CR_ERCA BIT(2) 63#define EDMA_CR_ERGA BIT(3) 64#define EDMA_CR_HOE BIT(4) 65#define EDMA_CR_HALT BIT(5) 66#define EDMA_CR_CLM BIT(6) 67#define EDMA_CR_EMLM BIT(7) 68#define EDMA_CR_ECX BIT(16) 69#define EDMA_CR_CX BIT(17) 70 71#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) 72#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) 73#define EDMA_CINT_CINT(x) ((x) & 0x1F) 74#define EDMA_CERR_CERR(x) ((x) & 0x1F) 75 76#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) 77#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) 78#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) 79#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) 80#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) 81#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) 82#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) 83#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) 84#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) 85#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) 86#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) 87#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) 88#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) 89#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) 90 91#define EDMA_TCD_SOFF_SOFF(x) (x) 92#define EDMA_TCD_NBYTES_NBYTES(x) (x) 93#define EDMA_TCD_SLAST_SLAST(x) (x) 94#define EDMA_TCD_DADDR_DADDR(x) (x) 95#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) 96#define EDMA_TCD_DOFF_DOFF(x) (x) 97#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) 98#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) 99 100#define EDMA_TCD_CSR_START BIT(0) 101#define EDMA_TCD_CSR_INT_MAJOR BIT(1) 102#define EDMA_TCD_CSR_INT_HALF BIT(2) 103#define EDMA_TCD_CSR_D_REQ BIT(3) 104#define EDMA_TCD_CSR_E_SG BIT(4) 105#define EDMA_TCD_CSR_E_LINK BIT(5) 106#define EDMA_TCD_CSR_ACTIVE BIT(6) 107#define EDMA_TCD_CSR_DONE BIT(7) 108 109#define EDMAMUX_CHCFG_DIS 0x0 110#define EDMAMUX_CHCFG_ENBL 0x80 111#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) 112 113#define DMAMUX_NR 2 114 115#define FSL_EDMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 116 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 117 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 118 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) 119 120struct fsl_edma_hw_tcd { 121 __le32 saddr; 122 __le16 soff; 123 __le16 attr; 124 __le32 nbytes; 125 __le32 slast; 126 __le32 daddr; 127 __le16 doff; 128 __le16 citer; 129 __le32 dlast_sga; 130 __le16 csr; 131 __le16 biter; 132}; 133 134struct fsl_edma_sw_tcd { 135 dma_addr_t ptcd; 136 struct fsl_edma_hw_tcd *vtcd; 137}; 138 139struct fsl_edma_slave_config { 140 enum dma_transfer_direction dir; 141 enum dma_slave_buswidth addr_width; 142 u32 dev_addr; 143 u32 burst; 144 u32 attr; 145}; 146 147struct fsl_edma_chan { 148 struct virt_dma_chan vchan; 149 enum dma_status status; 150 struct fsl_edma_engine *edma; 151 struct fsl_edma_desc *edesc; 152 struct fsl_edma_slave_config fsc; 153 struct dma_pool *tcd_pool; 154}; 155 156struct fsl_edma_desc { 157 struct virt_dma_desc vdesc; 158 struct fsl_edma_chan *echan; 159 bool iscyclic; 160 unsigned int n_tcds; 161 struct fsl_edma_sw_tcd tcd[]; 162}; 163 164struct fsl_edma_engine { 165 struct dma_device dma_dev; 166 void __iomem *membase; 167 void __iomem *muxbase[DMAMUX_NR]; 168 struct clk *muxclk[DMAMUX_NR]; 169 struct mutex fsl_edma_mutex; 170 u32 n_chans; 171 int txirq; 172 int errirq; 173 bool big_endian; 174 struct fsl_edma_chan chans[]; 175}; 176 177/* 178 * R/W functions for big- or little-endian registers: 179 * The eDMA controller's endian is independent of the CPU core's endian. 180 * For the big-endian IP module, the offset for 8-bit or 16-bit registers 181 * should also be swapped opposite to that in little-endian IP. 182 */ 183 184static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) 185{ 186 if (edma->big_endian) 187 return ioread32be(addr); 188 else 189 return ioread32(addr); 190} 191 192static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) 193{ 194 /* swap the reg offset for these in big-endian mode */ 195 if (edma->big_endian) 196 iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); 197 else 198 iowrite8(val, addr); 199} 200 201static void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) 202{ 203 /* swap the reg offset for these in big-endian mode */ 204 if (edma->big_endian) 205 iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); 206 else 207 iowrite16(val, addr); 208} 209 210static void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) 211{ 212 if (edma->big_endian) 213 iowrite32be(val, addr); 214 else 215 iowrite32(val, addr); 216} 217 218static struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) 219{ 220 return container_of(chan, struct fsl_edma_chan, vchan.chan); 221} 222 223static struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) 224{ 225 return container_of(vd, struct fsl_edma_desc, vdesc); 226} 227 228static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) 229{ 230 void __iomem *addr = fsl_chan->edma->membase; 231 u32 ch = fsl_chan->vchan.chan.chan_id; 232 233 edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); 234 edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); 235} 236 237static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) 238{ 239 void __iomem *addr = fsl_chan->edma->membase; 240 u32 ch = fsl_chan->vchan.chan.chan_id; 241 242 edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); 243 edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); 244} 245 246static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, 247 unsigned int slot, bool enable) 248{ 249 u32 ch = fsl_chan->vchan.chan.chan_id; 250 void __iomem *muxaddr; 251 unsigned chans_per_mux, ch_off; 252 253 chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; 254 ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; 255 muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; 256 slot = EDMAMUX_CHCFG_SOURCE(slot); 257 258 if (enable) 259 iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); 260 else 261 iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); 262} 263 264static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) 265{ 266 switch (addr_width) { 267 case 1: 268 return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; 269 case 2: 270 return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; 271 case 4: 272 return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; 273 case 8: 274 return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; 275 default: 276 return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; 277 } 278} 279 280static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) 281{ 282 struct fsl_edma_desc *fsl_desc; 283 int i; 284 285 fsl_desc = to_fsl_edma_desc(vdesc); 286 for (i = 0; i < fsl_desc->n_tcds; i++) 287 dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, 288 fsl_desc->tcd[i].ptcd); 289 kfree(fsl_desc); 290} 291 292static int fsl_edma_terminate_all(struct dma_chan *chan) 293{ 294 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 295 unsigned long flags; 296 LIST_HEAD(head); 297 298 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 299 fsl_edma_disable_request(fsl_chan); 300 fsl_chan->edesc = NULL; 301 vchan_get_all_descriptors(&fsl_chan->vchan, &head); 302 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 303 vchan_dma_desc_free_list(&fsl_chan->vchan, &head); 304 return 0; 305} 306 307static int fsl_edma_pause(struct dma_chan *chan) 308{ 309 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 310 unsigned long flags; 311 312 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 313 if (fsl_chan->edesc) { 314 fsl_edma_disable_request(fsl_chan); 315 fsl_chan->status = DMA_PAUSED; 316 } 317 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 318 return 0; 319} 320 321static int fsl_edma_resume(struct dma_chan *chan) 322{ 323 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 324 unsigned long flags; 325 326 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 327 if (fsl_chan->edesc) { 328 fsl_edma_enable_request(fsl_chan); 329 fsl_chan->status = DMA_IN_PROGRESS; 330 } 331 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 332 return 0; 333} 334 335static int fsl_edma_slave_config(struct dma_chan *chan, 336 struct dma_slave_config *cfg) 337{ 338 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 339 340 fsl_chan->fsc.dir = cfg->direction; 341 if (cfg->direction == DMA_DEV_TO_MEM) { 342 fsl_chan->fsc.dev_addr = cfg->src_addr; 343 fsl_chan->fsc.addr_width = cfg->src_addr_width; 344 fsl_chan->fsc.burst = cfg->src_maxburst; 345 fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); 346 } else if (cfg->direction == DMA_MEM_TO_DEV) { 347 fsl_chan->fsc.dev_addr = cfg->dst_addr; 348 fsl_chan->fsc.addr_width = cfg->dst_addr_width; 349 fsl_chan->fsc.burst = cfg->dst_maxburst; 350 fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); 351 } else { 352 return -EINVAL; 353 } 354 return 0; 355} 356 357static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, 358 struct virt_dma_desc *vdesc, bool in_progress) 359{ 360 struct fsl_edma_desc *edesc = fsl_chan->edesc; 361 void __iomem *addr = fsl_chan->edma->membase; 362 u32 ch = fsl_chan->vchan.chan.chan_id; 363 enum dma_transfer_direction dir = fsl_chan->fsc.dir; 364 dma_addr_t cur_addr, dma_addr; 365 size_t len, size; 366 int i; 367 368 /* calculate the total size in this desc */ 369 for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) 370 len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) 371 * le16_to_cpu(edesc->tcd[i].vtcd->biter); 372 373 if (!in_progress) 374 return len; 375 376 if (dir == DMA_MEM_TO_DEV) 377 cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); 378 else 379 cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); 380 381 /* figure out the finished and calculate the residue */ 382 for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { 383 size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) 384 * le16_to_cpu(edesc->tcd[i].vtcd->biter); 385 if (dir == DMA_MEM_TO_DEV) 386 dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); 387 else 388 dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); 389 390 len -= size; 391 if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { 392 len += dma_addr + size - cur_addr; 393 break; 394 } 395 } 396 397 return len; 398} 399 400static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, 401 dma_cookie_t cookie, struct dma_tx_state *txstate) 402{ 403 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 404 struct virt_dma_desc *vdesc; 405 enum dma_status status; 406 unsigned long flags; 407 408 status = dma_cookie_status(chan, cookie, txstate); 409 if (status == DMA_COMPLETE) 410 return status; 411 412 if (!txstate) 413 return fsl_chan->status; 414 415 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 416 vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); 417 if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) 418 txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true); 419 else if (vdesc) 420 txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false); 421 else 422 txstate->residue = 0; 423 424 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 425 426 return fsl_chan->status; 427} 428 429static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, 430 struct fsl_edma_hw_tcd *tcd) 431{ 432 struct fsl_edma_engine *edma = fsl_chan->edma; 433 void __iomem *addr = fsl_chan->edma->membase; 434 u32 ch = fsl_chan->vchan.chan.chan_id; 435 436 /* 437 * TCD parameters are stored in struct fsl_edma_hw_tcd in little 438 * endian format. However, we need to load the TCD registers in 439 * big- or little-endian obeying the eDMA engine model endian. 440 */ 441 edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); 442 edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); 443 edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); 444 445 edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); 446 edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); 447 448 edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); 449 edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); 450 451 edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); 452 edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); 453 edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); 454 455 edma_writel(edma, le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); 456 457 edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); 458} 459 460static inline 461void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, 462 u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, 463 u16 biter, u16 doff, u32 dlast_sga, bool major_int, 464 bool disable_req, bool enable_sg) 465{ 466 u16 csr = 0; 467 468 /* 469 * eDMA hardware SGs require the TCDs to be stored in little 470 * endian format irrespective of the register endian model. 471 * So we put the value in little endian in memory, waiting 472 * for fsl_edma_set_tcd_regs doing the swap. 473 */ 474 tcd->saddr = cpu_to_le32(src); 475 tcd->daddr = cpu_to_le32(dst); 476 477 tcd->attr = cpu_to_le16(attr); 478 479 tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); 480 481 tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); 482 tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); 483 484 tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); 485 tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); 486 487 tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); 488 489 tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); 490 if (major_int) 491 csr |= EDMA_TCD_CSR_INT_MAJOR; 492 493 if (disable_req) 494 csr |= EDMA_TCD_CSR_D_REQ; 495 496 if (enable_sg) 497 csr |= EDMA_TCD_CSR_E_SG; 498 499 tcd->csr = cpu_to_le16(csr); 500} 501 502static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, 503 int sg_len) 504{ 505 struct fsl_edma_desc *fsl_desc; 506 int i; 507 508 fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct fsl_edma_sw_tcd) * sg_len, 509 GFP_NOWAIT); 510 if (!fsl_desc) 511 return NULL; 512 513 fsl_desc->echan = fsl_chan; 514 fsl_desc->n_tcds = sg_len; 515 for (i = 0; i < sg_len; i++) { 516 fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, 517 GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); 518 if (!fsl_desc->tcd[i].vtcd) 519 goto err; 520 } 521 return fsl_desc; 522 523err: 524 while (--i >= 0) 525 dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, 526 fsl_desc->tcd[i].ptcd); 527 kfree(fsl_desc); 528 return NULL; 529} 530 531static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( 532 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, 533 size_t period_len, enum dma_transfer_direction direction, 534 unsigned long flags) 535{ 536 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 537 struct fsl_edma_desc *fsl_desc; 538 dma_addr_t dma_buf_next; 539 int sg_len, i; 540 u32 src_addr, dst_addr, last_sg, nbytes; 541 u16 soff, doff, iter; 542 543 if (!is_slave_direction(fsl_chan->fsc.dir)) 544 return NULL; 545 546 sg_len = buf_len / period_len; 547 fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); 548 if (!fsl_desc) 549 return NULL; 550 fsl_desc->iscyclic = true; 551 552 dma_buf_next = dma_addr; 553 nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; 554 iter = period_len / nbytes; 555 556 for (i = 0; i < sg_len; i++) { 557 if (dma_buf_next >= dma_addr + buf_len) 558 dma_buf_next = dma_addr; 559 560 /* get next sg's physical address */ 561 last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; 562 563 if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { 564 src_addr = dma_buf_next; 565 dst_addr = fsl_chan->fsc.dev_addr; 566 soff = fsl_chan->fsc.addr_width; 567 doff = 0; 568 } else { 569 src_addr = fsl_chan->fsc.dev_addr; 570 dst_addr = dma_buf_next; 571 soff = 0; 572 doff = fsl_chan->fsc.addr_width; 573 } 574 575 fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, 576 fsl_chan->fsc.attr, soff, nbytes, 0, iter, 577 iter, doff, last_sg, true, false, true); 578 dma_buf_next += period_len; 579 } 580 581 return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); 582} 583 584static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( 585 struct dma_chan *chan, struct scatterlist *sgl, 586 unsigned int sg_len, enum dma_transfer_direction direction, 587 unsigned long flags, void *context) 588{ 589 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 590 struct fsl_edma_desc *fsl_desc; 591 struct scatterlist *sg; 592 u32 src_addr, dst_addr, last_sg, nbytes; 593 u16 soff, doff, iter; 594 int i; 595 596 if (!is_slave_direction(fsl_chan->fsc.dir)) 597 return NULL; 598 599 fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); 600 if (!fsl_desc) 601 return NULL; 602 fsl_desc->iscyclic = false; 603 604 nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; 605 for_each_sg(sgl, sg, sg_len, i) { 606 /* get next sg's physical address */ 607 last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; 608 609 if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { 610 src_addr = sg_dma_address(sg); 611 dst_addr = fsl_chan->fsc.dev_addr; 612 soff = fsl_chan->fsc.addr_width; 613 doff = 0; 614 } else { 615 src_addr = fsl_chan->fsc.dev_addr; 616 dst_addr = sg_dma_address(sg); 617 soff = 0; 618 doff = fsl_chan->fsc.addr_width; 619 } 620 621 iter = sg_dma_len(sg) / nbytes; 622 if (i < sg_len - 1) { 623 last_sg = fsl_desc->tcd[(i + 1)].ptcd; 624 fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, 625 dst_addr, fsl_chan->fsc.attr, soff, 626 nbytes, 0, iter, iter, doff, last_sg, 627 false, false, true); 628 } else { 629 last_sg = 0; 630 fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, 631 dst_addr, fsl_chan->fsc.attr, soff, 632 nbytes, 0, iter, iter, doff, last_sg, 633 true, true, false); 634 } 635 } 636 637 return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); 638} 639 640static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) 641{ 642 struct virt_dma_desc *vdesc; 643 644 vdesc = vchan_next_desc(&fsl_chan->vchan); 645 if (!vdesc) 646 return; 647 fsl_chan->edesc = to_fsl_edma_desc(vdesc); 648 fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); 649 fsl_edma_enable_request(fsl_chan); 650 fsl_chan->status = DMA_IN_PROGRESS; 651} 652 653static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) 654{ 655 struct fsl_edma_engine *fsl_edma = dev_id; 656 unsigned int intr, ch; 657 void __iomem *base_addr; 658 struct fsl_edma_chan *fsl_chan; 659 660 base_addr = fsl_edma->membase; 661 662 intr = edma_readl(fsl_edma, base_addr + EDMA_INTR); 663 if (!intr) 664 return IRQ_NONE; 665 666 for (ch = 0; ch < fsl_edma->n_chans; ch++) { 667 if (intr & (0x1 << ch)) { 668 edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), 669 base_addr + EDMA_CINT); 670 671 fsl_chan = &fsl_edma->chans[ch]; 672 673 spin_lock(&fsl_chan->vchan.lock); 674 if (!fsl_chan->edesc->iscyclic) { 675 list_del(&fsl_chan->edesc->vdesc.node); 676 vchan_cookie_complete(&fsl_chan->edesc->vdesc); 677 fsl_chan->edesc = NULL; 678 fsl_chan->status = DMA_COMPLETE; 679 } else { 680 vchan_cyclic_callback(&fsl_chan->edesc->vdesc); 681 } 682 683 if (!fsl_chan->edesc) 684 fsl_edma_xfer_desc(fsl_chan); 685 686 spin_unlock(&fsl_chan->vchan.lock); 687 } 688 } 689 return IRQ_HANDLED; 690} 691 692static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) 693{ 694 struct fsl_edma_engine *fsl_edma = dev_id; 695 unsigned int err, ch; 696 697 err = edma_readl(fsl_edma, fsl_edma->membase + EDMA_ERR); 698 if (!err) 699 return IRQ_NONE; 700 701 for (ch = 0; ch < fsl_edma->n_chans; ch++) { 702 if (err & (0x1 << ch)) { 703 fsl_edma_disable_request(&fsl_edma->chans[ch]); 704 edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), 705 fsl_edma->membase + EDMA_CERR); 706 fsl_edma->chans[ch].status = DMA_ERROR; 707 } 708 } 709 return IRQ_HANDLED; 710} 711 712static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) 713{ 714 if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) 715 return IRQ_HANDLED; 716 717 return fsl_edma_err_handler(irq, dev_id); 718} 719 720static void fsl_edma_issue_pending(struct dma_chan *chan) 721{ 722 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 723 unsigned long flags; 724 725 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 726 727 if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) 728 fsl_edma_xfer_desc(fsl_chan); 729 730 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 731} 732 733static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, 734 struct of_dma *ofdma) 735{ 736 struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; 737 struct dma_chan *chan, *_chan; 738 unsigned long chans_per_mux = fsl_edma->n_chans / DMAMUX_NR; 739 740 if (dma_spec->args_count != 2) 741 return NULL; 742 743 mutex_lock(&fsl_edma->fsl_edma_mutex); 744 list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { 745 if (chan->client_count) 746 continue; 747 if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { 748 chan = dma_get_slave_channel(chan); 749 if (chan) { 750 chan->device->privatecnt++; 751 fsl_edma_chan_mux(to_fsl_edma_chan(chan), 752 dma_spec->args[1], true); 753 mutex_unlock(&fsl_edma->fsl_edma_mutex); 754 return chan; 755 } 756 } 757 } 758 mutex_unlock(&fsl_edma->fsl_edma_mutex); 759 return NULL; 760} 761 762static int fsl_edma_alloc_chan_resources(struct dma_chan *chan) 763{ 764 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 765 766 fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, 767 sizeof(struct fsl_edma_hw_tcd), 768 32, 0); 769 return 0; 770} 771 772static void fsl_edma_free_chan_resources(struct dma_chan *chan) 773{ 774 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); 775 unsigned long flags; 776 LIST_HEAD(head); 777 778 spin_lock_irqsave(&fsl_chan->vchan.lock, flags); 779 fsl_edma_disable_request(fsl_chan); 780 fsl_edma_chan_mux(fsl_chan, 0, false); 781 fsl_chan->edesc = NULL; 782 vchan_get_all_descriptors(&fsl_chan->vchan, &head); 783 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); 784 785 vchan_dma_desc_free_list(&fsl_chan->vchan, &head); 786 dma_pool_destroy(fsl_chan->tcd_pool); 787 fsl_chan->tcd_pool = NULL; 788} 789 790static int 791fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) 792{ 793 int ret; 794 795 fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); 796 if (fsl_edma->txirq < 0) { 797 dev_err(&pdev->dev, "Can't get edma-tx irq.\n"); 798 return fsl_edma->txirq; 799 } 800 801 fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); 802 if (fsl_edma->errirq < 0) { 803 dev_err(&pdev->dev, "Can't get edma-err irq.\n"); 804 return fsl_edma->errirq; 805 } 806 807 if (fsl_edma->txirq == fsl_edma->errirq) { 808 ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, 809 fsl_edma_irq_handler, 0, "eDMA", fsl_edma); 810 if (ret) { 811 dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); 812 return ret; 813 } 814 } else { 815 ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, 816 fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); 817 if (ret) { 818 dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); 819 return ret; 820 } 821 822 ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, 823 fsl_edma_err_handler, 0, "eDMA err", fsl_edma); 824 if (ret) { 825 dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); 826 return ret; 827 } 828 } 829 830 return 0; 831} 832 833static int fsl_edma_probe(struct platform_device *pdev) 834{ 835 struct device_node *np = pdev->dev.of_node; 836 struct fsl_edma_engine *fsl_edma; 837 struct fsl_edma_chan *fsl_chan; 838 struct resource *res; 839 int len, chans; 840 int ret, i; 841 842 ret = of_property_read_u32(np, "dma-channels", &chans); 843 if (ret) { 844 dev_err(&pdev->dev, "Can't get dma-channels.\n"); 845 return ret; 846 } 847 848 len = sizeof(*fsl_edma) + sizeof(*fsl_chan) * chans; 849 fsl_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); 850 if (!fsl_edma) 851 return -ENOMEM; 852 853 fsl_edma->n_chans = chans; 854 mutex_init(&fsl_edma->fsl_edma_mutex); 855 856 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 857 fsl_edma->membase = devm_ioremap_resource(&pdev->dev, res); 858 if (IS_ERR(fsl_edma->membase)) 859 return PTR_ERR(fsl_edma->membase); 860 861 for (i = 0; i < DMAMUX_NR; i++) { 862 char clkname[32]; 863 864 res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); 865 fsl_edma->muxbase[i] = devm_ioremap_resource(&pdev->dev, res); 866 if (IS_ERR(fsl_edma->muxbase[i])) 867 return PTR_ERR(fsl_edma->muxbase[i]); 868 869 sprintf(clkname, "dmamux%d", i); 870 fsl_edma->muxclk[i] = devm_clk_get(&pdev->dev, clkname); 871 if (IS_ERR(fsl_edma->muxclk[i])) { 872 dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); 873 return PTR_ERR(fsl_edma->muxclk[i]); 874 } 875 876 ret = clk_prepare_enable(fsl_edma->muxclk[i]); 877 if (ret) { 878 dev_err(&pdev->dev, "DMAMUX clk block failed.\n"); 879 return ret; 880 } 881 882 } 883 884 fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); 885 886 INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); 887 for (i = 0; i < fsl_edma->n_chans; i++) { 888 struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; 889 890 fsl_chan->edma = fsl_edma; 891 892 fsl_chan->vchan.desc_free = fsl_edma_free_desc; 893 vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); 894 895 edma_writew(fsl_edma, 0x0, fsl_edma->membase + EDMA_TCD_CSR(i)); 896 fsl_edma_chan_mux(fsl_chan, 0, false); 897 } 898 899 edma_writel(fsl_edma, ~0, fsl_edma->membase + EDMA_INTR); 900 ret = fsl_edma_irq_init(pdev, fsl_edma); 901 if (ret) 902 return ret; 903 904 dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); 905 dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); 906 dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); 907 908 fsl_edma->dma_dev.dev = &pdev->dev; 909 fsl_edma->dma_dev.device_alloc_chan_resources 910 = fsl_edma_alloc_chan_resources; 911 fsl_edma->dma_dev.device_free_chan_resources 912 = fsl_edma_free_chan_resources; 913 fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; 914 fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; 915 fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; 916 fsl_edma->dma_dev.device_config = fsl_edma_slave_config; 917 fsl_edma->dma_dev.device_pause = fsl_edma_pause; 918 fsl_edma->dma_dev.device_resume = fsl_edma_resume; 919 fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; 920 fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; 921 922 fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; 923 fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; 924 fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 925 926 platform_set_drvdata(pdev, fsl_edma); 927 928 ret = dma_async_device_register(&fsl_edma->dma_dev); 929 if (ret) { 930 dev_err(&pdev->dev, "Can't register Freescale eDMA engine.\n"); 931 return ret; 932 } 933 934 ret = of_dma_controller_register(np, fsl_edma_xlate, fsl_edma); 935 if (ret) { 936 dev_err(&pdev->dev, "Can't register Freescale eDMA of_dma.\n"); 937 dma_async_device_unregister(&fsl_edma->dma_dev); 938 return ret; 939 } 940 941 /* enable round robin arbitration */ 942 edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, fsl_edma->membase + EDMA_CR); 943 944 return 0; 945} 946 947static int fsl_edma_remove(struct platform_device *pdev) 948{ 949 struct device_node *np = pdev->dev.of_node; 950 struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); 951 int i; 952 953 of_dma_controller_free(np); 954 dma_async_device_unregister(&fsl_edma->dma_dev); 955 956 for (i = 0; i < DMAMUX_NR; i++) 957 clk_disable_unprepare(fsl_edma->muxclk[i]); 958 959 return 0; 960} 961 962static const struct of_device_id fsl_edma_dt_ids[] = { 963 { .compatible = "fsl,vf610-edma", }, 964 { /* sentinel */ } 965}; 966MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); 967 968static struct platform_driver fsl_edma_driver = { 969 .driver = { 970 .name = "fsl-edma", 971 .of_match_table = fsl_edma_dt_ids, 972 }, 973 .probe = fsl_edma_probe, 974 .remove = fsl_edma_remove, 975}; 976 977static int __init fsl_edma_init(void) 978{ 979 return platform_driver_register(&fsl_edma_driver); 980} 981subsys_initcall(fsl_edma_init); 982 983static void __exit fsl_edma_exit(void) 984{ 985 platform_driver_unregister(&fsl_edma_driver); 986} 987module_exit(fsl_edma_exit); 988 989MODULE_ALIAS("platform:fsl-edma"); 990MODULE_DESCRIPTION("Freescale eDMA engine driver"); 991MODULE_LICENSE("GPL v2"); 992