root/drivers/dma/ti/omap-dma.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. to_omap_dma_dev
  2. to_omap_dma_chan
  3. to_omap_dma_desc
  4. omap_dma_desc_free
  5. omap_dma_fill_type2_desc
  6. omap_dma_write
  7. omap_dma_read
  8. omap_dma_glbl_write
  9. omap_dma_glbl_read
  10. omap_dma_chan_write
  11. omap_dma_chan_read
  12. omap_dma_clear_csr
  13. omap_dma_get_csr
  14. omap_dma_assign
  15. omap_dma_start
  16. omap_dma_drain_chan
  17. omap_dma_stop
  18. omap_dma_start_sg
  19. omap_dma_start_desc
  20. omap_dma_callback
  21. omap_dma_irq
  22. omap_dma_alloc_chan_resources
  23. omap_dma_free_chan_resources
  24. omap_dma_sg_size
  25. omap_dma_desc_size
  26. omap_dma_desc_size_pos
  27. omap_dma_chan_read_3_3
  28. omap_dma_get_src_pos
  29. omap_dma_get_dst_pos
  30. omap_dma_tx_status
  31. omap_dma_issue_pending
  32. omap_dma_prep_slave_sg
  33. omap_dma_prep_dma_cyclic
  34. omap_dma_prep_dma_memcpy
  35. omap_dma_prep_dma_interleaved
  36. omap_dma_slave_config
  37. omap_dma_terminate_all
  38. omap_dma_synchronize
  39. omap_dma_pause
  40. omap_dma_resume
  41. omap_dma_chan_init
  42. omap_dma_free
  43. omap_dma_probe
  44. omap_dma_remove
  45. omap_dma_filter_fn
  46. omap_dma_init
  47. omap_dma_exit

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * OMAP DMAengine support
   4  */
   5 #include <linux/delay.h>
   6 #include <linux/dmaengine.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/dmapool.h>
   9 #include <linux/err.h>
  10 #include <linux/init.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/list.h>
  13 #include <linux/module.h>
  14 #include <linux/omap-dma.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/spinlock.h>
  18 #include <linux/of_dma.h>
  19 #include <linux/of_device.h>
  20 
  21 #include "../virt-dma.h"
  22 
  23 #define OMAP_SDMA_REQUESTS      127
  24 #define OMAP_SDMA_CHANNELS      32
  25 
  26 struct omap_dmadev {
  27         struct dma_device ddev;
  28         spinlock_t lock;
  29         void __iomem *base;
  30         const struct omap_dma_reg *reg_map;
  31         struct omap_system_dma_plat_info *plat;
  32         bool legacy;
  33         bool ll123_supported;
  34         struct dma_pool *desc_pool;
  35         unsigned dma_requests;
  36         spinlock_t irq_lock;
  37         uint32_t irq_enable_mask;
  38         struct omap_chan **lch_map;
  39 };
  40 
  41 struct omap_chan {
  42         struct virt_dma_chan vc;
  43         void __iomem *channel_base;
  44         const struct omap_dma_reg *reg_map;
  45         uint32_t ccr;
  46 
  47         struct dma_slave_config cfg;
  48         unsigned dma_sig;
  49         bool cyclic;
  50         bool paused;
  51         bool running;
  52 
  53         int dma_ch;
  54         struct omap_desc *desc;
  55         unsigned sgidx;
  56 };
  57 
  58 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
  59 #define DESC_NXT_SV_REUSE       (0x2 << 24)
  60 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
  61 #define DESC_NXT_DV_REUSE       (0x2 << 26)
  62 #define DESC_NTYPE_TYPE2        (0x2 << 29)
  63 
  64 /* Type 2 descriptor with Source or Destination address update */
  65 struct omap_type2_desc {
  66         uint32_t next_desc;
  67         uint32_t en;
  68         uint32_t addr; /* src or dst */
  69         uint16_t fn;
  70         uint16_t cicr;
  71         int16_t cdei;
  72         int16_t csei;
  73         int32_t cdfi;
  74         int32_t csfi;
  75 } __packed;
  76 
  77 struct omap_sg {
  78         dma_addr_t addr;
  79         uint32_t en;            /* number of elements (24-bit) */
  80         uint32_t fn;            /* number of frames (16-bit) */
  81         int32_t fi;             /* for double indexing */
  82         int16_t ei;             /* for double indexing */
  83 
  84         /* Linked list */
  85         struct omap_type2_desc *t2_desc;
  86         dma_addr_t t2_desc_paddr;
  87 };
  88 
  89 struct omap_desc {
  90         struct virt_dma_desc vd;
  91         bool using_ll;
  92         enum dma_transfer_direction dir;
  93         dma_addr_t dev_addr;
  94         bool polled;
  95 
  96         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  97         int16_t ei;             /* for double indexing */
  98         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
  99         uint32_t ccr;           /* CCR value */
 100         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
 101         uint16_t cicr;          /* CICR value */
 102         uint32_t csdp;          /* CSDP value */
 103 
 104         unsigned sglen;
 105         struct omap_sg sg[0];
 106 };
 107 
 108 enum {
 109         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
 110         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
 111 
 112         CCR_FS                  = BIT(5),
 113         CCR_READ_PRIORITY       = BIT(6),
 114         CCR_ENABLE              = BIT(7),
 115         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
 116         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
 117         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
 118         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
 119         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
 120         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
 121         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
 122         CCR_SRC_AMODE_POSTINC   = 1 << 12,
 123         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
 124         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
 125         CCR_DST_AMODE_CONSTANT  = 0 << 14,
 126         CCR_DST_AMODE_POSTINC   = 1 << 14,
 127         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 128         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 129         CCR_CONSTANT_FILL       = BIT(16),
 130         CCR_TRANSPARENT_COPY    = BIT(17),
 131         CCR_BS                  = BIT(18),
 132         CCR_SUPERVISOR          = BIT(22),
 133         CCR_PREFETCH            = BIT(23),
 134         CCR_TRIGGER_SRC         = BIT(24),
 135         CCR_BUFFERING_DISABLE   = BIT(25),
 136         CCR_WRITE_PRIORITY      = BIT(26),
 137         CCR_SYNC_ELEMENT        = 0,
 138         CCR_SYNC_FRAME          = CCR_FS,
 139         CCR_SYNC_BLOCK          = CCR_BS,
 140         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 141 
 142         CSDP_DATA_TYPE_8        = 0,
 143         CSDP_DATA_TYPE_16       = 1,
 144         CSDP_DATA_TYPE_32       = 2,
 145         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 146         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 147         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 148         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 149         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 150         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 151         CSDP_SRC_PACKED         = BIT(6),
 152         CSDP_SRC_BURST_1        = 0 << 7,
 153         CSDP_SRC_BURST_16       = 1 << 7,
 154         CSDP_SRC_BURST_32       = 2 << 7,
 155         CSDP_SRC_BURST_64       = 3 << 7,
 156         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 157         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 158         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 159         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 160         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 161         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 162         CSDP_DST_PACKED         = BIT(13),
 163         CSDP_DST_BURST_1        = 0 << 14,
 164         CSDP_DST_BURST_16       = 1 << 14,
 165         CSDP_DST_BURST_32       = 2 << 14,
 166         CSDP_DST_BURST_64       = 3 << 14,
 167         CSDP_WRITE_NON_POSTED   = 0 << 16,
 168         CSDP_WRITE_POSTED       = 1 << 16,
 169         CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 170 
 171         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 172         CICR_DROP_IE            = BIT(1),
 173         CICR_HALF_IE            = BIT(2),
 174         CICR_FRAME_IE           = BIT(3),
 175         CICR_LAST_IE            = BIT(4),
 176         CICR_BLOCK_IE           = BIT(5),
 177         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 178         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 179         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 180         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 181         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 182         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 183 
 184         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 185 
 186         CDP_DST_VALID_INC       = 0 << 0,
 187         CDP_DST_VALID_RELOAD    = 1 << 0,
 188         CDP_DST_VALID_REUSE     = 2 << 0,
 189         CDP_SRC_VALID_INC       = 0 << 2,
 190         CDP_SRC_VALID_RELOAD    = 1 << 2,
 191         CDP_SRC_VALID_REUSE     = 2 << 2,
 192         CDP_NTYPE_TYPE1         = 1 << 4,
 193         CDP_NTYPE_TYPE2         = 2 << 4,
 194         CDP_NTYPE_TYPE3         = 3 << 4,
 195         CDP_TMODE_NORMAL        = 0 << 8,
 196         CDP_TMODE_LLIST         = 1 << 8,
 197         CDP_FAST                = BIT(10),
 198 };
 199 
 200 static const unsigned es_bytes[] = {
 201         [CSDP_DATA_TYPE_8] = 1,
 202         [CSDP_DATA_TYPE_16] = 2,
 203         [CSDP_DATA_TYPE_32] = 4,
 204 };
 205 
 206 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param);
 207 static struct of_dma_filter_info omap_dma_info = {
 208         .filter_fn = omap_dma_filter_fn,
 209 };
 210 
 211 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 212 {
 213         return container_of(d, struct omap_dmadev, ddev);
 214 }
 215 
 216 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 217 {
 218         return container_of(c, struct omap_chan, vc.chan);
 219 }
 220 
 221 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 222 {
 223         return container_of(t, struct omap_desc, vd.tx);
 224 }
 225 
 226 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 227 {
 228         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 229 
 230         if (d->using_ll) {
 231                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 232                 int i;
 233 
 234                 for (i = 0; i < d->sglen; i++) {
 235                         if (d->sg[i].t2_desc)
 236                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 237                                               d->sg[i].t2_desc_paddr);
 238                 }
 239         }
 240 
 241         kfree(d);
 242 }
 243 
 244 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 245                                      enum dma_transfer_direction dir, bool last)
 246 {
 247         struct omap_sg *sg = &d->sg[idx];
 248         struct omap_type2_desc *t2_desc = sg->t2_desc;
 249 
 250         if (idx)
 251                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 252         if (last)
 253                 t2_desc->next_desc = 0xfffffffc;
 254 
 255         t2_desc->en = sg->en;
 256         t2_desc->addr = sg->addr;
 257         t2_desc->fn = sg->fn & 0xffff;
 258         t2_desc->cicr = d->cicr;
 259         if (!last)
 260                 t2_desc->cicr &= ~CICR_BLOCK_IE;
 261 
 262         switch (dir) {
 263         case DMA_DEV_TO_MEM:
 264                 t2_desc->cdei = sg->ei;
 265                 t2_desc->csei = d->ei;
 266                 t2_desc->cdfi = sg->fi;
 267                 t2_desc->csfi = d->fi;
 268 
 269                 t2_desc->en |= DESC_NXT_DV_REFRESH;
 270                 t2_desc->en |= DESC_NXT_SV_REUSE;
 271                 break;
 272         case DMA_MEM_TO_DEV:
 273                 t2_desc->cdei = d->ei;
 274                 t2_desc->csei = sg->ei;
 275                 t2_desc->cdfi = d->fi;
 276                 t2_desc->csfi = sg->fi;
 277 
 278                 t2_desc->en |= DESC_NXT_SV_REFRESH;
 279                 t2_desc->en |= DESC_NXT_DV_REUSE;
 280                 break;
 281         default:
 282                 return;
 283         }
 284 
 285         t2_desc->en |= DESC_NTYPE_TYPE2;
 286 }
 287 
 288 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 289 {
 290         switch (type) {
 291         case OMAP_DMA_REG_16BIT:
 292                 writew_relaxed(val, addr);
 293                 break;
 294         case OMAP_DMA_REG_2X16BIT:
 295                 writew_relaxed(val, addr);
 296                 writew_relaxed(val >> 16, addr + 2);
 297                 break;
 298         case OMAP_DMA_REG_32BIT:
 299                 writel_relaxed(val, addr);
 300                 break;
 301         default:
 302                 WARN_ON(1);
 303         }
 304 }
 305 
 306 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 307 {
 308         unsigned val;
 309 
 310         switch (type) {
 311         case OMAP_DMA_REG_16BIT:
 312                 val = readw_relaxed(addr);
 313                 break;
 314         case OMAP_DMA_REG_2X16BIT:
 315                 val = readw_relaxed(addr);
 316                 val |= readw_relaxed(addr + 2) << 16;
 317                 break;
 318         case OMAP_DMA_REG_32BIT:
 319                 val = readl_relaxed(addr);
 320                 break;
 321         default:
 322                 WARN_ON(1);
 323                 val = 0;
 324         }
 325 
 326         return val;
 327 }
 328 
 329 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 330 {
 331         const struct omap_dma_reg *r = od->reg_map + reg;
 332 
 333         WARN_ON(r->stride);
 334 
 335         omap_dma_write(val, r->type, od->base + r->offset);
 336 }
 337 
 338 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 339 {
 340         const struct omap_dma_reg *r = od->reg_map + reg;
 341 
 342         WARN_ON(r->stride);
 343 
 344         return omap_dma_read(r->type, od->base + r->offset);
 345 }
 346 
 347 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 348 {
 349         const struct omap_dma_reg *r = c->reg_map + reg;
 350 
 351         omap_dma_write(val, r->type, c->channel_base + r->offset);
 352 }
 353 
 354 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 355 {
 356         const struct omap_dma_reg *r = c->reg_map + reg;
 357 
 358         return omap_dma_read(r->type, c->channel_base + r->offset);
 359 }
 360 
 361 static void omap_dma_clear_csr(struct omap_chan *c)
 362 {
 363         if (dma_omap1())
 364                 omap_dma_chan_read(c, CSR);
 365         else
 366                 omap_dma_chan_write(c, CSR, ~0);
 367 }
 368 
 369 static unsigned omap_dma_get_csr(struct omap_chan *c)
 370 {
 371         unsigned val = omap_dma_chan_read(c, CSR);
 372 
 373         if (!dma_omap1())
 374                 omap_dma_chan_write(c, CSR, val);
 375 
 376         return val;
 377 }
 378 
 379 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 380         unsigned lch)
 381 {
 382         c->channel_base = od->base + od->plat->channel_stride * lch;
 383 
 384         od->lch_map[lch] = c;
 385 }
 386 
 387 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 388 {
 389         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 390         uint16_t cicr = d->cicr;
 391 
 392         if (__dma_omap15xx(od->plat->dma_attr))
 393                 omap_dma_chan_write(c, CPC, 0);
 394         else
 395                 omap_dma_chan_write(c, CDAC, 0);
 396 
 397         omap_dma_clear_csr(c);
 398 
 399         if (d->using_ll) {
 400                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 401 
 402                 if (d->dir == DMA_DEV_TO_MEM)
 403                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 404                 else
 405                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 406                 omap_dma_chan_write(c, CDP, cdp);
 407 
 408                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 409                 omap_dma_chan_write(c, CCDN, 0);
 410                 omap_dma_chan_write(c, CCFN, 0xffff);
 411                 omap_dma_chan_write(c, CCEN, 0xffffff);
 412 
 413                 cicr &= ~CICR_BLOCK_IE;
 414         } else if (od->ll123_supported) {
 415                 omap_dma_chan_write(c, CDP, 0);
 416         }
 417 
 418         /* Enable interrupts */
 419         omap_dma_chan_write(c, CICR, cicr);
 420 
 421         /* Enable channel */
 422         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 423 
 424         c->running = true;
 425 }
 426 
 427 static void omap_dma_drain_chan(struct omap_chan *c)
 428 {
 429         int i;
 430         u32 val;
 431 
 432         /* Wait for sDMA FIFO to drain */
 433         for (i = 0; ; i++) {
 434                 val = omap_dma_chan_read(c, CCR);
 435                 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 436                         break;
 437 
 438                 if (i > 100)
 439                         break;
 440 
 441                 udelay(5);
 442         }
 443 
 444         if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 445                 dev_err(c->vc.chan.device->dev,
 446                         "DMA drain did not complete on lch %d\n",
 447                         c->dma_ch);
 448 }
 449 
 450 static int omap_dma_stop(struct omap_chan *c)
 451 {
 452         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 453         uint32_t val;
 454 
 455         /* disable irq */
 456         omap_dma_chan_write(c, CICR, 0);
 457 
 458         omap_dma_clear_csr(c);
 459 
 460         val = omap_dma_chan_read(c, CCR);
 461         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 462                 uint32_t sysconfig;
 463 
 464                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 465                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 466                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 467                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 468 
 469                 val = omap_dma_chan_read(c, CCR);
 470                 val &= ~CCR_ENABLE;
 471                 omap_dma_chan_write(c, CCR, val);
 472 
 473                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 474                         omap_dma_drain_chan(c);
 475 
 476                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 477         } else {
 478                 if (!(val & CCR_ENABLE))
 479                         return -EINVAL;
 480 
 481                 val &= ~CCR_ENABLE;
 482                 omap_dma_chan_write(c, CCR, val);
 483 
 484                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 485                         omap_dma_drain_chan(c);
 486         }
 487 
 488         mb();
 489 
 490         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 491                 val = omap_dma_chan_read(c, CLNK_CTRL);
 492 
 493                 if (dma_omap1())
 494                         val |= 1 << 14; /* set the STOP_LNK bit */
 495                 else
 496                         val &= ~CLNK_CTRL_ENABLE_LNK;
 497 
 498                 omap_dma_chan_write(c, CLNK_CTRL, val);
 499         }
 500         c->running = false;
 501         return 0;
 502 }
 503 
 504 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 505 {
 506         struct omap_sg *sg = d->sg + c->sgidx;
 507         unsigned cxsa, cxei, cxfi;
 508 
 509         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 510                 cxsa = CDSA;
 511                 cxei = CDEI;
 512                 cxfi = CDFI;
 513         } else {
 514                 cxsa = CSSA;
 515                 cxei = CSEI;
 516                 cxfi = CSFI;
 517         }
 518 
 519         omap_dma_chan_write(c, cxsa, sg->addr);
 520         omap_dma_chan_write(c, cxei, sg->ei);
 521         omap_dma_chan_write(c, cxfi, sg->fi);
 522         omap_dma_chan_write(c, CEN, sg->en);
 523         omap_dma_chan_write(c, CFN, sg->fn);
 524 
 525         omap_dma_start(c, d);
 526         c->sgidx++;
 527 }
 528 
 529 static void omap_dma_start_desc(struct omap_chan *c)
 530 {
 531         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 532         struct omap_desc *d;
 533         unsigned cxsa, cxei, cxfi;
 534 
 535         if (!vd) {
 536                 c->desc = NULL;
 537                 return;
 538         }
 539 
 540         list_del(&vd->node);
 541 
 542         c->desc = d = to_omap_dma_desc(&vd->tx);
 543         c->sgidx = 0;
 544 
 545         /*
 546          * This provides the necessary barrier to ensure data held in
 547          * DMA coherent memory is visible to the DMA engine prior to
 548          * the transfer starting.
 549          */
 550         mb();
 551 
 552         omap_dma_chan_write(c, CCR, d->ccr);
 553         if (dma_omap1())
 554                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 555 
 556         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 557                 cxsa = CSSA;
 558                 cxei = CSEI;
 559                 cxfi = CSFI;
 560         } else {
 561                 cxsa = CDSA;
 562                 cxei = CDEI;
 563                 cxfi = CDFI;
 564         }
 565 
 566         omap_dma_chan_write(c, cxsa, d->dev_addr);
 567         omap_dma_chan_write(c, cxei, d->ei);
 568         omap_dma_chan_write(c, cxfi, d->fi);
 569         omap_dma_chan_write(c, CSDP, d->csdp);
 570         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 571 
 572         omap_dma_start_sg(c, d);
 573 }
 574 
 575 static void omap_dma_callback(int ch, u16 status, void *data)
 576 {
 577         struct omap_chan *c = data;
 578         struct omap_desc *d;
 579         unsigned long flags;
 580 
 581         spin_lock_irqsave(&c->vc.lock, flags);
 582         d = c->desc;
 583         if (d) {
 584                 if (c->cyclic) {
 585                         vchan_cyclic_callback(&d->vd);
 586                 } else if (d->using_ll || c->sgidx == d->sglen) {
 587                         omap_dma_start_desc(c);
 588                         vchan_cookie_complete(&d->vd);
 589                 } else {
 590                         omap_dma_start_sg(c, d);
 591                 }
 592         }
 593         spin_unlock_irqrestore(&c->vc.lock, flags);
 594 }
 595 
 596 static irqreturn_t omap_dma_irq(int irq, void *devid)
 597 {
 598         struct omap_dmadev *od = devid;
 599         unsigned status, channel;
 600 
 601         spin_lock(&od->irq_lock);
 602 
 603         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 604         status &= od->irq_enable_mask;
 605         if (status == 0) {
 606                 spin_unlock(&od->irq_lock);
 607                 return IRQ_NONE;
 608         }
 609 
 610         while ((channel = ffs(status)) != 0) {
 611                 unsigned mask, csr;
 612                 struct omap_chan *c;
 613 
 614                 channel -= 1;
 615                 mask = BIT(channel);
 616                 status &= ~mask;
 617 
 618                 c = od->lch_map[channel];
 619                 if (c == NULL) {
 620                         /* This should never happen */
 621                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 622                         continue;
 623                 }
 624 
 625                 csr = omap_dma_get_csr(c);
 626                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 627 
 628                 omap_dma_callback(channel, csr, c);
 629         }
 630 
 631         spin_unlock(&od->irq_lock);
 632 
 633         return IRQ_HANDLED;
 634 }
 635 
 636 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 637 {
 638         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 639         struct omap_chan *c = to_omap_dma_chan(chan);
 640         struct device *dev = od->ddev.dev;
 641         int ret;
 642 
 643         if (od->legacy) {
 644                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 645                                        omap_dma_callback, c, &c->dma_ch);
 646         } else {
 647                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 648                                        &c->dma_ch);
 649         }
 650 
 651         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 652 
 653         if (ret >= 0) {
 654                 omap_dma_assign(od, c, c->dma_ch);
 655 
 656                 if (!od->legacy) {
 657                         unsigned val;
 658 
 659                         spin_lock_irq(&od->irq_lock);
 660                         val = BIT(c->dma_ch);
 661                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 662                         od->irq_enable_mask |= val;
 663                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 664 
 665                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 666                         val &= ~BIT(c->dma_ch);
 667                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 668                         spin_unlock_irq(&od->irq_lock);
 669                 }
 670         }
 671 
 672         if (dma_omap1()) {
 673                 if (__dma_omap16xx(od->plat->dma_attr)) {
 674                         c->ccr = CCR_OMAP31_DISABLE;
 675                         /* Duplicate what plat-omap/dma.c does */
 676                         c->ccr |= c->dma_ch + 1;
 677                 } else {
 678                         c->ccr = c->dma_sig & 0x1f;
 679                 }
 680         } else {
 681                 c->ccr = c->dma_sig & 0x1f;
 682                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 683         }
 684         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 685                 c->ccr |= CCR_BUFFERING_DISABLE;
 686 
 687         return ret;
 688 }
 689 
 690 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 691 {
 692         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 693         struct omap_chan *c = to_omap_dma_chan(chan);
 694 
 695         if (!od->legacy) {
 696                 spin_lock_irq(&od->irq_lock);
 697                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 698                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 699                 spin_unlock_irq(&od->irq_lock);
 700         }
 701 
 702         c->channel_base = NULL;
 703         od->lch_map[c->dma_ch] = NULL;
 704         vchan_free_chan_resources(&c->vc);
 705         omap_free_dma(c->dma_ch);
 706 
 707         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 708                 c->dma_sig);
 709         c->dma_sig = 0;
 710 }
 711 
 712 static size_t omap_dma_sg_size(struct omap_sg *sg)
 713 {
 714         return sg->en * sg->fn;
 715 }
 716 
 717 static size_t omap_dma_desc_size(struct omap_desc *d)
 718 {
 719         unsigned i;
 720         size_t size;
 721 
 722         for (size = i = 0; i < d->sglen; i++)
 723                 size += omap_dma_sg_size(&d->sg[i]);
 724 
 725         return size * es_bytes[d->es];
 726 }
 727 
 728 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 729 {
 730         unsigned i;
 731         size_t size, es_size = es_bytes[d->es];
 732 
 733         for (size = i = 0; i < d->sglen; i++) {
 734                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 735 
 736                 if (size)
 737                         size += this_size;
 738                 else if (addr >= d->sg[i].addr &&
 739                          addr < d->sg[i].addr + this_size)
 740                         size += d->sg[i].addr + this_size - addr;
 741         }
 742         return size;
 743 }
 744 
 745 /*
 746  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 747  * read before the DMA controller finished disabling the channel.
 748  */
 749 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 750 {
 751         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 752         uint32_t val;
 753 
 754         val = omap_dma_chan_read(c, reg);
 755         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 756                 val = omap_dma_chan_read(c, reg);
 757 
 758         return val;
 759 }
 760 
 761 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 762 {
 763         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 764         dma_addr_t addr, cdac;
 765 
 766         if (__dma_omap15xx(od->plat->dma_attr)) {
 767                 addr = omap_dma_chan_read(c, CPC);
 768         } else {
 769                 addr = omap_dma_chan_read_3_3(c, CSAC);
 770                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 771 
 772                 /*
 773                  * CDAC == 0 indicates that the DMA transfer on the channel has
 774                  * not been started (no data has been transferred so far).
 775                  * Return the programmed source start address in this case.
 776                  */
 777                 if (cdac == 0)
 778                         addr = omap_dma_chan_read(c, CSSA);
 779         }
 780 
 781         if (dma_omap1())
 782                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 783 
 784         return addr;
 785 }
 786 
 787 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 788 {
 789         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 790         dma_addr_t addr;
 791 
 792         if (__dma_omap15xx(od->plat->dma_attr)) {
 793                 addr = omap_dma_chan_read(c, CPC);
 794         } else {
 795                 addr = omap_dma_chan_read_3_3(c, CDAC);
 796 
 797                 /*
 798                  * CDAC == 0 indicates that the DMA transfer on the channel
 799                  * has not been started (no data has been transferred so
 800                  * far).  Return the programmed destination start address in
 801                  * this case.
 802                  */
 803                 if (addr == 0)
 804                         addr = omap_dma_chan_read(c, CDSA);
 805         }
 806 
 807         if (dma_omap1())
 808                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 809 
 810         return addr;
 811 }
 812 
 813 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 814         dma_cookie_t cookie, struct dma_tx_state *txstate)
 815 {
 816         struct omap_chan *c = to_omap_dma_chan(chan);
 817         enum dma_status ret;
 818         unsigned long flags;
 819         struct omap_desc *d = NULL;
 820 
 821         ret = dma_cookie_status(chan, cookie, txstate);
 822         if (ret == DMA_COMPLETE)
 823                 return ret;
 824 
 825         spin_lock_irqsave(&c->vc.lock, flags);
 826         if (c->desc && c->desc->vd.tx.cookie == cookie)
 827                 d = c->desc;
 828 
 829         if (!txstate)
 830                 goto out;
 831 
 832         if (d) {
 833                 dma_addr_t pos;
 834 
 835                 if (d->dir == DMA_MEM_TO_DEV)
 836                         pos = omap_dma_get_src_pos(c);
 837                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 838                         pos = omap_dma_get_dst_pos(c);
 839                 else
 840                         pos = 0;
 841 
 842                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 843         } else {
 844                 struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie);
 845 
 846                 if (vd)
 847                         txstate->residue = omap_dma_desc_size(
 848                                                 to_omap_dma_desc(&vd->tx));
 849                 else
 850                         txstate->residue = 0;
 851         }
 852 
 853 out:
 854         if (ret == DMA_IN_PROGRESS && c->paused) {
 855                 ret = DMA_PAUSED;
 856         } else if (d && d->polled && c->running) {
 857                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 858                 /*
 859                  * The channel is no longer active, set the return value
 860                  * accordingly and mark it as completed
 861                  */
 862                 if (!(ccr & CCR_ENABLE)) {
 863                         ret = DMA_COMPLETE;
 864                         omap_dma_start_desc(c);
 865                         vchan_cookie_complete(&d->vd);
 866                 }
 867         }
 868 
 869         spin_unlock_irqrestore(&c->vc.lock, flags);
 870 
 871         return ret;
 872 }
 873 
 874 static void omap_dma_issue_pending(struct dma_chan *chan)
 875 {
 876         struct omap_chan *c = to_omap_dma_chan(chan);
 877         unsigned long flags;
 878 
 879         spin_lock_irqsave(&c->vc.lock, flags);
 880         if (vchan_issue_pending(&c->vc) && !c->desc)
 881                 omap_dma_start_desc(c);
 882         spin_unlock_irqrestore(&c->vc.lock, flags);
 883 }
 884 
 885 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 886         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 887         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 888 {
 889         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 890         struct omap_chan *c = to_omap_dma_chan(chan);
 891         enum dma_slave_buswidth dev_width;
 892         struct scatterlist *sgent;
 893         struct omap_desc *d;
 894         dma_addr_t dev_addr;
 895         unsigned i, es, en, frame_bytes;
 896         bool ll_failed = false;
 897         u32 burst;
 898         u32 port_window, port_window_bytes;
 899 
 900         if (dir == DMA_DEV_TO_MEM) {
 901                 dev_addr = c->cfg.src_addr;
 902                 dev_width = c->cfg.src_addr_width;
 903                 burst = c->cfg.src_maxburst;
 904                 port_window = c->cfg.src_port_window_size;
 905         } else if (dir == DMA_MEM_TO_DEV) {
 906                 dev_addr = c->cfg.dst_addr;
 907                 dev_width = c->cfg.dst_addr_width;
 908                 burst = c->cfg.dst_maxburst;
 909                 port_window = c->cfg.dst_port_window_size;
 910         } else {
 911                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 912                 return NULL;
 913         }
 914 
 915         /* Bus width translates to the element size (ES) */
 916         switch (dev_width) {
 917         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 918                 es = CSDP_DATA_TYPE_8;
 919                 break;
 920         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 921                 es = CSDP_DATA_TYPE_16;
 922                 break;
 923         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 924                 es = CSDP_DATA_TYPE_32;
 925                 break;
 926         default: /* not reached */
 927                 return NULL;
 928         }
 929 
 930         /* Now allocate and setup the descriptor. */
 931         d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC);
 932         if (!d)
 933                 return NULL;
 934 
 935         d->dir = dir;
 936         d->dev_addr = dev_addr;
 937         d->es = es;
 938 
 939         /* When the port_window is used, one frame must cover the window */
 940         if (port_window) {
 941                 burst = port_window;
 942                 port_window_bytes = port_window * es_bytes[es];
 943 
 944                 d->ei = 1;
 945                 /*
 946                  * One frame covers the port_window and by  configure
 947                  * the source frame index to be -1 * (port_window - 1)
 948                  * we instruct the sDMA that after a frame is processed
 949                  * it should move back to the start of the window.
 950                  */
 951                 d->fi = -(port_window_bytes - 1);
 952         }
 953 
 954         d->ccr = c->ccr | CCR_SYNC_FRAME;
 955         if (dir == DMA_DEV_TO_MEM) {
 956                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
 957 
 958                 d->ccr |= CCR_DST_AMODE_POSTINC;
 959                 if (port_window) {
 960                         d->ccr |= CCR_SRC_AMODE_DBLIDX;
 961 
 962                         if (port_window_bytes >= 64)
 963                                 d->csdp |= CSDP_SRC_BURST_64;
 964                         else if (port_window_bytes >= 32)
 965                                 d->csdp |= CSDP_SRC_BURST_32;
 966                         else if (port_window_bytes >= 16)
 967                                 d->csdp |= CSDP_SRC_BURST_16;
 968 
 969                 } else {
 970                         d->ccr |= CCR_SRC_AMODE_CONSTANT;
 971                 }
 972         } else {
 973                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
 974 
 975                 d->ccr |= CCR_SRC_AMODE_POSTINC;
 976                 if (port_window) {
 977                         d->ccr |= CCR_DST_AMODE_DBLIDX;
 978 
 979                         if (port_window_bytes >= 64)
 980                                 d->csdp |= CSDP_DST_BURST_64;
 981                         else if (port_window_bytes >= 32)
 982                                 d->csdp |= CSDP_DST_BURST_32;
 983                         else if (port_window_bytes >= 16)
 984                                 d->csdp |= CSDP_DST_BURST_16;
 985                 } else {
 986                         d->ccr |= CCR_DST_AMODE_CONSTANT;
 987                 }
 988         }
 989 
 990         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 991         d->csdp |= es;
 992 
 993         if (dma_omap1()) {
 994                 d->cicr |= CICR_TOUT_IE;
 995 
 996                 if (dir == DMA_DEV_TO_MEM)
 997                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 998                 else
 999                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
1000         } else {
1001                 if (dir == DMA_DEV_TO_MEM)
1002                         d->ccr |= CCR_TRIGGER_SRC;
1003 
1004                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1005 
1006                 if (port_window)
1007                         d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
1008         }
1009         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
1010                 d->clnk_ctrl = c->dma_ch;
1011 
1012         /*
1013          * Build our scatterlist entries: each contains the address,
1014          * the number of elements (EN) in each frame, and the number of
1015          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1016          *
1017          * Burst size translates to number of elements with frame sync.
1018          * Note: DMA engine defines burst to be the number of dev-width
1019          * transfers.
1020          */
1021         en = burst;
1022         frame_bytes = es_bytes[es] * en;
1023 
1024         if (sglen >= 2)
1025                 d->using_ll = od->ll123_supported;
1026 
1027         for_each_sg(sgl, sgent, sglen, i) {
1028                 struct omap_sg *osg = &d->sg[i];
1029 
1030                 osg->addr = sg_dma_address(sgent);
1031                 osg->en = en;
1032                 osg->fn = sg_dma_len(sgent) / frame_bytes;
1033 
1034                 if (d->using_ll) {
1035                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1036                                                       &osg->t2_desc_paddr);
1037                         if (!osg->t2_desc) {
1038                                 dev_err(chan->device->dev,
1039                                         "t2_desc[%d] allocation failed\n", i);
1040                                 ll_failed = true;
1041                                 d->using_ll = false;
1042                                 continue;
1043                         }
1044 
1045                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1046                 }
1047         }
1048 
1049         d->sglen = sglen;
1050 
1051         /* Release the dma_pool entries if one allocation failed */
1052         if (ll_failed) {
1053                 for (i = 0; i < d->sglen; i++) {
1054                         struct omap_sg *osg = &d->sg[i];
1055 
1056                         if (osg->t2_desc) {
1057                                 dma_pool_free(od->desc_pool, osg->t2_desc,
1058                                               osg->t2_desc_paddr);
1059                                 osg->t2_desc = NULL;
1060                         }
1061                 }
1062         }
1063 
1064         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1065 }
1066 
1067 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1068         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1069         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1070 {
1071         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1072         struct omap_chan *c = to_omap_dma_chan(chan);
1073         enum dma_slave_buswidth dev_width;
1074         struct omap_desc *d;
1075         dma_addr_t dev_addr;
1076         unsigned es;
1077         u32 burst;
1078 
1079         if (dir == DMA_DEV_TO_MEM) {
1080                 dev_addr = c->cfg.src_addr;
1081                 dev_width = c->cfg.src_addr_width;
1082                 burst = c->cfg.src_maxburst;
1083         } else if (dir == DMA_MEM_TO_DEV) {
1084                 dev_addr = c->cfg.dst_addr;
1085                 dev_width = c->cfg.dst_addr_width;
1086                 burst = c->cfg.dst_maxburst;
1087         } else {
1088                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1089                 return NULL;
1090         }
1091 
1092         /* Bus width translates to the element size (ES) */
1093         switch (dev_width) {
1094         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1095                 es = CSDP_DATA_TYPE_8;
1096                 break;
1097         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1098                 es = CSDP_DATA_TYPE_16;
1099                 break;
1100         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1101                 es = CSDP_DATA_TYPE_32;
1102                 break;
1103         default: /* not reached */
1104                 return NULL;
1105         }
1106 
1107         /* Now allocate and setup the descriptor. */
1108         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1109         if (!d)
1110                 return NULL;
1111 
1112         d->dir = dir;
1113         d->dev_addr = dev_addr;
1114         d->fi = burst;
1115         d->es = es;
1116         d->sg[0].addr = buf_addr;
1117         d->sg[0].en = period_len / es_bytes[es];
1118         d->sg[0].fn = buf_len / period_len;
1119         d->sglen = 1;
1120 
1121         d->ccr = c->ccr;
1122         if (dir == DMA_DEV_TO_MEM)
1123                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1124         else
1125                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1126 
1127         d->cicr = CICR_DROP_IE;
1128         if (flags & DMA_PREP_INTERRUPT)
1129                 d->cicr |= CICR_FRAME_IE;
1130 
1131         d->csdp = es;
1132 
1133         if (dma_omap1()) {
1134                 d->cicr |= CICR_TOUT_IE;
1135 
1136                 if (dir == DMA_DEV_TO_MEM)
1137                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1138                 else
1139                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1140         } else {
1141                 if (burst)
1142                         d->ccr |= CCR_SYNC_PACKET;
1143                 else
1144                         d->ccr |= CCR_SYNC_ELEMENT;
1145 
1146                 if (dir == DMA_DEV_TO_MEM) {
1147                         d->ccr |= CCR_TRIGGER_SRC;
1148                         d->csdp |= CSDP_DST_PACKED;
1149                 } else {
1150                         d->csdp |= CSDP_SRC_PACKED;
1151                 }
1152 
1153                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1154 
1155                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1156         }
1157 
1158         if (__dma_omap15xx(od->plat->dma_attr))
1159                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1160         else
1161                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1162 
1163         c->cyclic = true;
1164 
1165         return vchan_tx_prep(&c->vc, &d->vd, flags);
1166 }
1167 
1168 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1169         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1170         size_t len, unsigned long tx_flags)
1171 {
1172         struct omap_chan *c = to_omap_dma_chan(chan);
1173         struct omap_desc *d;
1174         uint8_t data_type;
1175 
1176         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1177         if (!d)
1178                 return NULL;
1179 
1180         data_type = __ffs((src | dest | len));
1181         if (data_type > CSDP_DATA_TYPE_32)
1182                 data_type = CSDP_DATA_TYPE_32;
1183 
1184         d->dir = DMA_MEM_TO_MEM;
1185         d->dev_addr = src;
1186         d->fi = 0;
1187         d->es = data_type;
1188         d->sg[0].en = len / BIT(data_type);
1189         d->sg[0].fn = 1;
1190         d->sg[0].addr = dest;
1191         d->sglen = 1;
1192         d->ccr = c->ccr;
1193         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1194 
1195         if (tx_flags & DMA_PREP_INTERRUPT)
1196                 d->cicr |= CICR_FRAME_IE;
1197         else
1198                 d->polled = true;
1199 
1200         d->csdp = data_type;
1201 
1202         if (dma_omap1()) {
1203                 d->cicr |= CICR_TOUT_IE;
1204                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1205         } else {
1206                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1207                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1208                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1209         }
1210 
1211         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1212 }
1213 
1214 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1215         struct dma_chan *chan, struct dma_interleaved_template *xt,
1216         unsigned long flags)
1217 {
1218         struct omap_chan *c = to_omap_dma_chan(chan);
1219         struct omap_desc *d;
1220         struct omap_sg *sg;
1221         uint8_t data_type;
1222         size_t src_icg, dst_icg;
1223 
1224         /* Slave mode is not supported */
1225         if (is_slave_direction(xt->dir))
1226                 return NULL;
1227 
1228         if (xt->frame_size != 1 || xt->numf == 0)
1229                 return NULL;
1230 
1231         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1232         if (!d)
1233                 return NULL;
1234 
1235         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1236         if (data_type > CSDP_DATA_TYPE_32)
1237                 data_type = CSDP_DATA_TYPE_32;
1238 
1239         sg = &d->sg[0];
1240         d->dir = DMA_MEM_TO_MEM;
1241         d->dev_addr = xt->src_start;
1242         d->es = data_type;
1243         sg->en = xt->sgl[0].size / BIT(data_type);
1244         sg->fn = xt->numf;
1245         sg->addr = xt->dst_start;
1246         d->sglen = 1;
1247         d->ccr = c->ccr;
1248 
1249         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1250         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1251         if (src_icg) {
1252                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1253                 d->ei = 1;
1254                 d->fi = src_icg + 1;
1255         } else if (xt->src_inc) {
1256                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1257                 d->fi = 0;
1258         } else {
1259                 dev_err(chan->device->dev,
1260                         "%s: SRC constant addressing is not supported\n",
1261                         __func__);
1262                 kfree(d);
1263                 return NULL;
1264         }
1265 
1266         if (dst_icg) {
1267                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1268                 sg->ei = 1;
1269                 sg->fi = dst_icg + 1;
1270         } else if (xt->dst_inc) {
1271                 d->ccr |= CCR_DST_AMODE_POSTINC;
1272                 sg->fi = 0;
1273         } else {
1274                 dev_err(chan->device->dev,
1275                         "%s: DST constant addressing is not supported\n",
1276                         __func__);
1277                 kfree(d);
1278                 return NULL;
1279         }
1280 
1281         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1282 
1283         d->csdp = data_type;
1284 
1285         if (dma_omap1()) {
1286                 d->cicr |= CICR_TOUT_IE;
1287                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1288         } else {
1289                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1290                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1291                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1292         }
1293 
1294         return vchan_tx_prep(&c->vc, &d->vd, flags);
1295 }
1296 
1297 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1298 {
1299         struct omap_chan *c = to_omap_dma_chan(chan);
1300 
1301         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1302             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1303                 return -EINVAL;
1304 
1305         if (cfg->src_maxburst > chan->device->max_burst ||
1306             cfg->dst_maxburst > chan->device->max_burst)
1307                 return -EINVAL;
1308 
1309         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1310 
1311         return 0;
1312 }
1313 
1314 static int omap_dma_terminate_all(struct dma_chan *chan)
1315 {
1316         struct omap_chan *c = to_omap_dma_chan(chan);
1317         unsigned long flags;
1318         LIST_HEAD(head);
1319 
1320         spin_lock_irqsave(&c->vc.lock, flags);
1321 
1322         /*
1323          * Stop DMA activity: we assume the callback will not be called
1324          * after omap_dma_stop() returns (even if it does, it will see
1325          * c->desc is NULL and exit.)
1326          */
1327         if (c->desc) {
1328                 vchan_terminate_vdesc(&c->desc->vd);
1329                 c->desc = NULL;
1330                 /* Avoid stopping the dma twice */
1331                 if (!c->paused)
1332                         omap_dma_stop(c);
1333         }
1334 
1335         c->cyclic = false;
1336         c->paused = false;
1337 
1338         vchan_get_all_descriptors(&c->vc, &head);
1339         spin_unlock_irqrestore(&c->vc.lock, flags);
1340         vchan_dma_desc_free_list(&c->vc, &head);
1341 
1342         return 0;
1343 }
1344 
1345 static void omap_dma_synchronize(struct dma_chan *chan)
1346 {
1347         struct omap_chan *c = to_omap_dma_chan(chan);
1348 
1349         vchan_synchronize(&c->vc);
1350 }
1351 
1352 static int omap_dma_pause(struct dma_chan *chan)
1353 {
1354         struct omap_chan *c = to_omap_dma_chan(chan);
1355         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1356         unsigned long flags;
1357         int ret = -EINVAL;
1358         bool can_pause = false;
1359 
1360         spin_lock_irqsave(&od->irq_lock, flags);
1361 
1362         if (!c->desc)
1363                 goto out;
1364 
1365         if (c->cyclic)
1366                 can_pause = true;
1367 
1368         /*
1369          * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1370          * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1371          * "When a channel is disabled during a transfer, the channel undergoes
1372          * an abort, unless it is hardware-source-synchronized …".
1373          * A source-synchronised channel is one where the fetching of data is
1374          * under control of the device. In other words, a device-to-memory
1375          * transfer. So, a destination-synchronised channel (which would be a
1376          * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1377          * bit is cleared.
1378          * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1379          * aborts immediately after completion of current read/write
1380          * transactions and then the FIFO is cleaned up." The term "cleaned up"
1381          * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1382          * are both clear _before_ disabling the channel, otherwise data loss
1383          * will occur.
1384          * The problem is that if the channel is active, then device activity
1385          * can result in DMA activity starting between reading those as both
1386          * clear and the write to DMA_CCR to clear the enable bit hitting the
1387          * hardware. If the DMA hardware can't drain the data in its FIFO to the
1388          * destination, then data loss "might" occur (say if we write to an UART
1389          * and the UART is not accepting any further data).
1390          */
1391         else if (c->desc->dir == DMA_DEV_TO_MEM)
1392                 can_pause = true;
1393 
1394         if (can_pause && !c->paused) {
1395                 ret = omap_dma_stop(c);
1396                 if (!ret)
1397                         c->paused = true;
1398         }
1399 out:
1400         spin_unlock_irqrestore(&od->irq_lock, flags);
1401 
1402         return ret;
1403 }
1404 
1405 static int omap_dma_resume(struct dma_chan *chan)
1406 {
1407         struct omap_chan *c = to_omap_dma_chan(chan);
1408         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1409         unsigned long flags;
1410         int ret = -EINVAL;
1411 
1412         spin_lock_irqsave(&od->irq_lock, flags);
1413 
1414         if (c->paused && c->desc) {
1415                 mb();
1416 
1417                 /* Restore channel link register */
1418                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1419 
1420                 omap_dma_start(c, c->desc);
1421                 c->paused = false;
1422                 ret = 0;
1423         }
1424         spin_unlock_irqrestore(&od->irq_lock, flags);
1425 
1426         return ret;
1427 }
1428 
1429 static int omap_dma_chan_init(struct omap_dmadev *od)
1430 {
1431         struct omap_chan *c;
1432 
1433         c = kzalloc(sizeof(*c), GFP_KERNEL);
1434         if (!c)
1435                 return -ENOMEM;
1436 
1437         c->reg_map = od->reg_map;
1438         c->vc.desc_free = omap_dma_desc_free;
1439         vchan_init(&c->vc, &od->ddev);
1440 
1441         return 0;
1442 }
1443 
1444 static void omap_dma_free(struct omap_dmadev *od)
1445 {
1446         while (!list_empty(&od->ddev.channels)) {
1447                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1448                         struct omap_chan, vc.chan.device_node);
1449 
1450                 list_del(&c->vc.chan.device_node);
1451                 tasklet_kill(&c->vc.task);
1452                 kfree(c);
1453         }
1454 }
1455 
1456 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1457                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1458                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1459 
1460 static int omap_dma_probe(struct platform_device *pdev)
1461 {
1462         struct omap_dmadev *od;
1463         struct resource *res;
1464         int rc, i, irq;
1465         u32 lch_count;
1466 
1467         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1468         if (!od)
1469                 return -ENOMEM;
1470 
1471         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1472         od->base = devm_ioremap_resource(&pdev->dev, res);
1473         if (IS_ERR(od->base))
1474                 return PTR_ERR(od->base);
1475 
1476         od->plat = omap_get_plat_info();
1477         if (!od->plat)
1478                 return -EPROBE_DEFER;
1479 
1480         od->reg_map = od->plat->reg_map;
1481 
1482         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1483         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1484         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1485         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1486         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1487         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1488         od->ddev.device_tx_status = omap_dma_tx_status;
1489         od->ddev.device_issue_pending = omap_dma_issue_pending;
1490         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1491         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1492         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1493         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1494         od->ddev.device_config = omap_dma_slave_config;
1495         od->ddev.device_pause = omap_dma_pause;
1496         od->ddev.device_resume = omap_dma_resume;
1497         od->ddev.device_terminate_all = omap_dma_terminate_all;
1498         od->ddev.device_synchronize = omap_dma_synchronize;
1499         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1500         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1501         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1502         if (__dma_omap15xx(od->plat->dma_attr))
1503                 od->ddev.residue_granularity =
1504                                 DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1505         else
1506                 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1507         od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
1508         od->ddev.dev = &pdev->dev;
1509         INIT_LIST_HEAD(&od->ddev.channels);
1510         spin_lock_init(&od->lock);
1511         spin_lock_init(&od->irq_lock);
1512 
1513         /* Number of DMA requests */
1514         od->dma_requests = OMAP_SDMA_REQUESTS;
1515         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1516                                                       "dma-requests",
1517                                                       &od->dma_requests)) {
1518                 dev_info(&pdev->dev,
1519                          "Missing dma-requests property, using %u.\n",
1520                          OMAP_SDMA_REQUESTS);
1521         }
1522 
1523         /* Number of available logical channels */
1524         if (!pdev->dev.of_node) {
1525                 lch_count = od->plat->dma_attr->lch_count;
1526                 if (unlikely(!lch_count))
1527                         lch_count = OMAP_SDMA_CHANNELS;
1528         } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1529                                         &lch_count)) {
1530                 dev_info(&pdev->dev,
1531                          "Missing dma-channels property, using %u.\n",
1532                          OMAP_SDMA_CHANNELS);
1533                 lch_count = OMAP_SDMA_CHANNELS;
1534         }
1535 
1536         od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
1537                                    GFP_KERNEL);
1538         if (!od->lch_map)
1539                 return -ENOMEM;
1540 
1541         for (i = 0; i < od->dma_requests; i++) {
1542                 rc = omap_dma_chan_init(od);
1543                 if (rc) {
1544                         omap_dma_free(od);
1545                         return rc;
1546                 }
1547         }
1548 
1549         irq = platform_get_irq(pdev, 1);
1550         if (irq <= 0) {
1551                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1552                 od->legacy = true;
1553         } else {
1554                 /* Disable all interrupts */
1555                 od->irq_enable_mask = 0;
1556                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1557 
1558                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1559                                       IRQF_SHARED, "omap-dma-engine", od);
1560                 if (rc) {
1561                         omap_dma_free(od);
1562                         return rc;
1563                 }
1564         }
1565 
1566         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1567                 od->ll123_supported = true;
1568 
1569         od->ddev.filter.map = od->plat->slave_map;
1570         od->ddev.filter.mapcnt = od->plat->slavecnt;
1571         od->ddev.filter.fn = omap_dma_filter_fn;
1572 
1573         if (od->ll123_supported) {
1574                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1575                                                 &pdev->dev,
1576                                                 sizeof(struct omap_type2_desc),
1577                                                 4, 0);
1578                 if (!od->desc_pool) {
1579                         dev_err(&pdev->dev,
1580                                 "unable to allocate descriptor pool\n");
1581                         od->ll123_supported = false;
1582                 }
1583         }
1584 
1585         rc = dma_async_device_register(&od->ddev);
1586         if (rc) {
1587                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1588                         rc);
1589                 omap_dma_free(od);
1590                 return rc;
1591         }
1592 
1593         platform_set_drvdata(pdev, od);
1594 
1595         if (pdev->dev.of_node) {
1596                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1597 
1598                 /* Device-tree DMA controller registration */
1599                 rc = of_dma_controller_register(pdev->dev.of_node,
1600                                 of_dma_simple_xlate, &omap_dma_info);
1601                 if (rc) {
1602                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1603                         dma_async_device_unregister(&od->ddev);
1604                         omap_dma_free(od);
1605                 }
1606         }
1607 
1608         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1609                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1610 
1611         return rc;
1612 }
1613 
1614 static int omap_dma_remove(struct platform_device *pdev)
1615 {
1616         struct omap_dmadev *od = platform_get_drvdata(pdev);
1617         int irq;
1618 
1619         if (pdev->dev.of_node)
1620                 of_dma_controller_free(pdev->dev.of_node);
1621 
1622         irq = platform_get_irq(pdev, 1);
1623         devm_free_irq(&pdev->dev, irq, od);
1624 
1625         dma_async_device_unregister(&od->ddev);
1626 
1627         if (!od->legacy) {
1628                 /* Disable all interrupts */
1629                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1630         }
1631 
1632         if (od->ll123_supported)
1633                 dma_pool_destroy(od->desc_pool);
1634 
1635         omap_dma_free(od);
1636 
1637         return 0;
1638 }
1639 
1640 static const struct of_device_id omap_dma_match[] = {
1641         { .compatible = "ti,omap2420-sdma", },
1642         { .compatible = "ti,omap2430-sdma", },
1643         { .compatible = "ti,omap3430-sdma", },
1644         { .compatible = "ti,omap3630-sdma", },
1645         { .compatible = "ti,omap4430-sdma", },
1646         {},
1647 };
1648 MODULE_DEVICE_TABLE(of, omap_dma_match);
1649 
1650 static struct platform_driver omap_dma_driver = {
1651         .probe  = omap_dma_probe,
1652         .remove = omap_dma_remove,
1653         .driver = {
1654                 .name = "omap-dma-engine",
1655                 .of_match_table = of_match_ptr(omap_dma_match),
1656         },
1657 };
1658 
1659 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1660 {
1661         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1662                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1663                 struct omap_chan *c = to_omap_dma_chan(chan);
1664                 unsigned req = *(unsigned *)param;
1665 
1666                 if (req <= od->dma_requests) {
1667                         c->dma_sig = req;
1668                         return true;
1669                 }
1670         }
1671         return false;
1672 }
1673 
1674 static int omap_dma_init(void)
1675 {
1676         return platform_driver_register(&omap_dma_driver);
1677 }
1678 subsys_initcall(omap_dma_init);
1679 
1680 static void __exit omap_dma_exit(void)
1681 {
1682         platform_driver_unregister(&omap_dma_driver);
1683 }
1684 module_exit(omap_dma_exit);
1685 
1686 MODULE_AUTHOR("Russell King");
1687 MODULE_LICENSE("GPL");

/* [<][>][^][v][top][bottom][index][help] */