1/* 2 * Copyright (c) 2006 ARM Ltd. 3 * Copyright (c) 2010 ST-Ericsson SA 4 * 5 * Author: Peter Pearse <peter.pearse@arm.com> 6 * Author: Linus Walleij <linus.walleij@stericsson.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the Free 10 * Software Foundation; either version 2 of the License, or (at your option) 11 * any later version. 12 * 13 * This program is distributed in the hope that it will be useful, but WITHOUT 14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 16 * more details. 17 * 18 * The full GNU General Public License is in this distribution in the file 19 * called COPYING. 20 * 21 * Documentation: ARM DDI 0196G == PL080 22 * Documentation: ARM DDI 0218E == PL081 23 * Documentation: S3C6410 User's Manual == PL080S 24 * 25 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any 26 * channel. 27 * 28 * The PL080 has 8 channels available for simultaneous use, and the PL081 29 * has only two channels. So on these DMA controllers the number of channels 30 * and the number of incoming DMA signals are two totally different things. 31 * It is usually not possible to theoretically handle all physical signals, 32 * so a multiplexing scheme with possible denial of use is necessary. 33 * 34 * The PL080 has a dual bus master, PL081 has a single master. 35 * 36 * PL080S is a version modified by Samsung and used in S3C64xx SoCs. 37 * It differs in following aspects: 38 * - CH_CONFIG register at different offset, 39 * - separate CH_CONTROL2 register for transfer size, 40 * - bigger maximum transfer size, 41 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, 42 * - no support for peripheral flow control. 43 * 44 * Memory to peripheral transfer may be visualized as 45 * Get data from memory to DMAC 46 * Until no data left 47 * On burst request from peripheral 48 * Destination burst from DMAC to peripheral 49 * Clear burst request 50 * Raise terminal count interrupt 51 * 52 * For peripherals with a FIFO: 53 * Source burst size == half the depth of the peripheral FIFO 54 * Destination burst size == the depth of the peripheral FIFO 55 * 56 * (Bursts are irrelevant for mem to mem transfers - there are no burst 57 * signals, the DMA controller will simply facilitate its AHB master.) 58 * 59 * ASSUMES default (little) endianness for DMA transfers 60 * 61 * The PL08x has two flow control settings: 62 * - DMAC flow control: the transfer size defines the number of transfers 63 * which occur for the current LLI entry, and the DMAC raises TC at the 64 * end of every LLI entry. Observed behaviour shows the DMAC listening 65 * to both the BREQ and SREQ signals (contrary to documented), 66 * transferring data if either is active. The LBREQ and LSREQ signals 67 * are ignored. 68 * 69 * - Peripheral flow control: the transfer size is ignored (and should be 70 * zero). The data is transferred from the current LLI entry, until 71 * after the final transfer signalled by LBREQ or LSREQ. The DMAC 72 * will then move to the next LLI entry. Unsupported by PL080S. 73 */ 74#include <linux/amba/bus.h> 75#include <linux/amba/pl08x.h> 76#include <linux/debugfs.h> 77#include <linux/delay.h> 78#include <linux/device.h> 79#include <linux/dmaengine.h> 80#include <linux/dmapool.h> 81#include <linux/dma-mapping.h> 82#include <linux/export.h> 83#include <linux/init.h> 84#include <linux/interrupt.h> 85#include <linux/module.h> 86#include <linux/of.h> 87#include <linux/of_dma.h> 88#include <linux/pm_runtime.h> 89#include <linux/seq_file.h> 90#include <linux/slab.h> 91#include <linux/amba/pl080.h> 92 93#include "dmaengine.h" 94#include "virt-dma.h" 95 96#define DRIVER_NAME "pl08xdmac" 97 98#define PL80X_DMA_BUSWIDTHS \ 99 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 100 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 101 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 102 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 103 104static struct amba_driver pl08x_amba_driver; 105struct pl08x_driver_data; 106 107/** 108 * struct vendor_data - vendor-specific config parameters for PL08x derivatives 109 * @channels: the number of channels available in this variant 110 * @dualmaster: whether this version supports dual AHB masters or not. 111 * @nomadik: whether the channels have Nomadik security extension bits 112 * that need to be checked for permission before use and some registers are 113 * missing 114 * @pl080s: whether this version is a PL080S, which has separate register and 115 * LLI word for transfer size. 116 */ 117struct vendor_data { 118 u8 config_offset; 119 u8 channels; 120 bool dualmaster; 121 bool nomadik; 122 bool pl080s; 123 u32 max_transfer_size; 124}; 125 126/** 127 * struct pl08x_bus_data - information of source or destination 128 * busses for a transfer 129 * @addr: current address 130 * @maxwidth: the maximum width of a transfer on this bus 131 * @buswidth: the width of this bus in bytes: 1, 2 or 4 132 */ 133struct pl08x_bus_data { 134 dma_addr_t addr; 135 u8 maxwidth; 136 u8 buswidth; 137}; 138 139#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) 140 141/** 142 * struct pl08x_phy_chan - holder for the physical channels 143 * @id: physical index to this channel 144 * @lock: a lock to use when altering an instance of this struct 145 * @serving: the virtual channel currently being served by this physical 146 * channel 147 * @locked: channel unavailable for the system, e.g. dedicated to secure 148 * world 149 */ 150struct pl08x_phy_chan { 151 unsigned int id; 152 void __iomem *base; 153 void __iomem *reg_config; 154 spinlock_t lock; 155 struct pl08x_dma_chan *serving; 156 bool locked; 157}; 158 159/** 160 * struct pl08x_sg - structure containing data per sg 161 * @src_addr: src address of sg 162 * @dst_addr: dst address of sg 163 * @len: transfer len in bytes 164 * @node: node for txd's dsg_list 165 */ 166struct pl08x_sg { 167 dma_addr_t src_addr; 168 dma_addr_t dst_addr; 169 size_t len; 170 struct list_head node; 171}; 172 173/** 174 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor 175 * @vd: virtual DMA descriptor 176 * @dsg_list: list of children sg's 177 * @llis_bus: DMA memory address (physical) start for the LLIs 178 * @llis_va: virtual memory address start for the LLIs 179 * @cctl: control reg values for current txd 180 * @ccfg: config reg values for current txd 181 * @done: this marks completed descriptors, which should not have their 182 * mux released. 183 * @cyclic: indicate cyclic transfers 184 */ 185struct pl08x_txd { 186 struct virt_dma_desc vd; 187 struct list_head dsg_list; 188 dma_addr_t llis_bus; 189 u32 *llis_va; 190 /* Default cctl value for LLIs */ 191 u32 cctl; 192 /* 193 * Settings to be put into the physical channel when we 194 * trigger this txd. Other registers are in llis_va[0]. 195 */ 196 u32 ccfg; 197 bool done; 198 bool cyclic; 199}; 200 201/** 202 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel 203 * states 204 * @PL08X_CHAN_IDLE: the channel is idle 205 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport 206 * channel and is running a transfer on it 207 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport 208 * channel, but the transfer is currently paused 209 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport 210 * channel to become available (only pertains to memcpy channels) 211 */ 212enum pl08x_dma_chan_state { 213 PL08X_CHAN_IDLE, 214 PL08X_CHAN_RUNNING, 215 PL08X_CHAN_PAUSED, 216 PL08X_CHAN_WAITING, 217}; 218 219/** 220 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 221 * @vc: wrappped virtual channel 222 * @phychan: the physical channel utilized by this channel, if there is one 223 * @name: name of channel 224 * @cd: channel platform data 225 * @runtime_addr: address for RX/TX according to the runtime config 226 * @at: active transaction on this channel 227 * @lock: a lock for this channel data 228 * @host: a pointer to the host (internal use) 229 * @state: whether the channel is idle, paused, running etc 230 * @slave: whether this channel is a device (slave) or for memcpy 231 * @signal: the physical DMA request signal which this channel is using 232 * @mux_use: count of descriptors using this DMA request signal setting 233 */ 234struct pl08x_dma_chan { 235 struct virt_dma_chan vc; 236 struct pl08x_phy_chan *phychan; 237 const char *name; 238 const struct pl08x_channel_data *cd; 239 struct dma_slave_config cfg; 240 struct pl08x_txd *at; 241 struct pl08x_driver_data *host; 242 enum pl08x_dma_chan_state state; 243 bool slave; 244 int signal; 245 unsigned mux_use; 246}; 247 248/** 249 * struct pl08x_driver_data - the local state holder for the PL08x 250 * @slave: slave engine for this instance 251 * @memcpy: memcpy engine for this instance 252 * @base: virtual memory base (remapped) for the PL08x 253 * @adev: the corresponding AMBA (PrimeCell) bus entry 254 * @vd: vendor data for this PL08x variant 255 * @pd: platform data passed in from the platform/machine 256 * @phy_chans: array of data for the physical channels 257 * @pool: a pool for the LLI descriptors 258 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI 259 * fetches 260 * @mem_buses: set to indicate memory transfers on AHB2. 261 * @lock: a spinlock for this struct 262 */ 263struct pl08x_driver_data { 264 struct dma_device slave; 265 struct dma_device memcpy; 266 void __iomem *base; 267 struct amba_device *adev; 268 const struct vendor_data *vd; 269 struct pl08x_platform_data *pd; 270 struct pl08x_phy_chan *phy_chans; 271 struct dma_pool *pool; 272 u8 lli_buses; 273 u8 mem_buses; 274 u8 lli_words; 275}; 276 277/* 278 * PL08X specific defines 279 */ 280 281/* The order of words in an LLI. */ 282#define PL080_LLI_SRC 0 283#define PL080_LLI_DST 1 284#define PL080_LLI_LLI 2 285#define PL080_LLI_CCTL 3 286#define PL080S_LLI_CCTL2 4 287 288/* Total words in an LLI. */ 289#define PL080_LLI_WORDS 4 290#define PL080S_LLI_WORDS 8 291 292/* 293 * Number of LLIs in each LLI buffer allocated for one transfer 294 * (maximum times we call dma_pool_alloc on this pool without freeing) 295 */ 296#define MAX_NUM_TSFR_LLIS 512 297#define PL08X_ALIGN 8 298 299static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 300{ 301 return container_of(chan, struct pl08x_dma_chan, vc.chan); 302} 303 304static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) 305{ 306 return container_of(tx, struct pl08x_txd, vd.tx); 307} 308 309/* 310 * Mux handling. 311 * 312 * This gives us the DMA request input to the PL08x primecell which the 313 * peripheral described by the channel data will be routed to, possibly 314 * via a board/SoC specific external MUX. One important point to note 315 * here is that this does not depend on the physical channel. 316 */ 317static int pl08x_request_mux(struct pl08x_dma_chan *plchan) 318{ 319 const struct pl08x_platform_data *pd = plchan->host->pd; 320 int ret; 321 322 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { 323 ret = pd->get_xfer_signal(plchan->cd); 324 if (ret < 0) { 325 plchan->mux_use = 0; 326 return ret; 327 } 328 329 plchan->signal = ret; 330 } 331 return 0; 332} 333 334static void pl08x_release_mux(struct pl08x_dma_chan *plchan) 335{ 336 const struct pl08x_platform_data *pd = plchan->host->pd; 337 338 if (plchan->signal >= 0) { 339 WARN_ON(plchan->mux_use == 0); 340 341 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { 342 pd->put_xfer_signal(plchan->cd, plchan->signal); 343 plchan->signal = -1; 344 } 345 } 346} 347 348/* 349 * Physical channel handling 350 */ 351 352/* Whether a certain channel is busy or not */ 353static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) 354{ 355 unsigned int val; 356 357 val = readl(ch->reg_config); 358 return val & PL080_CONFIG_ACTIVE; 359} 360 361static void pl08x_write_lli(struct pl08x_driver_data *pl08x, 362 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) 363{ 364 if (pl08x->vd->pl080s) 365 dev_vdbg(&pl08x->adev->dev, 366 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 367 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", 368 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 369 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], 370 lli[PL080S_LLI_CCTL2], ccfg); 371 else 372 dev_vdbg(&pl08x->adev->dev, 373 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 374 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", 375 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 376 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); 377 378 writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR); 379 writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR); 380 writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI); 381 writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL); 382 383 if (pl08x->vd->pl080s) 384 writel_relaxed(lli[PL080S_LLI_CCTL2], 385 phychan->base + PL080S_CH_CONTROL2); 386 387 writel(ccfg, phychan->reg_config); 388} 389 390/* 391 * Set the initial DMA register values i.e. those for the first LLI 392 * The next LLI pointer and the configuration interrupt bit have 393 * been set when the LLIs were constructed. Poke them into the hardware 394 * and start the transfer. 395 */ 396static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) 397{ 398 struct pl08x_driver_data *pl08x = plchan->host; 399 struct pl08x_phy_chan *phychan = plchan->phychan; 400 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); 401 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 402 u32 val; 403 404 list_del(&txd->vd.node); 405 406 plchan->at = txd; 407 408 /* Wait for channel inactive */ 409 while (pl08x_phy_channel_busy(phychan)) 410 cpu_relax(); 411 412 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); 413 414 /* Enable the DMA channel */ 415 /* Do not access config register until channel shows as disabled */ 416 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id)) 417 cpu_relax(); 418 419 /* Do not access config register until channel shows as inactive */ 420 val = readl(phychan->reg_config); 421 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE)) 422 val = readl(phychan->reg_config); 423 424 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); 425} 426 427/* 428 * Pause the channel by setting the HALT bit. 429 * 430 * For M->P transfers, pause the DMAC first and then stop the peripheral - 431 * the FIFO can only drain if the peripheral is still requesting data. 432 * (note: this can still timeout if the DMAC FIFO never drains of data.) 433 * 434 * For P->M transfers, disable the peripheral first to stop it filling 435 * the DMAC FIFO, and then pause the DMAC. 436 */ 437static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) 438{ 439 u32 val; 440 int timeout; 441 442 /* Set the HALT bit and wait for the FIFO to drain */ 443 val = readl(ch->reg_config); 444 val |= PL080_CONFIG_HALT; 445 writel(val, ch->reg_config); 446 447 /* Wait for channel inactive */ 448 for (timeout = 1000; timeout; timeout--) { 449 if (!pl08x_phy_channel_busy(ch)) 450 break; 451 udelay(1); 452 } 453 if (pl08x_phy_channel_busy(ch)) 454 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); 455} 456 457static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 458{ 459 u32 val; 460 461 /* Clear the HALT bit */ 462 val = readl(ch->reg_config); 463 val &= ~PL080_CONFIG_HALT; 464 writel(val, ch->reg_config); 465} 466 467/* 468 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and 469 * clears any pending interrupt status. This should not be used for 470 * an on-going transfer, but as a method of shutting down a channel 471 * (eg, when it's no longer used) or terminating a transfer. 472 */ 473static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, 474 struct pl08x_phy_chan *ch) 475{ 476 u32 val = readl(ch->reg_config); 477 478 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | 479 PL080_CONFIG_TC_IRQ_MASK); 480 481 writel(val, ch->reg_config); 482 483 writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR); 484 writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR); 485} 486 487static inline u32 get_bytes_in_cctl(u32 cctl) 488{ 489 /* The source width defines the number of bytes */ 490 u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK; 491 492 cctl &= PL080_CONTROL_SWIDTH_MASK; 493 494 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) { 495 case PL080_WIDTH_8BIT: 496 break; 497 case PL080_WIDTH_16BIT: 498 bytes *= 2; 499 break; 500 case PL080_WIDTH_32BIT: 501 bytes *= 4; 502 break; 503 } 504 return bytes; 505} 506 507static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1) 508{ 509 /* The source width defines the number of bytes */ 510 u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK; 511 512 cctl &= PL080_CONTROL_SWIDTH_MASK; 513 514 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) { 515 case PL080_WIDTH_8BIT: 516 break; 517 case PL080_WIDTH_16BIT: 518 bytes *= 2; 519 break; 520 case PL080_WIDTH_32BIT: 521 bytes *= 4; 522 break; 523 } 524 return bytes; 525} 526 527/* The channel should be paused when calling this */ 528static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 529{ 530 struct pl08x_driver_data *pl08x = plchan->host; 531 const u32 *llis_va, *llis_va_limit; 532 struct pl08x_phy_chan *ch; 533 dma_addr_t llis_bus; 534 struct pl08x_txd *txd; 535 u32 llis_max_words; 536 size_t bytes; 537 u32 clli; 538 539 ch = plchan->phychan; 540 txd = plchan->at; 541 542 if (!ch || !txd) 543 return 0; 544 545 /* 546 * Follow the LLIs to get the number of remaining 547 * bytes in the currently active transaction. 548 */ 549 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2; 550 551 /* First get the remaining bytes in the active transfer */ 552 if (pl08x->vd->pl080s) 553 bytes = get_bytes_in_cctl_pl080s( 554 readl(ch->base + PL080_CH_CONTROL), 555 readl(ch->base + PL080S_CH_CONTROL2)); 556 else 557 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL)); 558 559 if (!clli) 560 return bytes; 561 562 llis_va = txd->llis_va; 563 llis_bus = txd->llis_bus; 564 565 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; 566 BUG_ON(clli < llis_bus || clli >= llis_bus + 567 sizeof(u32) * llis_max_words); 568 569 /* 570 * Locate the next LLI - as this is an array, 571 * it's simple maths to find. 572 */ 573 llis_va += (clli - llis_bus) / sizeof(u32); 574 575 llis_va_limit = llis_va + llis_max_words; 576 577 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { 578 if (pl08x->vd->pl080s) 579 bytes += get_bytes_in_cctl_pl080s( 580 llis_va[PL080_LLI_CCTL], 581 llis_va[PL080S_LLI_CCTL2]); 582 else 583 bytes += get_bytes_in_cctl(llis_va[PL080_LLI_CCTL]); 584 585 /* 586 * A LLI pointer going backward terminates the LLI list 587 */ 588 if (llis_va[PL080_LLI_LLI] <= clli) 589 break; 590 } 591 592 return bytes; 593} 594 595/* 596 * Allocate a physical channel for a virtual channel 597 * 598 * Try to locate a physical channel to be used for this transfer. If all 599 * are taken return NULL and the requester will have to cope by using 600 * some fallback PIO mode or retrying later. 601 */ 602static struct pl08x_phy_chan * 603pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, 604 struct pl08x_dma_chan *virt_chan) 605{ 606 struct pl08x_phy_chan *ch = NULL; 607 unsigned long flags; 608 int i; 609 610 for (i = 0; i < pl08x->vd->channels; i++) { 611 ch = &pl08x->phy_chans[i]; 612 613 spin_lock_irqsave(&ch->lock, flags); 614 615 if (!ch->locked && !ch->serving) { 616 ch->serving = virt_chan; 617 spin_unlock_irqrestore(&ch->lock, flags); 618 break; 619 } 620 621 spin_unlock_irqrestore(&ch->lock, flags); 622 } 623 624 if (i == pl08x->vd->channels) { 625 /* No physical channel available, cope with it */ 626 return NULL; 627 } 628 629 return ch; 630} 631 632/* Mark the physical channel as free. Note, this write is atomic. */ 633static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, 634 struct pl08x_phy_chan *ch) 635{ 636 ch->serving = NULL; 637} 638 639/* 640 * Try to allocate a physical channel. When successful, assign it to 641 * this virtual channel, and initiate the next descriptor. The 642 * virtual channel lock must be held at this point. 643 */ 644static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) 645{ 646 struct pl08x_driver_data *pl08x = plchan->host; 647 struct pl08x_phy_chan *ch; 648 649 ch = pl08x_get_phy_channel(pl08x, plchan); 650 if (!ch) { 651 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); 652 plchan->state = PL08X_CHAN_WAITING; 653 return; 654 } 655 656 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", 657 ch->id, plchan->name); 658 659 plchan->phychan = ch; 660 plchan->state = PL08X_CHAN_RUNNING; 661 pl08x_start_next_txd(plchan); 662} 663 664static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, 665 struct pl08x_dma_chan *plchan) 666{ 667 struct pl08x_driver_data *pl08x = plchan->host; 668 669 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", 670 ch->id, plchan->name); 671 672 /* 673 * We do this without taking the lock; we're really only concerned 674 * about whether this pointer is NULL or not, and we're guaranteed 675 * that this will only be called when it _already_ is non-NULL. 676 */ 677 ch->serving = plchan; 678 plchan->phychan = ch; 679 plchan->state = PL08X_CHAN_RUNNING; 680 pl08x_start_next_txd(plchan); 681} 682 683/* 684 * Free a physical DMA channel, potentially reallocating it to another 685 * virtual channel if we have any pending. 686 */ 687static void pl08x_phy_free(struct pl08x_dma_chan *plchan) 688{ 689 struct pl08x_driver_data *pl08x = plchan->host; 690 struct pl08x_dma_chan *p, *next; 691 692 retry: 693 next = NULL; 694 695 /* Find a waiting virtual channel for the next transfer. */ 696 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) 697 if (p->state == PL08X_CHAN_WAITING) { 698 next = p; 699 break; 700 } 701 702 if (!next) { 703 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) 704 if (p->state == PL08X_CHAN_WAITING) { 705 next = p; 706 break; 707 } 708 } 709 710 /* Ensure that the physical channel is stopped */ 711 pl08x_terminate_phy_chan(pl08x, plchan->phychan); 712 713 if (next) { 714 bool success; 715 716 /* 717 * Eww. We know this isn't going to deadlock 718 * but lockdep probably doesn't. 719 */ 720 spin_lock(&next->vc.lock); 721 /* Re-check the state now that we have the lock */ 722 success = next->state == PL08X_CHAN_WAITING; 723 if (success) 724 pl08x_phy_reassign_start(plchan->phychan, next); 725 spin_unlock(&next->vc.lock); 726 727 /* If the state changed, try to find another channel */ 728 if (!success) 729 goto retry; 730 } else { 731 /* No more jobs, so free up the physical channel */ 732 pl08x_put_phy_channel(pl08x, plchan->phychan); 733 } 734 735 plchan->phychan = NULL; 736 plchan->state = PL08X_CHAN_IDLE; 737} 738 739/* 740 * LLI handling 741 */ 742 743static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded) 744{ 745 switch (coded) { 746 case PL080_WIDTH_8BIT: 747 return 1; 748 case PL080_WIDTH_16BIT: 749 return 2; 750 case PL080_WIDTH_32BIT: 751 return 4; 752 default: 753 break; 754 } 755 BUG(); 756 return 0; 757} 758 759static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, 760 size_t tsize) 761{ 762 u32 retbits = cctl; 763 764 /* Remove all src, dst and transfer size bits */ 765 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 766 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 767 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 768 769 /* Then set the bits according to the parameters */ 770 switch (srcwidth) { 771 case 1: 772 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT; 773 break; 774 case 2: 775 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT; 776 break; 777 case 4: 778 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT; 779 break; 780 default: 781 BUG(); 782 break; 783 } 784 785 switch (dstwidth) { 786 case 1: 787 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 788 break; 789 case 2: 790 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 791 break; 792 case 4: 793 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 794 break; 795 default: 796 BUG(); 797 break; 798 } 799 800 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; 801 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; 802 return retbits; 803} 804 805struct pl08x_lli_build_data { 806 struct pl08x_txd *txd; 807 struct pl08x_bus_data srcbus; 808 struct pl08x_bus_data dstbus; 809 size_t remainder; 810 u32 lli_bus; 811}; 812 813/* 814 * Autoselect a master bus to use for the transfer. Slave will be the chosen as 815 * victim in case src & dest are not similarly aligned. i.e. If after aligning 816 * masters address with width requirements of transfer (by sending few byte by 817 * byte data), slave is still not aligned, then its width will be reduced to 818 * BYTE. 819 * - prefers the destination bus if both available 820 * - prefers bus with fixed address (i.e. peripheral) 821 */ 822static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd, 823 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl) 824{ 825 if (!(cctl & PL080_CONTROL_DST_INCR)) { 826 *mbus = &bd->dstbus; 827 *sbus = &bd->srcbus; 828 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { 829 *mbus = &bd->srcbus; 830 *sbus = &bd->dstbus; 831 } else { 832 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { 833 *mbus = &bd->dstbus; 834 *sbus = &bd->srcbus; 835 } else { 836 *mbus = &bd->srcbus; 837 *sbus = &bd->dstbus; 838 } 839 } 840} 841 842/* 843 * Fills in one LLI for a certain transfer descriptor and advance the counter 844 */ 845static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 846 struct pl08x_lli_build_data *bd, 847 int num_llis, int len, u32 cctl, u32 cctl2) 848{ 849 u32 offset = num_llis * pl08x->lli_words; 850 u32 *llis_va = bd->txd->llis_va + offset; 851 dma_addr_t llis_bus = bd->txd->llis_bus; 852 853 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 854 855 /* Advance the offset to next LLI. */ 856 offset += pl08x->lli_words; 857 858 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; 859 llis_va[PL080_LLI_DST] = bd->dstbus.addr; 860 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); 861 llis_va[PL080_LLI_LLI] |= bd->lli_bus; 862 llis_va[PL080_LLI_CCTL] = cctl; 863 if (pl08x->vd->pl080s) 864 llis_va[PL080S_LLI_CCTL2] = cctl2; 865 866 if (cctl & PL080_CONTROL_SRC_INCR) 867 bd->srcbus.addr += len; 868 if (cctl & PL080_CONTROL_DST_INCR) 869 bd->dstbus.addr += len; 870 871 BUG_ON(bd->remainder < len); 872 873 bd->remainder -= len; 874} 875 876static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, 877 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, 878 int num_llis, size_t *total_bytes) 879{ 880 *cctl = pl08x_cctl_bits(*cctl, 1, 1, len); 881 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); 882 (*total_bytes) += len; 883} 884 885#ifdef VERBOSE_DEBUG 886static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 887 const u32 *llis_va, int num_llis) 888{ 889 int i; 890 891 if (pl08x->vd->pl080s) { 892 dev_vdbg(&pl08x->adev->dev, 893 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", 894 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); 895 for (i = 0; i < num_llis; i++) { 896 dev_vdbg(&pl08x->adev->dev, 897 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 898 i, llis_va, llis_va[PL080_LLI_SRC], 899 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 900 llis_va[PL080_LLI_CCTL], 901 llis_va[PL080S_LLI_CCTL2]); 902 llis_va += pl08x->lli_words; 903 } 904 } else { 905 dev_vdbg(&pl08x->adev->dev, 906 "%-3s %-9s %-10s %-10s %-10s %s\n", 907 "lli", "", "csrc", "cdst", "clli", "cctl"); 908 for (i = 0; i < num_llis; i++) { 909 dev_vdbg(&pl08x->adev->dev, 910 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", 911 i, llis_va, llis_va[PL080_LLI_SRC], 912 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 913 llis_va[PL080_LLI_CCTL]); 914 llis_va += pl08x->lli_words; 915 } 916 } 917} 918#else 919static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 920 const u32 *llis_va, int num_llis) {} 921#endif 922 923/* 924 * This fills in the table of LLIs for the transfer descriptor 925 * Note that we assume we never have to change the burst sizes 926 * Return 0 for error 927 */ 928static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 929 struct pl08x_txd *txd) 930{ 931 struct pl08x_bus_data *mbus, *sbus; 932 struct pl08x_lli_build_data bd; 933 int num_llis = 0; 934 u32 cctl, early_bytes = 0; 935 size_t max_bytes_per_lli, total_bytes; 936 u32 *llis_va, *last_lli; 937 struct pl08x_sg *dsg; 938 939 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); 940 if (!txd->llis_va) { 941 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); 942 return 0; 943 } 944 945 bd.txd = txd; 946 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; 947 cctl = txd->cctl; 948 949 /* Find maximum width of the source bus */ 950 bd.srcbus.maxwidth = 951 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >> 952 PL080_CONTROL_SWIDTH_SHIFT); 953 954 /* Find maximum width of the destination bus */ 955 bd.dstbus.maxwidth = 956 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> 957 PL080_CONTROL_DWIDTH_SHIFT); 958 959 list_for_each_entry(dsg, &txd->dsg_list, node) { 960 total_bytes = 0; 961 cctl = txd->cctl; 962 963 bd.srcbus.addr = dsg->src_addr; 964 bd.dstbus.addr = dsg->dst_addr; 965 bd.remainder = dsg->len; 966 bd.srcbus.buswidth = bd.srcbus.maxwidth; 967 bd.dstbus.buswidth = bd.dstbus.maxwidth; 968 969 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); 970 971 dev_vdbg(&pl08x->adev->dev, 972 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", 973 (u64)bd.srcbus.addr, 974 cctl & PL080_CONTROL_SRC_INCR ? "+" : "", 975 bd.srcbus.buswidth, 976 (u64)bd.dstbus.addr, 977 cctl & PL080_CONTROL_DST_INCR ? "+" : "", 978 bd.dstbus.buswidth, 979 bd.remainder); 980 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", 981 mbus == &bd.srcbus ? "src" : "dst", 982 sbus == &bd.srcbus ? "src" : "dst"); 983 984 /* 985 * Zero length is only allowed if all these requirements are 986 * met: 987 * - flow controller is peripheral. 988 * - src.addr is aligned to src.width 989 * - dst.addr is aligned to dst.width 990 * 991 * sg_len == 1 should be true, as there can be two cases here: 992 * 993 * - Memory addresses are contiguous and are not scattered. 994 * Here, Only one sg will be passed by user driver, with 995 * memory address and zero length. We pass this to controller 996 * and after the transfer it will receive the last burst 997 * request from peripheral and so transfer finishes. 998 * 999 * - Memory addresses are scattered and are not contiguous. 1000 * Here, Obviously as DMA controller doesn't know when a lli's 1001 * transfer gets over, it can't load next lli. So in this 1002 * case, there has to be an assumption that only one lli is 1003 * supported. Thus, we can't have scattered addresses. 1004 */ 1005 if (!bd.remainder) { 1006 u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> 1007 PL080_CONFIG_FLOW_CONTROL_SHIFT; 1008 if (!((fc >= PL080_FLOW_SRC2DST_DST) && 1009 (fc <= PL080_FLOW_SRC2DST_SRC))) { 1010 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", 1011 __func__); 1012 return 0; 1013 } 1014 1015 if (!IS_BUS_ALIGNED(&bd.srcbus) || 1016 !IS_BUS_ALIGNED(&bd.dstbus)) { 1017 dev_err(&pl08x->adev->dev, 1018 "%s src & dst address must be aligned to src" 1019 " & dst width if peripheral is flow controller", 1020 __func__); 1021 return 0; 1022 } 1023 1024 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, 1025 bd.dstbus.buswidth, 0); 1026 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1027 0, cctl, 0); 1028 break; 1029 } 1030 1031 /* 1032 * Send byte by byte for following cases 1033 * - Less than a bus width available 1034 * - until master bus is aligned 1035 */ 1036 if (bd.remainder < mbus->buswidth) 1037 early_bytes = bd.remainder; 1038 else if (!IS_BUS_ALIGNED(mbus)) { 1039 early_bytes = mbus->buswidth - 1040 (mbus->addr & (mbus->buswidth - 1)); 1041 if ((bd.remainder - early_bytes) < mbus->buswidth) 1042 early_bytes = bd.remainder; 1043 } 1044 1045 if (early_bytes) { 1046 dev_vdbg(&pl08x->adev->dev, 1047 "%s byte width LLIs (remain 0x%08zx)\n", 1048 __func__, bd.remainder); 1049 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, 1050 num_llis++, &total_bytes); 1051 } 1052 1053 if (bd.remainder) { 1054 /* 1055 * Master now aligned 1056 * - if slave is not then we must set its width down 1057 */ 1058 if (!IS_BUS_ALIGNED(sbus)) { 1059 dev_dbg(&pl08x->adev->dev, 1060 "%s set down bus width to one byte\n", 1061 __func__); 1062 1063 sbus->buswidth = 1; 1064 } 1065 1066 /* 1067 * Bytes transferred = tsize * src width, not 1068 * MIN(buswidths) 1069 */ 1070 max_bytes_per_lli = bd.srcbus.buswidth * 1071 pl08x->vd->max_transfer_size; 1072 dev_vdbg(&pl08x->adev->dev, 1073 "%s max bytes per lli = %zu\n", 1074 __func__, max_bytes_per_lli); 1075 1076 /* 1077 * Make largest possible LLIs until less than one bus 1078 * width left 1079 */ 1080 while (bd.remainder > (mbus->buswidth - 1)) { 1081 size_t lli_len, tsize, width; 1082 1083 /* 1084 * If enough left try to send max possible, 1085 * otherwise try to send the remainder 1086 */ 1087 lli_len = min(bd.remainder, max_bytes_per_lli); 1088 1089 /* 1090 * Check against maximum bus alignment: 1091 * Calculate actual transfer size in relation to 1092 * bus width an get a maximum remainder of the 1093 * highest bus width - 1 1094 */ 1095 width = max(mbus->buswidth, sbus->buswidth); 1096 lli_len = (lli_len / width) * width; 1097 tsize = lli_len / bd.srcbus.buswidth; 1098 1099 dev_vdbg(&pl08x->adev->dev, 1100 "%s fill lli with single lli chunk of " 1101 "size 0x%08zx (remainder 0x%08zx)\n", 1102 __func__, lli_len, bd.remainder); 1103 1104 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth, 1105 bd.dstbus.buswidth, tsize); 1106 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1107 lli_len, cctl, tsize); 1108 total_bytes += lli_len; 1109 } 1110 1111 /* 1112 * Send any odd bytes 1113 */ 1114 if (bd.remainder) { 1115 dev_vdbg(&pl08x->adev->dev, 1116 "%s align with boundary, send odd bytes (remain %zu)\n", 1117 __func__, bd.remainder); 1118 prep_byte_width_lli(pl08x, &bd, &cctl, 1119 bd.remainder, num_llis++, &total_bytes); 1120 } 1121 } 1122 1123 if (total_bytes != dsg->len) { 1124 dev_err(&pl08x->adev->dev, 1125 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", 1126 __func__, total_bytes, dsg->len); 1127 return 0; 1128 } 1129 1130 if (num_llis >= MAX_NUM_TSFR_LLIS) { 1131 dev_err(&pl08x->adev->dev, 1132 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", 1133 __func__, MAX_NUM_TSFR_LLIS); 1134 return 0; 1135 } 1136 } 1137 1138 llis_va = txd->llis_va; 1139 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; 1140 1141 if (txd->cyclic) { 1142 /* Link back to the first LLI. */ 1143 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; 1144 } else { 1145 /* The final LLI terminates the LLI. */ 1146 last_lli[PL080_LLI_LLI] = 0; 1147 /* The final LLI element shall also fire an interrupt. */ 1148 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; 1149 } 1150 1151 pl08x_dump_lli(pl08x, llis_va, num_llis); 1152 1153 return num_llis; 1154} 1155 1156static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 1157 struct pl08x_txd *txd) 1158{ 1159 struct pl08x_sg *dsg, *_dsg; 1160 1161 if (txd->llis_va) 1162 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); 1163 1164 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { 1165 list_del(&dsg->node); 1166 kfree(dsg); 1167 } 1168 1169 kfree(txd); 1170} 1171 1172static void pl08x_desc_free(struct virt_dma_desc *vd) 1173{ 1174 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1175 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1176 1177 dma_descriptor_unmap(&vd->tx); 1178 if (!txd->done) 1179 pl08x_release_mux(plchan); 1180 1181 pl08x_free_txd(plchan->host, txd); 1182} 1183 1184static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, 1185 struct pl08x_dma_chan *plchan) 1186{ 1187 LIST_HEAD(head); 1188 1189 vchan_get_all_descriptors(&plchan->vc, &head); 1190 vchan_dma_desc_free_list(&plchan->vc, &head); 1191} 1192 1193/* 1194 * The DMA ENGINE API 1195 */ 1196static void pl08x_free_chan_resources(struct dma_chan *chan) 1197{ 1198 /* Ensure all queued descriptors are freed */ 1199 vchan_free_chan_resources(to_virt_chan(chan)); 1200} 1201 1202static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( 1203 struct dma_chan *chan, unsigned long flags) 1204{ 1205 struct dma_async_tx_descriptor *retval = NULL; 1206 1207 return retval; 1208} 1209 1210/* 1211 * Code accessing dma_async_is_complete() in a tight loop may give problems. 1212 * If slaves are relying on interrupts to signal completion this function 1213 * must not be called with interrupts disabled. 1214 */ 1215static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, 1216 dma_cookie_t cookie, struct dma_tx_state *txstate) 1217{ 1218 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1219 struct virt_dma_desc *vd; 1220 unsigned long flags; 1221 enum dma_status ret; 1222 size_t bytes = 0; 1223 1224 ret = dma_cookie_status(chan, cookie, txstate); 1225 if (ret == DMA_COMPLETE) 1226 return ret; 1227 1228 /* 1229 * There's no point calculating the residue if there's 1230 * no txstate to store the value. 1231 */ 1232 if (!txstate) { 1233 if (plchan->state == PL08X_CHAN_PAUSED) 1234 ret = DMA_PAUSED; 1235 return ret; 1236 } 1237 1238 spin_lock_irqsave(&plchan->vc.lock, flags); 1239 ret = dma_cookie_status(chan, cookie, txstate); 1240 if (ret != DMA_COMPLETE) { 1241 vd = vchan_find_desc(&plchan->vc, cookie); 1242 if (vd) { 1243 /* On the issued list, so hasn't been processed yet */ 1244 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1245 struct pl08x_sg *dsg; 1246 1247 list_for_each_entry(dsg, &txd->dsg_list, node) 1248 bytes += dsg->len; 1249 } else { 1250 bytes = pl08x_getbytes_chan(plchan); 1251 } 1252 } 1253 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1254 1255 /* 1256 * This cookie not complete yet 1257 * Get number of bytes left in the active transactions and queue 1258 */ 1259 dma_set_residue(txstate, bytes); 1260 1261 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) 1262 ret = DMA_PAUSED; 1263 1264 /* Whether waiting or running, we're in progress */ 1265 return ret; 1266} 1267 1268/* PrimeCell DMA extension */ 1269struct burst_table { 1270 u32 burstwords; 1271 u32 reg; 1272}; 1273 1274static const struct burst_table burst_sizes[] = { 1275 { 1276 .burstwords = 256, 1277 .reg = PL080_BSIZE_256, 1278 }, 1279 { 1280 .burstwords = 128, 1281 .reg = PL080_BSIZE_128, 1282 }, 1283 { 1284 .burstwords = 64, 1285 .reg = PL080_BSIZE_64, 1286 }, 1287 { 1288 .burstwords = 32, 1289 .reg = PL080_BSIZE_32, 1290 }, 1291 { 1292 .burstwords = 16, 1293 .reg = PL080_BSIZE_16, 1294 }, 1295 { 1296 .burstwords = 8, 1297 .reg = PL080_BSIZE_8, 1298 }, 1299 { 1300 .burstwords = 4, 1301 .reg = PL080_BSIZE_4, 1302 }, 1303 { 1304 .burstwords = 0, 1305 .reg = PL080_BSIZE_1, 1306 }, 1307}; 1308 1309/* 1310 * Given the source and destination available bus masks, select which 1311 * will be routed to each port. We try to have source and destination 1312 * on separate ports, but always respect the allowable settings. 1313 */ 1314static u32 pl08x_select_bus(u8 src, u8 dst) 1315{ 1316 u32 cctl = 0; 1317 1318 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) 1319 cctl |= PL080_CONTROL_DST_AHB2; 1320 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) 1321 cctl |= PL080_CONTROL_SRC_AHB2; 1322 1323 return cctl; 1324} 1325 1326static u32 pl08x_cctl(u32 cctl) 1327{ 1328 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | 1329 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | 1330 PL080_CONTROL_PROT_MASK); 1331 1332 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1333 return cctl | PL080_CONTROL_PROT_SYS; 1334} 1335 1336static u32 pl08x_width(enum dma_slave_buswidth width) 1337{ 1338 switch (width) { 1339 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1340 return PL080_WIDTH_8BIT; 1341 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1342 return PL080_WIDTH_16BIT; 1343 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1344 return PL080_WIDTH_32BIT; 1345 default: 1346 return ~0; 1347 } 1348} 1349 1350static u32 pl08x_burst(u32 maxburst) 1351{ 1352 int i; 1353 1354 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) 1355 if (burst_sizes[i].burstwords <= maxburst) 1356 break; 1357 1358 return burst_sizes[i].reg; 1359} 1360 1361static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, 1362 enum dma_slave_buswidth addr_width, u32 maxburst) 1363{ 1364 u32 width, burst, cctl = 0; 1365 1366 width = pl08x_width(addr_width); 1367 if (width == ~0) 1368 return ~0; 1369 1370 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; 1371 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; 1372 1373 /* 1374 * If this channel will only request single transfers, set this 1375 * down to ONE element. Also select one element if no maxburst 1376 * is specified. 1377 */ 1378 if (plchan->cd->single) 1379 maxburst = 1; 1380 1381 burst = pl08x_burst(maxburst); 1382 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; 1383 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; 1384 1385 return pl08x_cctl(cctl); 1386} 1387 1388/* 1389 * Slave transactions callback to the slave device to allow 1390 * synchronization of slave DMA signals with the DMAC enable 1391 */ 1392static void pl08x_issue_pending(struct dma_chan *chan) 1393{ 1394 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1395 unsigned long flags; 1396 1397 spin_lock_irqsave(&plchan->vc.lock, flags); 1398 if (vchan_issue_pending(&plchan->vc)) { 1399 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) 1400 pl08x_phy_alloc_and_start(plchan); 1401 } 1402 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1403} 1404 1405static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) 1406{ 1407 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 1408 1409 if (txd) { 1410 INIT_LIST_HEAD(&txd->dsg_list); 1411 1412 /* Always enable error and terminal interrupts */ 1413 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1414 PL080_CONFIG_TC_IRQ_MASK; 1415 } 1416 return txd; 1417} 1418 1419/* 1420 * Initialize a descriptor to be used by memcpy submit 1421 */ 1422static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1423 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1424 size_t len, unsigned long flags) 1425{ 1426 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1427 struct pl08x_driver_data *pl08x = plchan->host; 1428 struct pl08x_txd *txd; 1429 struct pl08x_sg *dsg; 1430 int ret; 1431 1432 txd = pl08x_get_txd(plchan); 1433 if (!txd) { 1434 dev_err(&pl08x->adev->dev, 1435 "%s no memory for descriptor\n", __func__); 1436 return NULL; 1437 } 1438 1439 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1440 if (!dsg) { 1441 pl08x_free_txd(pl08x, txd); 1442 dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n", 1443 __func__); 1444 return NULL; 1445 } 1446 list_add_tail(&dsg->node, &txd->dsg_list); 1447 1448 dsg->src_addr = src; 1449 dsg->dst_addr = dest; 1450 dsg->len = len; 1451 1452 /* Set platform data for m2m */ 1453 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1454 txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy & 1455 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2); 1456 1457 /* Both to be incremented or the code will break */ 1458 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1459 1460 if (pl08x->vd->dualmaster) 1461 txd->cctl |= pl08x_select_bus(pl08x->mem_buses, 1462 pl08x->mem_buses); 1463 1464 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1465 if (!ret) { 1466 pl08x_free_txd(pl08x, txd); 1467 return NULL; 1468 } 1469 1470 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1471} 1472 1473static struct pl08x_txd *pl08x_init_txd( 1474 struct dma_chan *chan, 1475 enum dma_transfer_direction direction, 1476 dma_addr_t *slave_addr) 1477{ 1478 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1479 struct pl08x_driver_data *pl08x = plchan->host; 1480 struct pl08x_txd *txd; 1481 enum dma_slave_buswidth addr_width; 1482 int ret, tmp; 1483 u8 src_buses, dst_buses; 1484 u32 maxburst, cctl; 1485 1486 txd = pl08x_get_txd(plchan); 1487 if (!txd) { 1488 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1489 return NULL; 1490 } 1491 1492 /* 1493 * Set up addresses, the PrimeCell configured address 1494 * will take precedence since this may configure the 1495 * channel target address dynamically at runtime. 1496 */ 1497 if (direction == DMA_MEM_TO_DEV) { 1498 cctl = PL080_CONTROL_SRC_INCR; 1499 *slave_addr = plchan->cfg.dst_addr; 1500 addr_width = plchan->cfg.dst_addr_width; 1501 maxburst = plchan->cfg.dst_maxburst; 1502 src_buses = pl08x->mem_buses; 1503 dst_buses = plchan->cd->periph_buses; 1504 } else if (direction == DMA_DEV_TO_MEM) { 1505 cctl = PL080_CONTROL_DST_INCR; 1506 *slave_addr = plchan->cfg.src_addr; 1507 addr_width = plchan->cfg.src_addr_width; 1508 maxburst = plchan->cfg.src_maxburst; 1509 src_buses = plchan->cd->periph_buses; 1510 dst_buses = pl08x->mem_buses; 1511 } else { 1512 pl08x_free_txd(pl08x, txd); 1513 dev_err(&pl08x->adev->dev, 1514 "%s direction unsupported\n", __func__); 1515 return NULL; 1516 } 1517 1518 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); 1519 if (cctl == ~0) { 1520 pl08x_free_txd(pl08x, txd); 1521 dev_err(&pl08x->adev->dev, 1522 "DMA slave configuration botched?\n"); 1523 return NULL; 1524 } 1525 1526 txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses); 1527 1528 if (plchan->cfg.device_fc) 1529 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : 1530 PL080_FLOW_PER2MEM_PER; 1531 else 1532 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : 1533 PL080_FLOW_PER2MEM; 1534 1535 txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1536 1537 ret = pl08x_request_mux(plchan); 1538 if (ret < 0) { 1539 pl08x_free_txd(pl08x, txd); 1540 dev_dbg(&pl08x->adev->dev, 1541 "unable to mux for transfer on %s due to platform restrictions\n", 1542 plchan->name); 1543 return NULL; 1544 } 1545 1546 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", 1547 plchan->signal, plchan->name); 1548 1549 /* Assign the flow control signal to this channel */ 1550 if (direction == DMA_MEM_TO_DEV) 1551 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; 1552 else 1553 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; 1554 1555 return txd; 1556} 1557 1558static int pl08x_tx_add_sg(struct pl08x_txd *txd, 1559 enum dma_transfer_direction direction, 1560 dma_addr_t slave_addr, 1561 dma_addr_t buf_addr, 1562 unsigned int len) 1563{ 1564 struct pl08x_sg *dsg; 1565 1566 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1567 if (!dsg) 1568 return -ENOMEM; 1569 1570 list_add_tail(&dsg->node, &txd->dsg_list); 1571 1572 dsg->len = len; 1573 if (direction == DMA_MEM_TO_DEV) { 1574 dsg->src_addr = buf_addr; 1575 dsg->dst_addr = slave_addr; 1576 } else { 1577 dsg->src_addr = slave_addr; 1578 dsg->dst_addr = buf_addr; 1579 } 1580 1581 return 0; 1582} 1583 1584static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 1585 struct dma_chan *chan, struct scatterlist *sgl, 1586 unsigned int sg_len, enum dma_transfer_direction direction, 1587 unsigned long flags, void *context) 1588{ 1589 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1590 struct pl08x_driver_data *pl08x = plchan->host; 1591 struct pl08x_txd *txd; 1592 struct scatterlist *sg; 1593 int ret, tmp; 1594 dma_addr_t slave_addr; 1595 1596 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 1597 __func__, sg_dma_len(sgl), plchan->name); 1598 1599 txd = pl08x_init_txd(chan, direction, &slave_addr); 1600 if (!txd) 1601 return NULL; 1602 1603 for_each_sg(sgl, sg, sg_len, tmp) { 1604 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 1605 sg_dma_address(sg), 1606 sg_dma_len(sg)); 1607 if (ret) { 1608 pl08x_release_mux(plchan); 1609 pl08x_free_txd(pl08x, txd); 1610 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", 1611 __func__); 1612 return NULL; 1613 } 1614 } 1615 1616 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1617 if (!ret) { 1618 pl08x_release_mux(plchan); 1619 pl08x_free_txd(pl08x, txd); 1620 return NULL; 1621 } 1622 1623 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1624} 1625 1626static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( 1627 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 1628 size_t period_len, enum dma_transfer_direction direction, 1629 unsigned long flags) 1630{ 1631 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1632 struct pl08x_driver_data *pl08x = plchan->host; 1633 struct pl08x_txd *txd; 1634 int ret, tmp; 1635 dma_addr_t slave_addr; 1636 1637 dev_dbg(&pl08x->adev->dev, 1638 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", 1639 __func__, period_len, buf_len, 1640 direction == DMA_MEM_TO_DEV ? "to" : "from", 1641 plchan->name); 1642 1643 txd = pl08x_init_txd(chan, direction, &slave_addr); 1644 if (!txd) 1645 return NULL; 1646 1647 txd->cyclic = true; 1648 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; 1649 for (tmp = 0; tmp < buf_len; tmp += period_len) { 1650 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 1651 buf_addr + tmp, period_len); 1652 if (ret) { 1653 pl08x_release_mux(plchan); 1654 pl08x_free_txd(pl08x, txd); 1655 return NULL; 1656 } 1657 } 1658 1659 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1660 if (!ret) { 1661 pl08x_release_mux(plchan); 1662 pl08x_free_txd(pl08x, txd); 1663 return NULL; 1664 } 1665 1666 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1667} 1668 1669static int pl08x_config(struct dma_chan *chan, 1670 struct dma_slave_config *config) 1671{ 1672 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1673 struct pl08x_driver_data *pl08x = plchan->host; 1674 1675 if (!plchan->slave) 1676 return -EINVAL; 1677 1678 /* Reject definitely invalid configurations */ 1679 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 1680 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 1681 return -EINVAL; 1682 1683 if (config->device_fc && pl08x->vd->pl080s) { 1684 dev_err(&pl08x->adev->dev, 1685 "%s: PL080S does not support peripheral flow control\n", 1686 __func__); 1687 return -EINVAL; 1688 } 1689 1690 plchan->cfg = *config; 1691 1692 return 0; 1693} 1694 1695static int pl08x_terminate_all(struct dma_chan *chan) 1696{ 1697 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1698 struct pl08x_driver_data *pl08x = plchan->host; 1699 unsigned long flags; 1700 1701 spin_lock_irqsave(&plchan->vc.lock, flags); 1702 if (!plchan->phychan && !plchan->at) { 1703 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1704 return 0; 1705 } 1706 1707 plchan->state = PL08X_CHAN_IDLE; 1708 1709 if (plchan->phychan) { 1710 /* 1711 * Mark physical channel as free and free any slave 1712 * signal 1713 */ 1714 pl08x_phy_free(plchan); 1715 } 1716 /* Dequeue jobs and free LLIs */ 1717 if (plchan->at) { 1718 pl08x_desc_free(&plchan->at->vd); 1719 plchan->at = NULL; 1720 } 1721 /* Dequeue jobs not yet fired as well */ 1722 pl08x_free_txd_list(pl08x, plchan); 1723 1724 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1725 1726 return 0; 1727} 1728 1729static int pl08x_pause(struct dma_chan *chan) 1730{ 1731 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1732 unsigned long flags; 1733 1734 /* 1735 * Anything succeeds on channels with no physical allocation and 1736 * no queued transfers. 1737 */ 1738 spin_lock_irqsave(&plchan->vc.lock, flags); 1739 if (!plchan->phychan && !plchan->at) { 1740 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1741 return 0; 1742 } 1743 1744 pl08x_pause_phy_chan(plchan->phychan); 1745 plchan->state = PL08X_CHAN_PAUSED; 1746 1747 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1748 1749 return 0; 1750} 1751 1752static int pl08x_resume(struct dma_chan *chan) 1753{ 1754 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1755 unsigned long flags; 1756 1757 /* 1758 * Anything succeeds on channels with no physical allocation and 1759 * no queued transfers. 1760 */ 1761 spin_lock_irqsave(&plchan->vc.lock, flags); 1762 if (!plchan->phychan && !plchan->at) { 1763 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1764 return 0; 1765 } 1766 1767 pl08x_resume_phy_chan(plchan->phychan); 1768 plchan->state = PL08X_CHAN_RUNNING; 1769 1770 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1771 1772 return 0; 1773} 1774 1775bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) 1776{ 1777 struct pl08x_dma_chan *plchan; 1778 char *name = chan_id; 1779 1780 /* Reject channels for devices not bound to this driver */ 1781 if (chan->device->dev->driver != &pl08x_amba_driver.drv) 1782 return false; 1783 1784 plchan = to_pl08x_chan(chan); 1785 1786 /* Check that the channel is not taken! */ 1787 if (!strcmp(plchan->name, name)) 1788 return true; 1789 1790 return false; 1791} 1792EXPORT_SYMBOL_GPL(pl08x_filter_id); 1793 1794/* 1795 * Just check that the device is there and active 1796 * TODO: turn this bit on/off depending on the number of physical channels 1797 * actually used, if it is zero... well shut it off. That will save some 1798 * power. Cut the clock at the same time. 1799 */ 1800static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) 1801{ 1802 /* The Nomadik variant does not have the config register */ 1803 if (pl08x->vd->nomadik) 1804 return; 1805 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); 1806} 1807 1808static irqreturn_t pl08x_irq(int irq, void *dev) 1809{ 1810 struct pl08x_driver_data *pl08x = dev; 1811 u32 mask = 0, err, tc, i; 1812 1813 /* check & clear - ERR & TC interrupts */ 1814 err = readl(pl08x->base + PL080_ERR_STATUS); 1815 if (err) { 1816 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", 1817 __func__, err); 1818 writel(err, pl08x->base + PL080_ERR_CLEAR); 1819 } 1820 tc = readl(pl08x->base + PL080_TC_STATUS); 1821 if (tc) 1822 writel(tc, pl08x->base + PL080_TC_CLEAR); 1823 1824 if (!err && !tc) 1825 return IRQ_NONE; 1826 1827 for (i = 0; i < pl08x->vd->channels; i++) { 1828 if (((1 << i) & err) || ((1 << i) & tc)) { 1829 /* Locate physical channel */ 1830 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; 1831 struct pl08x_dma_chan *plchan = phychan->serving; 1832 struct pl08x_txd *tx; 1833 1834 if (!plchan) { 1835 dev_err(&pl08x->adev->dev, 1836 "%s Error TC interrupt on unused channel: 0x%08x\n", 1837 __func__, i); 1838 continue; 1839 } 1840 1841 spin_lock(&plchan->vc.lock); 1842 tx = plchan->at; 1843 if (tx && tx->cyclic) { 1844 vchan_cyclic_callback(&tx->vd); 1845 } else if (tx) { 1846 plchan->at = NULL; 1847 /* 1848 * This descriptor is done, release its mux 1849 * reservation. 1850 */ 1851 pl08x_release_mux(plchan); 1852 tx->done = true; 1853 vchan_cookie_complete(&tx->vd); 1854 1855 /* 1856 * And start the next descriptor (if any), 1857 * otherwise free this channel. 1858 */ 1859 if (vchan_next_desc(&plchan->vc)) 1860 pl08x_start_next_txd(plchan); 1861 else 1862 pl08x_phy_free(plchan); 1863 } 1864 spin_unlock(&plchan->vc.lock); 1865 1866 mask |= (1 << i); 1867 } 1868 } 1869 1870 return mask ? IRQ_HANDLED : IRQ_NONE; 1871} 1872 1873static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) 1874{ 1875 chan->slave = true; 1876 chan->name = chan->cd->bus_id; 1877 chan->cfg.src_addr = chan->cd->addr; 1878 chan->cfg.dst_addr = chan->cd->addr; 1879} 1880 1881/* 1882 * Initialise the DMAC memcpy/slave channels. 1883 * Make a local wrapper to hold required data 1884 */ 1885static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, 1886 struct dma_device *dmadev, unsigned int channels, bool slave) 1887{ 1888 struct pl08x_dma_chan *chan; 1889 int i; 1890 1891 INIT_LIST_HEAD(&dmadev->channels); 1892 1893 /* 1894 * Register as many many memcpy as we have physical channels, 1895 * we won't always be able to use all but the code will have 1896 * to cope with that situation. 1897 */ 1898 for (i = 0; i < channels; i++) { 1899 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 1900 if (!chan) { 1901 dev_err(&pl08x->adev->dev, 1902 "%s no memory for channel\n", __func__); 1903 return -ENOMEM; 1904 } 1905 1906 chan->host = pl08x; 1907 chan->state = PL08X_CHAN_IDLE; 1908 chan->signal = -1; 1909 1910 if (slave) { 1911 chan->cd = &pl08x->pd->slave_channels[i]; 1912 pl08x_dma_slave_init(chan); 1913 } else { 1914 chan->cd = &pl08x->pd->memcpy_channel; 1915 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); 1916 if (!chan->name) { 1917 kfree(chan); 1918 return -ENOMEM; 1919 } 1920 } 1921 dev_dbg(&pl08x->adev->dev, 1922 "initialize virtual channel \"%s\"\n", 1923 chan->name); 1924 1925 chan->vc.desc_free = pl08x_desc_free; 1926 vchan_init(&chan->vc, dmadev); 1927 } 1928 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", 1929 i, slave ? "slave" : "memcpy"); 1930 return i; 1931} 1932 1933static void pl08x_free_virtual_channels(struct dma_device *dmadev) 1934{ 1935 struct pl08x_dma_chan *chan = NULL; 1936 struct pl08x_dma_chan *next; 1937 1938 list_for_each_entry_safe(chan, 1939 next, &dmadev->channels, vc.chan.device_node) { 1940 list_del(&chan->vc.chan.device_node); 1941 kfree(chan); 1942 } 1943} 1944 1945#ifdef CONFIG_DEBUG_FS 1946static const char *pl08x_state_str(enum pl08x_dma_chan_state state) 1947{ 1948 switch (state) { 1949 case PL08X_CHAN_IDLE: 1950 return "idle"; 1951 case PL08X_CHAN_RUNNING: 1952 return "running"; 1953 case PL08X_CHAN_PAUSED: 1954 return "paused"; 1955 case PL08X_CHAN_WAITING: 1956 return "waiting"; 1957 default: 1958 break; 1959 } 1960 return "UNKNOWN STATE"; 1961} 1962 1963static int pl08x_debugfs_show(struct seq_file *s, void *data) 1964{ 1965 struct pl08x_driver_data *pl08x = s->private; 1966 struct pl08x_dma_chan *chan; 1967 struct pl08x_phy_chan *ch; 1968 unsigned long flags; 1969 int i; 1970 1971 seq_printf(s, "PL08x physical channels:\n"); 1972 seq_printf(s, "CHANNEL:\tUSER:\n"); 1973 seq_printf(s, "--------\t-----\n"); 1974 for (i = 0; i < pl08x->vd->channels; i++) { 1975 struct pl08x_dma_chan *virt_chan; 1976 1977 ch = &pl08x->phy_chans[i]; 1978 1979 spin_lock_irqsave(&ch->lock, flags); 1980 virt_chan = ch->serving; 1981 1982 seq_printf(s, "%d\t\t%s%s\n", 1983 ch->id, 1984 virt_chan ? virt_chan->name : "(none)", 1985 ch->locked ? " LOCKED" : ""); 1986 1987 spin_unlock_irqrestore(&ch->lock, flags); 1988 } 1989 1990 seq_printf(s, "\nPL08x virtual memcpy channels:\n"); 1991 seq_printf(s, "CHANNEL:\tSTATE:\n"); 1992 seq_printf(s, "--------\t------\n"); 1993 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { 1994 seq_printf(s, "%s\t\t%s\n", chan->name, 1995 pl08x_state_str(chan->state)); 1996 } 1997 1998 seq_printf(s, "\nPL08x virtual slave channels:\n"); 1999 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2000 seq_printf(s, "--------\t------\n"); 2001 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2002 seq_printf(s, "%s\t\t%s\n", chan->name, 2003 pl08x_state_str(chan->state)); 2004 } 2005 2006 return 0; 2007} 2008 2009static int pl08x_debugfs_open(struct inode *inode, struct file *file) 2010{ 2011 return single_open(file, pl08x_debugfs_show, inode->i_private); 2012} 2013 2014static const struct file_operations pl08x_debugfs_operations = { 2015 .open = pl08x_debugfs_open, 2016 .read = seq_read, 2017 .llseek = seq_lseek, 2018 .release = single_release, 2019}; 2020 2021static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2022{ 2023 /* Expose a simple debugfs interface to view all clocks */ 2024 (void) debugfs_create_file(dev_name(&pl08x->adev->dev), 2025 S_IFREG | S_IRUGO, NULL, pl08x, 2026 &pl08x_debugfs_operations); 2027} 2028 2029#else 2030static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2031{ 2032} 2033#endif 2034 2035#ifdef CONFIG_OF 2036static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, 2037 u32 id) 2038{ 2039 struct pl08x_dma_chan *chan; 2040 2041 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2042 if (chan->signal == id) 2043 return &chan->vc.chan; 2044 } 2045 2046 return NULL; 2047} 2048 2049static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, 2050 struct of_dma *ofdma) 2051{ 2052 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; 2053 struct pl08x_channel_data *data; 2054 struct pl08x_dma_chan *chan; 2055 struct dma_chan *dma_chan; 2056 2057 if (!pl08x) 2058 return NULL; 2059 2060 if (dma_spec->args_count != 2) 2061 return NULL; 2062 2063 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); 2064 if (dma_chan) 2065 return dma_get_slave_channel(dma_chan); 2066 2067 chan = devm_kzalloc(pl08x->slave.dev, sizeof(*chan) + sizeof(*data), 2068 GFP_KERNEL); 2069 if (!chan) 2070 return NULL; 2071 2072 data = (void *)&chan[1]; 2073 data->bus_id = "(none)"; 2074 data->periph_buses = dma_spec->args[1]; 2075 2076 chan->cd = data; 2077 chan->host = pl08x; 2078 chan->slave = true; 2079 chan->name = data->bus_id; 2080 chan->state = PL08X_CHAN_IDLE; 2081 chan->signal = dma_spec->args[0]; 2082 chan->vc.desc_free = pl08x_desc_free; 2083 2084 vchan_init(&chan->vc, &pl08x->slave); 2085 2086 return dma_get_slave_channel(&chan->vc.chan); 2087} 2088 2089static int pl08x_of_probe(struct amba_device *adev, 2090 struct pl08x_driver_data *pl08x, 2091 struct device_node *np) 2092{ 2093 struct pl08x_platform_data *pd; 2094 u32 cctl_memcpy = 0; 2095 u32 val; 2096 int ret; 2097 2098 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); 2099 if (!pd) 2100 return -ENOMEM; 2101 2102 /* Eligible bus masters for fetching LLIs */ 2103 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) 2104 pd->lli_buses |= PL08X_AHB1; 2105 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) 2106 pd->lli_buses |= PL08X_AHB2; 2107 if (!pd->lli_buses) { 2108 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); 2109 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; 2110 } 2111 2112 /* Eligible bus masters for memory access */ 2113 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) 2114 pd->mem_buses |= PL08X_AHB1; 2115 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) 2116 pd->mem_buses |= PL08X_AHB2; 2117 if (!pd->mem_buses) { 2118 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); 2119 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; 2120 } 2121 2122 /* Parse the memcpy channel properties */ 2123 ret = of_property_read_u32(np, "memcpy-burst-size", &val); 2124 if (ret) { 2125 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); 2126 val = 1; 2127 } 2128 switch (val) { 2129 default: 2130 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); 2131 /* Fall through */ 2132 case 1: 2133 cctl_memcpy |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | 2134 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; 2135 break; 2136 case 4: 2137 cctl_memcpy |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | 2138 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; 2139 break; 2140 case 8: 2141 cctl_memcpy |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | 2142 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; 2143 break; 2144 case 16: 2145 cctl_memcpy |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | 2146 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; 2147 break; 2148 case 32: 2149 cctl_memcpy |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | 2150 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; 2151 break; 2152 case 64: 2153 cctl_memcpy |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | 2154 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; 2155 break; 2156 case 128: 2157 cctl_memcpy |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | 2158 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; 2159 break; 2160 case 256: 2161 cctl_memcpy |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | 2162 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; 2163 break; 2164 } 2165 2166 ret = of_property_read_u32(np, "memcpy-bus-width", &val); 2167 if (ret) { 2168 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); 2169 val = 8; 2170 } 2171 switch (val) { 2172 default: 2173 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); 2174 /* Fall through */ 2175 case 8: 2176 cctl_memcpy |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | 2177 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 2178 break; 2179 case 16: 2180 cctl_memcpy |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | 2181 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 2182 break; 2183 case 32: 2184 cctl_memcpy |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | 2185 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 2186 break; 2187 } 2188 2189 /* This is currently the only thing making sense */ 2190 cctl_memcpy |= PL080_CONTROL_PROT_SYS; 2191 2192 /* Set up memcpy channel */ 2193 pd->memcpy_channel.bus_id = "memcpy"; 2194 pd->memcpy_channel.cctl_memcpy = cctl_memcpy; 2195 /* Use the buses that can access memory, obviously */ 2196 pd->memcpy_channel.periph_buses = pd->mem_buses; 2197 2198 pl08x->pd = pd; 2199 2200 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, 2201 pl08x); 2202} 2203#else 2204static inline int pl08x_of_probe(struct amba_device *adev, 2205 struct pl08x_driver_data *pl08x, 2206 struct device_node *np) 2207{ 2208 return -EINVAL; 2209} 2210#endif 2211 2212static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2213{ 2214 struct pl08x_driver_data *pl08x; 2215 const struct vendor_data *vd = id->data; 2216 struct device_node *np = adev->dev.of_node; 2217 u32 tsfr_size; 2218 int ret = 0; 2219 int i; 2220 2221 ret = amba_request_regions(adev, NULL); 2222 if (ret) 2223 return ret; 2224 2225 /* Ensure that we can do DMA */ 2226 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2227 if (ret) 2228 goto out_no_pl08x; 2229 2230 /* Create the driver state holder */ 2231 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); 2232 if (!pl08x) { 2233 ret = -ENOMEM; 2234 goto out_no_pl08x; 2235 } 2236 2237 /* Initialize memcpy engine */ 2238 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); 2239 pl08x->memcpy.dev = &adev->dev; 2240 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; 2241 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; 2242 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2243 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; 2244 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; 2245 pl08x->memcpy.device_config = pl08x_config; 2246 pl08x->memcpy.device_pause = pl08x_pause; 2247 pl08x->memcpy.device_resume = pl08x_resume; 2248 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; 2249 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2250 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2251 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); 2252 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2253 2254 /* Initialize slave engine */ 2255 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); 2256 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); 2257 pl08x->slave.dev = &adev->dev; 2258 pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources; 2259 pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2260 pl08x->slave.device_tx_status = pl08x_dma_tx_status; 2261 pl08x->slave.device_issue_pending = pl08x_issue_pending; 2262 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; 2263 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; 2264 pl08x->slave.device_config = pl08x_config; 2265 pl08x->slave.device_pause = pl08x_pause; 2266 pl08x->slave.device_resume = pl08x_resume; 2267 pl08x->slave.device_terminate_all = pl08x_terminate_all; 2268 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2269 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2270 pl08x->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2271 pl08x->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2272 2273 /* Get the platform data */ 2274 pl08x->pd = dev_get_platdata(&adev->dev); 2275 if (!pl08x->pd) { 2276 if (np) { 2277 ret = pl08x_of_probe(adev, pl08x, np); 2278 if (ret) 2279 goto out_no_platdata; 2280 } else { 2281 dev_err(&adev->dev, "no platform data supplied\n"); 2282 ret = -EINVAL; 2283 goto out_no_platdata; 2284 } 2285 } 2286 2287 /* Assign useful pointers to the driver state */ 2288 pl08x->adev = adev; 2289 pl08x->vd = vd; 2290 2291 /* By default, AHB1 only. If dualmaster, from platform */ 2292 pl08x->lli_buses = PL08X_AHB1; 2293 pl08x->mem_buses = PL08X_AHB1; 2294 if (pl08x->vd->dualmaster) { 2295 pl08x->lli_buses = pl08x->pd->lli_buses; 2296 pl08x->mem_buses = pl08x->pd->mem_buses; 2297 } 2298 2299 if (vd->pl080s) 2300 pl08x->lli_words = PL080S_LLI_WORDS; 2301 else 2302 pl08x->lli_words = PL080_LLI_WORDS; 2303 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); 2304 2305 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 2306 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 2307 tsfr_size, PL08X_ALIGN, 0); 2308 if (!pl08x->pool) { 2309 ret = -ENOMEM; 2310 goto out_no_lli_pool; 2311 } 2312 2313 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); 2314 if (!pl08x->base) { 2315 ret = -ENOMEM; 2316 goto out_no_ioremap; 2317 } 2318 2319 /* Turn on the PL08x */ 2320 pl08x_ensure_on(pl08x); 2321 2322 /* Attach the interrupt handler */ 2323 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); 2324 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 2325 2326 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); 2327 if (ret) { 2328 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 2329 __func__, adev->irq[0]); 2330 goto out_no_irq; 2331 } 2332 2333 /* Initialize physical channels */ 2334 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), 2335 GFP_KERNEL); 2336 if (!pl08x->phy_chans) { 2337 dev_err(&adev->dev, "%s failed to allocate " 2338 "physical channel holders\n", 2339 __func__); 2340 ret = -ENOMEM; 2341 goto out_no_phychans; 2342 } 2343 2344 for (i = 0; i < vd->channels; i++) { 2345 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; 2346 2347 ch->id = i; 2348 ch->base = pl08x->base + PL080_Cx_BASE(i); 2349 ch->reg_config = ch->base + vd->config_offset; 2350 spin_lock_init(&ch->lock); 2351 2352 /* 2353 * Nomadik variants can have channels that are locked 2354 * down for the secure world only. Lock up these channels 2355 * by perpetually serving a dummy virtual channel. 2356 */ 2357 if (vd->nomadik) { 2358 u32 val; 2359 2360 val = readl(ch->reg_config); 2361 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { 2362 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); 2363 ch->locked = true; 2364 } 2365 } 2366 2367 dev_dbg(&adev->dev, "physical channel %d is %s\n", 2368 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); 2369 } 2370 2371 /* Register as many memcpy channels as there are physical channels */ 2372 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, 2373 pl08x->vd->channels, false); 2374 if (ret <= 0) { 2375 dev_warn(&pl08x->adev->dev, 2376 "%s failed to enumerate memcpy channels - %d\n", 2377 __func__, ret); 2378 goto out_no_memcpy; 2379 } 2380 2381 /* Register slave channels */ 2382 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, 2383 pl08x->pd->num_slave_channels, true); 2384 if (ret < 0) { 2385 dev_warn(&pl08x->adev->dev, 2386 "%s failed to enumerate slave channels - %d\n", 2387 __func__, ret); 2388 goto out_no_slave; 2389 } 2390 2391 ret = dma_async_device_register(&pl08x->memcpy); 2392 if (ret) { 2393 dev_warn(&pl08x->adev->dev, 2394 "%s failed to register memcpy as an async device - %d\n", 2395 __func__, ret); 2396 goto out_no_memcpy_reg; 2397 } 2398 2399 ret = dma_async_device_register(&pl08x->slave); 2400 if (ret) { 2401 dev_warn(&pl08x->adev->dev, 2402 "%s failed to register slave as an async device - %d\n", 2403 __func__, ret); 2404 goto out_no_slave_reg; 2405 } 2406 2407 amba_set_drvdata(adev, pl08x); 2408 init_pl08x_debugfs(pl08x); 2409 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", 2410 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), 2411 (unsigned long long)adev->res.start, adev->irq[0]); 2412 2413 return 0; 2414 2415out_no_slave_reg: 2416 dma_async_device_unregister(&pl08x->memcpy); 2417out_no_memcpy_reg: 2418 pl08x_free_virtual_channels(&pl08x->slave); 2419out_no_slave: 2420 pl08x_free_virtual_channels(&pl08x->memcpy); 2421out_no_memcpy: 2422 kfree(pl08x->phy_chans); 2423out_no_phychans: 2424 free_irq(adev->irq[0], pl08x); 2425out_no_irq: 2426 iounmap(pl08x->base); 2427out_no_ioremap: 2428 dma_pool_destroy(pl08x->pool); 2429out_no_lli_pool: 2430out_no_platdata: 2431 kfree(pl08x); 2432out_no_pl08x: 2433 amba_release_regions(adev); 2434 return ret; 2435} 2436 2437/* PL080 has 8 channels and the PL080 have just 2 */ 2438static struct vendor_data vendor_pl080 = { 2439 .config_offset = PL080_CH_CONFIG, 2440 .channels = 8, 2441 .dualmaster = true, 2442 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2443}; 2444 2445static struct vendor_data vendor_nomadik = { 2446 .config_offset = PL080_CH_CONFIG, 2447 .channels = 8, 2448 .dualmaster = true, 2449 .nomadik = true, 2450 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2451}; 2452 2453static struct vendor_data vendor_pl080s = { 2454 .config_offset = PL080S_CH_CONFIG, 2455 .channels = 8, 2456 .pl080s = true, 2457 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, 2458}; 2459 2460static struct vendor_data vendor_pl081 = { 2461 .config_offset = PL080_CH_CONFIG, 2462 .channels = 2, 2463 .dualmaster = false, 2464 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2465}; 2466 2467static struct amba_id pl08x_ids[] = { 2468 /* Samsung PL080S variant */ 2469 { 2470 .id = 0x0a141080, 2471 .mask = 0xffffffff, 2472 .data = &vendor_pl080s, 2473 }, 2474 /* PL080 */ 2475 { 2476 .id = 0x00041080, 2477 .mask = 0x000fffff, 2478 .data = &vendor_pl080, 2479 }, 2480 /* PL081 */ 2481 { 2482 .id = 0x00041081, 2483 .mask = 0x000fffff, 2484 .data = &vendor_pl081, 2485 }, 2486 /* Nomadik 8815 PL080 variant */ 2487 { 2488 .id = 0x00280080, 2489 .mask = 0x00ffffff, 2490 .data = &vendor_nomadik, 2491 }, 2492 { 0, 0 }, 2493}; 2494 2495MODULE_DEVICE_TABLE(amba, pl08x_ids); 2496 2497static struct amba_driver pl08x_amba_driver = { 2498 .drv.name = DRIVER_NAME, 2499 .id_table = pl08x_ids, 2500 .probe = pl08x_probe, 2501}; 2502 2503static int __init pl08x_init(void) 2504{ 2505 int retval; 2506 retval = amba_driver_register(&pl08x_amba_driver); 2507 if (retval) 2508 printk(KERN_WARNING DRIVER_NAME 2509 "failed to register as an AMBA device (%d)\n", 2510 retval); 2511 return retval; 2512} 2513subsys_initcall(pl08x_init); 2514