1/* 2 * Copyright (C) 2005-2006 by Texas Instruments 3 * 4 * This file implements a DMA interface using TI's CPPI DMA. 5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB. 6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci. 7 */ 8 9#include <linux/module.h> 10#include <linux/platform_device.h> 11#include <linux/slab.h> 12#include <linux/usb.h> 13 14#include "musb_core.h" 15#include "musb_debug.h" 16#include "cppi_dma.h" 17 18 19/* CPPI DMA status 7-mar-2006: 20 * 21 * - See musb_{host,gadget}.c for more info 22 * 23 * - Correct RX DMA generally forces the engine into irq-per-packet mode, 24 * which can easily saturate the CPU under non-mass-storage loads. 25 * 26 * NOTES 24-aug-2006 (2.6.18-rc4): 27 * 28 * - peripheral RXDMA wedged in a test with packets of length 512/512/1. 29 * evidently after the 1 byte packet was received and acked, the queue 30 * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003, 31 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401 32 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx 33 * of its next (512 byte) packet. IRQ issues? 34 * 35 * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will 36 * evidently also directly update the RX and TX CSRs ... so audit all 37 * host and peripheral side DMA code to avoid CSR access after DMA has 38 * been started. 39 */ 40 41/* REVISIT now we can avoid preallocating these descriptors; or 42 * more simply, switch to a global freelist not per-channel ones. 43 * Note: at full speed, 64 descriptors == 4K bulk data. 44 */ 45#define NUM_TXCHAN_BD 64 46#define NUM_RXCHAN_BD 64 47 48static inline void cpu_drain_writebuffer(void) 49{ 50 wmb(); 51#ifdef CONFIG_CPU_ARM926T 52 /* REVISIT this "should not be needed", 53 * but lack of it sure seemed to hurt ... 54 */ 55 asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n"); 56#endif 57} 58 59static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c) 60{ 61 struct cppi_descriptor *bd = c->freelist; 62 63 if (bd) 64 c->freelist = bd->next; 65 return bd; 66} 67 68static inline void 69cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd) 70{ 71 if (!bd) 72 return; 73 bd->next = c->freelist; 74 c->freelist = bd; 75} 76 77/* 78 * Start DMA controller 79 * 80 * Initialize the DMA controller as necessary. 81 */ 82 83/* zero out entire rx state RAM entry for the channel */ 84static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) 85{ 86 musb_writel(&rx->rx_skipbytes, 0, 0); 87 musb_writel(&rx->rx_head, 0, 0); 88 musb_writel(&rx->rx_sop, 0, 0); 89 musb_writel(&rx->rx_current, 0, 0); 90 musb_writel(&rx->rx_buf_current, 0, 0); 91 musb_writel(&rx->rx_len_len, 0, 0); 92 musb_writel(&rx->rx_cnt_cnt, 0, 0); 93} 94 95/* zero out entire tx state RAM entry for the channel */ 96static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) 97{ 98 musb_writel(&tx->tx_head, 0, 0); 99 musb_writel(&tx->tx_buf, 0, 0); 100 musb_writel(&tx->tx_current, 0, 0); 101 musb_writel(&tx->tx_buf_current, 0, 0); 102 musb_writel(&tx->tx_info, 0, 0); 103 musb_writel(&tx->tx_rem_len, 0, 0); 104 /* musb_writel(&tx->tx_dummy, 0, 0); */ 105 musb_writel(&tx->tx_complete, 0, ptr); 106} 107 108static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c) 109{ 110 int j; 111 112 /* initialize channel fields */ 113 c->head = NULL; 114 c->tail = NULL; 115 c->last_processed = NULL; 116 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; 117 c->controller = cppi; 118 c->is_rndis = 0; 119 c->freelist = NULL; 120 121 /* build the BD Free list for the channel */ 122 for (j = 0; j < NUM_TXCHAN_BD + 1; j++) { 123 struct cppi_descriptor *bd; 124 dma_addr_t dma; 125 126 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); 127 bd->dma = dma; 128 cppi_bd_free(c, bd); 129 } 130} 131 132static int cppi_channel_abort(struct dma_channel *); 133 134static void cppi_pool_free(struct cppi_channel *c) 135{ 136 struct cppi *cppi = c->controller; 137 struct cppi_descriptor *bd; 138 139 (void) cppi_channel_abort(&c->channel); 140 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; 141 c->controller = NULL; 142 143 /* free all its bds */ 144 bd = c->last_processed; 145 do { 146 if (bd) 147 dma_pool_free(cppi->pool, bd, bd->dma); 148 bd = cppi_bd_alloc(c); 149 } while (bd); 150 c->last_processed = NULL; 151} 152 153static void cppi_controller_start(struct cppi *controller) 154{ 155 void __iomem *tibase; 156 int i; 157 158 /* do whatever is necessary to start controller */ 159 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { 160 controller->tx[i].transmit = true; 161 controller->tx[i].index = i; 162 } 163 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { 164 controller->rx[i].transmit = false; 165 controller->rx[i].index = i; 166 } 167 168 /* setup BD list on a per channel basis */ 169 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) 170 cppi_pool_init(controller, controller->tx + i); 171 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) 172 cppi_pool_init(controller, controller->rx + i); 173 174 tibase = controller->tibase; 175 INIT_LIST_HEAD(&controller->tx_complete); 176 177 /* initialise tx/rx channel head pointers to zero */ 178 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { 179 struct cppi_channel *tx_ch = controller->tx + i; 180 struct cppi_tx_stateram __iomem *tx; 181 182 INIT_LIST_HEAD(&tx_ch->tx_complete); 183 184 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); 185 tx_ch->state_ram = tx; 186 cppi_reset_tx(tx, 0); 187 } 188 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { 189 struct cppi_channel *rx_ch = controller->rx + i; 190 struct cppi_rx_stateram __iomem *rx; 191 192 INIT_LIST_HEAD(&rx_ch->tx_complete); 193 194 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); 195 rx_ch->state_ram = rx; 196 cppi_reset_rx(rx); 197 } 198 199 /* enable individual cppi channels */ 200 musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, 201 DAVINCI_DMA_ALL_CHANNELS_ENABLE); 202 musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG, 203 DAVINCI_DMA_ALL_CHANNELS_ENABLE); 204 205 /* enable tx/rx CPPI control */ 206 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); 207 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); 208 209 /* disable RNDIS mode, also host rx RNDIS autorequest */ 210 musb_writel(tibase, DAVINCI_RNDIS_REG, 0); 211 musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0); 212} 213 214/* 215 * Stop DMA controller 216 * 217 * De-Init the DMA controller as necessary. 218 */ 219 220static void cppi_controller_stop(struct cppi *controller) 221{ 222 void __iomem *tibase; 223 int i; 224 struct musb *musb; 225 226 musb = controller->musb; 227 228 tibase = controller->tibase; 229 /* DISABLE INDIVIDUAL CHANNEL Interrupts */ 230 musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, 231 DAVINCI_DMA_ALL_CHANNELS_ENABLE); 232 musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG, 233 DAVINCI_DMA_ALL_CHANNELS_ENABLE); 234 235 dev_dbg(musb->controller, "Tearing down RX and TX Channels\n"); 236 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { 237 /* FIXME restructure of txdma to use bds like rxdma */ 238 controller->tx[i].last_processed = NULL; 239 cppi_pool_free(controller->tx + i); 240 } 241 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) 242 cppi_pool_free(controller->rx + i); 243 244 /* in Tx Case proper teardown is supported. We resort to disabling 245 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is 246 * complete TX CPPI cannot be disabled. 247 */ 248 /*disable tx/rx cppi */ 249 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); 250 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); 251} 252 253/* While dma channel is allocated, we only want the core irqs active 254 * for fault reports, otherwise we'd get irqs that we don't care about. 255 * Except for TX irqs, where dma done != fifo empty and reusable ... 256 * 257 * NOTE: docs don't say either way, but irq masking **enables** irqs. 258 * 259 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma... 260 */ 261static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum) 262{ 263 musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8)); 264} 265 266static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum) 267{ 268 musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8)); 269} 270 271 272/* 273 * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to 274 * each transfer direction of a non-control endpoint, so allocating 275 * (and deallocating) is mostly a way to notice bad housekeeping on 276 * the software side. We assume the irqs are always active. 277 */ 278static struct dma_channel * 279cppi_channel_allocate(struct dma_controller *c, 280 struct musb_hw_ep *ep, u8 transmit) 281{ 282 struct cppi *controller; 283 u8 index; 284 struct cppi_channel *cppi_ch; 285 void __iomem *tibase; 286 struct musb *musb; 287 288 controller = container_of(c, struct cppi, controller); 289 tibase = controller->tibase; 290 musb = controller->musb; 291 292 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ 293 index = ep->epnum - 1; 294 295 /* return the corresponding CPPI Channel Handle, and 296 * probably disable the non-CPPI irq until we need it. 297 */ 298 if (transmit) { 299 if (index >= ARRAY_SIZE(controller->tx)) { 300 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index); 301 return NULL; 302 } 303 cppi_ch = controller->tx + index; 304 } else { 305 if (index >= ARRAY_SIZE(controller->rx)) { 306 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index); 307 return NULL; 308 } 309 cppi_ch = controller->rx + index; 310 core_rxirq_disable(tibase, ep->epnum); 311 } 312 313 /* REVISIT make this an error later once the same driver code works 314 * with the other DMA engine too 315 */ 316 if (cppi_ch->hw_ep) 317 dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n", 318 index, transmit ? 'T' : 'R', cppi_ch); 319 cppi_ch->hw_ep = ep; 320 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; 321 cppi_ch->channel.max_len = 0x7fffffff; 322 323 dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R'); 324 return &cppi_ch->channel; 325} 326 327/* Release a CPPI Channel. */ 328static void cppi_channel_release(struct dma_channel *channel) 329{ 330 struct cppi_channel *c; 331 void __iomem *tibase; 332 333 /* REVISIT: for paranoia, check state and abort if needed... */ 334 335 c = container_of(channel, struct cppi_channel, channel); 336 tibase = c->controller->tibase; 337 if (!c->hw_ep) 338 dev_dbg(c->controller->musb->controller, 339 "releasing idle DMA channel %p\n", c); 340 else if (!c->transmit) 341 core_rxirq_enable(tibase, c->index + 1); 342 343 /* for now, leave its cppi IRQ enabled (we won't trigger it) */ 344 c->hw_ep = NULL; 345 channel->status = MUSB_DMA_STATUS_UNKNOWN; 346} 347 348/* Context: controller irqlocked */ 349static void 350cppi_dump_rx(int level, struct cppi_channel *c, const char *tag) 351{ 352 void __iomem *base = c->controller->mregs; 353 struct cppi_rx_stateram __iomem *rx = c->state_ram; 354 355 musb_ep_select(base, c->index + 1); 356 357 dev_dbg(c->controller->musb->controller, 358 "RX DMA%d%s: %d left, csr %04x, " 359 "%08x H%08x S%08x C%08x, " 360 "B%08x L%08x %08x .. %08x" 361 "\n", 362 c->index, tag, 363 musb_readl(c->controller->tibase, 364 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), 365 musb_readw(c->hw_ep->regs, MUSB_RXCSR), 366 367 musb_readl(&rx->rx_skipbytes, 0), 368 musb_readl(&rx->rx_head, 0), 369 musb_readl(&rx->rx_sop, 0), 370 musb_readl(&rx->rx_current, 0), 371 372 musb_readl(&rx->rx_buf_current, 0), 373 musb_readl(&rx->rx_len_len, 0), 374 musb_readl(&rx->rx_cnt_cnt, 0), 375 musb_readl(&rx->rx_complete, 0) 376 ); 377} 378 379/* Context: controller irqlocked */ 380static void 381cppi_dump_tx(int level, struct cppi_channel *c, const char *tag) 382{ 383 void __iomem *base = c->controller->mregs; 384 struct cppi_tx_stateram __iomem *tx = c->state_ram; 385 386 musb_ep_select(base, c->index + 1); 387 388 dev_dbg(c->controller->musb->controller, 389 "TX DMA%d%s: csr %04x, " 390 "H%08x S%08x C%08x %08x, " 391 "F%08x L%08x .. %08x" 392 "\n", 393 c->index, tag, 394 musb_readw(c->hw_ep->regs, MUSB_TXCSR), 395 396 musb_readl(&tx->tx_head, 0), 397 musb_readl(&tx->tx_buf, 0), 398 musb_readl(&tx->tx_current, 0), 399 musb_readl(&tx->tx_buf_current, 0), 400 401 musb_readl(&tx->tx_info, 0), 402 musb_readl(&tx->tx_rem_len, 0), 403 /* dummy/unused word 6 */ 404 musb_readl(&tx->tx_complete, 0) 405 ); 406} 407 408/* Context: controller irqlocked */ 409static inline void 410cppi_rndis_update(struct cppi_channel *c, int is_rx, 411 void __iomem *tibase, int is_rndis) 412{ 413 /* we may need to change the rndis flag for this cppi channel */ 414 if (c->is_rndis != is_rndis) { 415 u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG); 416 u32 temp = 1 << (c->index); 417 418 if (is_rx) 419 temp <<= 16; 420 if (is_rndis) 421 value |= temp; 422 else 423 value &= ~temp; 424 musb_writel(tibase, DAVINCI_RNDIS_REG, value); 425 c->is_rndis = is_rndis; 426 } 427} 428 429static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd) 430{ 431 pr_debug("RXBD/%s %08x: " 432 "nxt %08x buf %08x off.blen %08x opt.plen %08x\n", 433 tag, bd->dma, 434 bd->hw_next, bd->hw_bufp, bd->hw_off_len, 435 bd->hw_options); 436} 437 438static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) 439{ 440 struct cppi_descriptor *bd; 441 442 cppi_dump_rx(level, rx, tag); 443 if (rx->last_processed) 444 cppi_dump_rxbd("last", rx->last_processed); 445 for (bd = rx->head; bd; bd = bd->next) 446 cppi_dump_rxbd("active", bd); 447} 448 449 450/* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX; 451 * so we won't ever use it (see "CPPI RX Woes" below). 452 */ 453static inline int cppi_autoreq_update(struct cppi_channel *rx, 454 void __iomem *tibase, int onepacket, unsigned n_bds) 455{ 456 u32 val; 457 458#ifdef RNDIS_RX_IS_USABLE 459 u32 tmp; 460 /* assert(is_host_active(musb)) */ 461 462 /* start from "AutoReq never" */ 463 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); 464 val = tmp & ~((0x3) << (rx->index * 2)); 465 466 /* HCD arranged reqpkt for packet #1. we arrange int 467 * for all but the last one, maybe in two segments. 468 */ 469 if (!onepacket) { 470#if 0 471 /* use two segments, autoreq "all" then the last "never" */ 472 val |= ((0x3) << (rx->index * 2)); 473 n_bds--; 474#else 475 /* one segment, autoreq "all-but-last" */ 476 val |= ((0x1) << (rx->index * 2)); 477#endif 478 } 479 480 if (val != tmp) { 481 int n = 100; 482 483 /* make sure that autoreq is updated before continuing */ 484 musb_writel(tibase, DAVINCI_AUTOREQ_REG, val); 485 do { 486 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); 487 if (tmp == val) 488 break; 489 cpu_relax(); 490 } while (n-- > 0); 491 } 492#endif 493 494 /* REQPKT is turned off after each segment */ 495 if (n_bds && rx->channel.actual_len) { 496 void __iomem *regs = rx->hw_ep->regs; 497 498 val = musb_readw(regs, MUSB_RXCSR); 499 if (!(val & MUSB_RXCSR_H_REQPKT)) { 500 val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS; 501 musb_writew(regs, MUSB_RXCSR, val); 502 /* flush writebuffer */ 503 val = musb_readw(regs, MUSB_RXCSR); 504 } 505 } 506 return n_bds; 507} 508 509 510/* Buffer enqueuing Logic: 511 * 512 * - RX builds new queues each time, to help handle routine "early 513 * termination" cases (faults, including errors and short reads) 514 * more correctly. 515 * 516 * - for now, TX reuses the same queue of BDs every time 517 * 518 * REVISIT long term, we want a normal dynamic model. 519 * ... the goal will be to append to the 520 * existing queue, processing completed "dma buffers" (segments) on the fly. 521 * 522 * Otherwise we force an IRQ latency between requests, which slows us a lot 523 * (especially in "transparent" dma). Unfortunately that model seems to be 524 * inherent in the DMA model from the Mentor code, except in the rare case 525 * of transfers big enough (~128+ KB) that we could append "middle" segments 526 * in the TX paths. (RX can't do this, see below.) 527 * 528 * That's true even in the CPPI- friendly iso case, where most urbs have 529 * several small segments provided in a group and where the "packet at a time" 530 * "transparent" DMA model is always correct, even on the RX side. 531 */ 532 533/* 534 * CPPI TX: 535 * ======== 536 * TX is a lot more reasonable than RX; it doesn't need to run in 537 * irq-per-packet mode very often. RNDIS mode seems to behave too 538 * (except how it handles the exactly-N-packets case). Building a 539 * txdma queue with multiple requests (urb or usb_request) looks 540 * like it would work ... but fault handling would need much testing. 541 * 542 * The main issue with TX mode RNDIS relates to transfer lengths that 543 * are an exact multiple of the packet length. It appears that there's 544 * a hiccup in that case (maybe the DMA completes before the ZLP gets 545 * written?) boiling down to not being able to rely on CPPI writing any 546 * terminating zero length packet before the next transfer is written. 547 * So that's punted to PIO; better yet, gadget drivers can avoid it. 548 * 549 * Plus, there's allegedly an undocumented constraint that rndis transfer 550 * length be a multiple of 64 bytes ... but the chip doesn't act that 551 * way, and we really don't _want_ that behavior anyway. 552 * 553 * On TX, "transparent" mode works ... although experiments have shown 554 * problems trying to use the SOP/EOP bits in different USB packets. 555 * 556 * REVISIT try to handle terminating zero length packets using CPPI 557 * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet 558 * links avoid that issue by forcing them to avoid zlps.) 559 */ 560static void 561cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) 562{ 563 unsigned maxpacket = tx->maxpacket; 564 dma_addr_t addr = tx->buf_dma + tx->offset; 565 size_t length = tx->buf_len - tx->offset; 566 struct cppi_descriptor *bd; 567 unsigned n_bds; 568 unsigned i; 569 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; 570 int rndis; 571 572 /* TX can use the CPPI "rndis" mode, where we can probably fit this 573 * transfer in one BD and one IRQ. The only time we would NOT want 574 * to use it is when hardware constraints prevent it, or if we'd 575 * trigger the "send a ZLP?" confusion. 576 */ 577 rndis = (maxpacket & 0x3f) == 0 578 && length > maxpacket 579 && length < 0xffff 580 && (length % maxpacket) != 0; 581 582 if (rndis) { 583 maxpacket = length; 584 n_bds = 1; 585 } else { 586 n_bds = length / maxpacket; 587 if (!length || (length % maxpacket)) 588 n_bds++; 589 n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD); 590 length = min(n_bds * maxpacket, length); 591 } 592 593 dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n", 594 tx->index, 595 maxpacket, 596 rndis ? "rndis" : "transparent", 597 n_bds, 598 (unsigned long long)addr, length); 599 600 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); 601 602 /* assuming here that channel_program is called during 603 * transfer initiation ... current code maintains state 604 * for one outstanding request only (no queues, not even 605 * the implicit ones of an iso urb). 606 */ 607 608 bd = tx->freelist; 609 tx->head = bd; 610 tx->last_processed = NULL; 611 612 /* FIXME use BD pool like RX side does, and just queue 613 * the minimum number for this request. 614 */ 615 616 /* Prepare queue of BDs first, then hand it to hardware. 617 * All BDs except maybe the last should be of full packet 618 * size; for RNDIS there _is_ only that last packet. 619 */ 620 for (i = 0; i < n_bds; ) { 621 if (++i < n_bds && bd->next) 622 bd->hw_next = bd->next->dma; 623 else 624 bd->hw_next = 0; 625 626 bd->hw_bufp = tx->buf_dma + tx->offset; 627 628 /* FIXME set EOP only on the last packet, 629 * SOP only on the first ... avoid IRQs 630 */ 631 if ((tx->offset + maxpacket) <= tx->buf_len) { 632 tx->offset += maxpacket; 633 bd->hw_off_len = maxpacket; 634 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET 635 | CPPI_OWN_SET | maxpacket; 636 } else { 637 /* only this one may be a partial USB Packet */ 638 u32 partial_len; 639 640 partial_len = tx->buf_len - tx->offset; 641 tx->offset = tx->buf_len; 642 bd->hw_off_len = partial_len; 643 644 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET 645 | CPPI_OWN_SET | partial_len; 646 if (partial_len == 0) 647 bd->hw_options |= CPPI_ZERO_SET; 648 } 649 650 dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n", 651 bd, bd->hw_next, bd->hw_bufp, 652 bd->hw_off_len, bd->hw_options); 653 654 /* update the last BD enqueued to the list */ 655 tx->tail = bd; 656 bd = bd->next; 657 } 658 659 /* BDs live in DMA-coherent memory, but writes might be pending */ 660 cpu_drain_writebuffer(); 661 662 /* Write to the HeadPtr in state RAM to trigger */ 663 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); 664 665 cppi_dump_tx(5, tx, "/S"); 666} 667 668/* 669 * CPPI RX Woes: 670 * ============= 671 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte 672 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back. 673 * (Full speed transfers have similar scenarios.) 674 * 675 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes, 676 * and the next packet goes into a buffer that's queued later; while (b) fills 677 * the buffer with 1024 bytes. How to do that with CPPI? 678 * 679 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but 680 * (b) loses **BADLY** because nothing (!) happens when that second packet 681 * fills the buffer, much less when a third one arrives. (Which makes this 682 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination 683 * is optional, and it's fine if peripherals -- not hosts! -- pad messages 684 * out to end-of-buffer. Standard PCI host controller DMA descriptors 685 * implement that mode by default ... which is no accident.) 686 * 687 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have 688 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX 689 * ignores SOP/EOP markings and processes both of those BDs; so both packets 690 * are loaded into the buffer (with a 212 byte gap between them), and the next 691 * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP 692 * are intended as outputs for RX queues, not inputs...) 693 * 694 * - A variant of "transparent" mode -- one BD at a time -- is the only way to 695 * reliably make both cases work, with software handling both cases correctly 696 * and at the significant penalty of needing an IRQ per packet. (The lack of 697 * I/O overlap can be slightly ameliorated by enabling double buffering.) 698 * 699 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could 700 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK 701 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors 702 * with guaranteed driver level fault recovery and scrubbing out what's left 703 * of that garbaged datastream. 704 * 705 * But there seems to be no way to identify the cases where CPPI RNDIS mode 706 * is appropriate -- which do NOT include RNDIS host drivers, but do include 707 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong. 708 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic 709 * that applies best on the peripheral side (and which could fail rudely). 710 * 711 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all 712 * cases other than mass storage class. Otherwise we're correct but slow, 713 * since CPPI penalizes our need for a "true RNDIS" default mode. 714 */ 715 716 717/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY 718 * 719 * IFF 720 * (a) peripheral mode ... since rndis peripherals could pad their 721 * writes to hosts, causing i/o failure; or we'd have to cope with 722 * a largely unknowable variety of host side protocol variants 723 * (b) and short reads are NOT errors ... since full reads would 724 * cause those same i/o failures 725 * (c) and read length is 726 * - less than 64KB (max per cppi descriptor) 727 * - not a multiple of 4096 (g_zero default, full reads typical) 728 * - N (>1) packets long, ditto (full reads not EXPECTED) 729 * THEN 730 * try rx rndis mode 731 * 732 * Cost of heuristic failing: RXDMA wedges at the end of transfers that 733 * fill out the whole buffer. Buggy host side usb network drivers could 734 * trigger that, but "in the field" such bugs seem to be all but unknown. 735 * 736 * So this module parameter lets the heuristic be disabled. When using 737 * gadgetfs, the heuristic will probably need to be disabled. 738 */ 739static bool cppi_rx_rndis = 1; 740 741module_param(cppi_rx_rndis, bool, 0); 742MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic"); 743 744 745/** 746 * cppi_next_rx_segment - dma read for the next chunk of a buffer 747 * @musb: the controller 748 * @rx: dma channel 749 * @onepacket: true unless caller treats short reads as errors, and 750 * performs fault recovery above usbcore. 751 * Context: controller irqlocked 752 * 753 * See above notes about why we can't use multi-BD RX queues except in 754 * rare cases (mass storage class), and can never use the hardware "rndis" 755 * mode (since it's not a "true" RNDIS mode) with complete safety.. 756 * 757 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in 758 * code to recover from corrupted datastreams after each short transfer. 759 */ 760static void 761cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) 762{ 763 unsigned maxpacket = rx->maxpacket; 764 dma_addr_t addr = rx->buf_dma + rx->offset; 765 size_t length = rx->buf_len - rx->offset; 766 struct cppi_descriptor *bd, *tail; 767 unsigned n_bds; 768 unsigned i; 769 void __iomem *tibase = musb->ctrl_base; 770 int is_rndis = 0; 771 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; 772 struct cppi_descriptor *d; 773 774 if (onepacket) { 775 /* almost every USB driver, host or peripheral side */ 776 n_bds = 1; 777 778 /* maybe apply the heuristic above */ 779 if (cppi_rx_rndis 780 && is_peripheral_active(musb) 781 && length > maxpacket 782 && (length & ~0xffff) == 0 783 && (length & 0x0fff) != 0 784 && (length & (maxpacket - 1)) == 0) { 785 maxpacket = length; 786 is_rndis = 1; 787 } 788 } else { 789 /* virtually nothing except mass storage class */ 790 if (length > 0xffff) { 791 n_bds = 0xffff / maxpacket; 792 length = n_bds * maxpacket; 793 } else { 794 n_bds = length / maxpacket; 795 if (length % maxpacket) 796 n_bds++; 797 } 798 if (n_bds == 1) 799 onepacket = 1; 800 else 801 n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); 802 } 803 804 /* In host mode, autorequest logic can generate some IN tokens; it's 805 * tricky since we can't leave REQPKT set in RXCSR after the transfer 806 * finishes. So: multipacket transfers involve two or more segments. 807 * And always at least two IRQs ... RNDIS mode is not an option. 808 */ 809 if (is_host_active(musb)) 810 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); 811 812 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); 813 814 length = min(n_bds * maxpacket, length); 815 816 dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " 817 "dma 0x%llx len %u %u/%u\n", 818 rx->index, maxpacket, 819 onepacket 820 ? (is_rndis ? "rndis" : "onepacket") 821 : "multipacket", 822 n_bds, 823 musb_readl(tibase, 824 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) 825 & 0xffff, 826 (unsigned long long)addr, length, 827 rx->channel.actual_len, rx->buf_len); 828 829 /* only queue one segment at a time, since the hardware prevents 830 * correct queue shutdown after unexpected short packets 831 */ 832 bd = cppi_bd_alloc(rx); 833 rx->head = bd; 834 835 /* Build BDs for all packets in this segment */ 836 for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { 837 u32 bd_len; 838 839 if (i) { 840 bd = cppi_bd_alloc(rx); 841 if (!bd) 842 break; 843 tail->next = bd; 844 tail->hw_next = bd->dma; 845 } 846 bd->hw_next = 0; 847 848 /* all but the last packet will be maxpacket size */ 849 if (maxpacket < length) 850 bd_len = maxpacket; 851 else 852 bd_len = length; 853 854 bd->hw_bufp = addr; 855 addr += bd_len; 856 rx->offset += bd_len; 857 858 bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; 859 bd->buflen = bd_len; 860 861 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); 862 length -= bd_len; 863 } 864 865 /* we always expect at least one reusable BD! */ 866 if (!tail) { 867 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); 868 return; 869 } else if (i < n_bds) 870 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); 871 872 tail->next = NULL; 873 tail->hw_next = 0; 874 875 bd = rx->head; 876 rx->tail = tail; 877 878 /* short reads and other faults should terminate this entire 879 * dma segment. we want one "dma packet" per dma segment, not 880 * one per USB packet, terminating the whole queue at once... 881 * NOTE that current hardware seems to ignore SOP and EOP. 882 */ 883 bd->hw_options |= CPPI_SOP_SET; 884 tail->hw_options |= CPPI_EOP_SET; 885 886 for (d = rx->head; d; d = d->next) 887 cppi_dump_rxbd("S", d); 888 889 /* in case the preceding transfer left some state... */ 890 tail = rx->last_processed; 891 if (tail) { 892 tail->next = bd; 893 tail->hw_next = bd->dma; 894 } 895 896 core_rxirq_enable(tibase, rx->index + 1); 897 898 /* BDs live in DMA-coherent memory, but writes might be pending */ 899 cpu_drain_writebuffer(); 900 901 /* REVISIT specs say to write this AFTER the BUFCNT register 902 * below ... but that loses badly. 903 */ 904 musb_writel(&rx_ram->rx_head, 0, bd->dma); 905 906 /* bufferCount must be at least 3, and zeroes on completion 907 * unless it underflows below zero, or stops at two, or keeps 908 * growing ... grr. 909 */ 910 i = musb_readl(tibase, 911 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) 912 & 0xffff; 913 914 if (!i) 915 musb_writel(tibase, 916 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), 917 n_bds + 2); 918 else if (n_bds > (i - 3)) 919 musb_writel(tibase, 920 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), 921 n_bds - (i - 3)); 922 923 i = musb_readl(tibase, 924 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) 925 & 0xffff; 926 if (i < (2 + n_bds)) { 927 dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n", 928 rx->index, i, n_bds); 929 musb_writel(tibase, 930 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), 931 n_bds + 2); 932 } 933 934 cppi_dump_rx(4, rx, "/S"); 935} 936 937/** 938 * cppi_channel_program - program channel for data transfer 939 * @ch: the channel 940 * @maxpacket: max packet size 941 * @mode: For RX, 1 unless the usb protocol driver promised to treat 942 * all short reads as errors and kick in high level fault recovery. 943 * For TX, ignored because of RNDIS mode races/glitches. 944 * @dma_addr: dma address of buffer 945 * @len: length of buffer 946 * Context: controller irqlocked 947 */ 948static int cppi_channel_program(struct dma_channel *ch, 949 u16 maxpacket, u8 mode, 950 dma_addr_t dma_addr, u32 len) 951{ 952 struct cppi_channel *cppi_ch; 953 struct cppi *controller; 954 struct musb *musb; 955 956 cppi_ch = container_of(ch, struct cppi_channel, channel); 957 controller = cppi_ch->controller; 958 musb = controller->musb; 959 960 switch (ch->status) { 961 case MUSB_DMA_STATUS_BUS_ABORT: 962 case MUSB_DMA_STATUS_CORE_ABORT: 963 /* fault irq handler should have handled cleanup */ 964 WARNING("%cX DMA%d not cleaned up after abort!\n", 965 cppi_ch->transmit ? 'T' : 'R', 966 cppi_ch->index); 967 /* WARN_ON(1); */ 968 break; 969 case MUSB_DMA_STATUS_BUSY: 970 WARNING("program active channel? %cX DMA%d\n", 971 cppi_ch->transmit ? 'T' : 'R', 972 cppi_ch->index); 973 /* WARN_ON(1); */ 974 break; 975 case MUSB_DMA_STATUS_UNKNOWN: 976 dev_dbg(musb->controller, "%cX DMA%d not allocated!\n", 977 cppi_ch->transmit ? 'T' : 'R', 978 cppi_ch->index); 979 /* FALLTHROUGH */ 980 case MUSB_DMA_STATUS_FREE: 981 break; 982 } 983 984 ch->status = MUSB_DMA_STATUS_BUSY; 985 986 /* set transfer parameters, then queue up its first segment */ 987 cppi_ch->buf_dma = dma_addr; 988 cppi_ch->offset = 0; 989 cppi_ch->maxpacket = maxpacket; 990 cppi_ch->buf_len = len; 991 cppi_ch->channel.actual_len = 0; 992 993 /* TX channel? or RX? */ 994 if (cppi_ch->transmit) 995 cppi_next_tx_segment(musb, cppi_ch); 996 else 997 cppi_next_rx_segment(musb, cppi_ch, mode); 998 999 return true; 1000} 1001 1002static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) 1003{ 1004 struct cppi_channel *rx = &cppi->rx[ch]; 1005 struct cppi_rx_stateram __iomem *state = rx->state_ram; 1006 struct cppi_descriptor *bd; 1007 struct cppi_descriptor *last = rx->last_processed; 1008 bool completed = false; 1009 bool acked = false; 1010 int i; 1011 dma_addr_t safe2ack; 1012 void __iomem *regs = rx->hw_ep->regs; 1013 struct musb *musb = cppi->musb; 1014 1015 cppi_dump_rx(6, rx, "/K"); 1016 1017 bd = last ? last->next : rx->head; 1018 if (!bd) 1019 return false; 1020 1021 /* run through all completed BDs */ 1022 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); 1023 (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; 1024 i++, bd = bd->next) { 1025 u16 len; 1026 1027 /* catch latest BD writes from CPPI */ 1028 rmb(); 1029 if (!completed && (bd->hw_options & CPPI_OWN_SET)) 1030 break; 1031 1032 dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x " 1033 "off.len %08x opt.len %08x (%d)\n", 1034 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, 1035 bd->hw_off_len, bd->hw_options, 1036 rx->channel.actual_len); 1037 1038 /* actual packet received length */ 1039 if ((bd->hw_options & CPPI_SOP_SET) && !completed) 1040 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; 1041 else 1042 len = 0; 1043 1044 if (bd->hw_options & CPPI_EOQ_MASK) 1045 completed = true; 1046 1047 if (!completed && len < bd->buflen) { 1048 /* NOTE: when we get a short packet, RXCSR_H_REQPKT 1049 * must have been cleared, and no more DMA packets may 1050 * active be in the queue... TI docs didn't say, but 1051 * CPPI ignores those BDs even though OWN is still set. 1052 */ 1053 completed = true; 1054 dev_dbg(musb->controller, "rx short %d/%d (%d)\n", 1055 len, bd->buflen, 1056 rx->channel.actual_len); 1057 } 1058 1059 /* If we got here, we expect to ack at least one BD; meanwhile 1060 * CPPI may completing other BDs while we scan this list... 1061 * 1062 * RACE: we can notice OWN cleared before CPPI raises the 1063 * matching irq by writing that BD as the completion pointer. 1064 * In such cases, stop scanning and wait for the irq, avoiding 1065 * lost acks and states where BD ownership is unclear. 1066 */ 1067 if (bd->dma == safe2ack) { 1068 musb_writel(&state->rx_complete, 0, safe2ack); 1069 safe2ack = musb_readl(&state->rx_complete, 0); 1070 acked = true; 1071 if (bd->dma == safe2ack) 1072 safe2ack = 0; 1073 } 1074 1075 rx->channel.actual_len += len; 1076 1077 cppi_bd_free(rx, last); 1078 last = bd; 1079 1080 /* stop scanning on end-of-segment */ 1081 if (bd->hw_next == 0) 1082 completed = true; 1083 } 1084 rx->last_processed = last; 1085 1086 /* dma abort, lost ack, or ... */ 1087 if (!acked && last) { 1088 int csr; 1089 1090 if (safe2ack == 0 || safe2ack == rx->last_processed->dma) 1091 musb_writel(&state->rx_complete, 0, safe2ack); 1092 if (safe2ack == 0) { 1093 cppi_bd_free(rx, last); 1094 rx->last_processed = NULL; 1095 1096 /* if we land here on the host side, H_REQPKT will 1097 * be clear and we need to restart the queue... 1098 */ 1099 WARN_ON(rx->head); 1100 } 1101 musb_ep_select(cppi->mregs, rx->index + 1); 1102 csr = musb_readw(regs, MUSB_RXCSR); 1103 if (csr & MUSB_RXCSR_DMAENAB) { 1104 dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n", 1105 rx->index, 1106 rx->head, rx->tail, 1107 rx->last_processed 1108 ? (unsigned long long) 1109 rx->last_processed->dma 1110 : 0, 1111 completed ? ", completed" : "", 1112 csr); 1113 cppi_dump_rxq(4, "/what?", rx); 1114 } 1115 } 1116 if (!completed) { 1117 int csr; 1118 1119 rx->head = bd; 1120 1121 /* REVISIT seems like "autoreq all but EOP" doesn't... 1122 * setting it here "should" be racey, but seems to work 1123 */ 1124 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); 1125 if (is_host_active(cppi->musb) 1126 && bd 1127 && !(csr & MUSB_RXCSR_H_REQPKT)) { 1128 csr |= MUSB_RXCSR_H_REQPKT; 1129 musb_writew(regs, MUSB_RXCSR, 1130 MUSB_RXCSR_H_WZC_BITS | csr); 1131 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); 1132 } 1133 } else { 1134 rx->head = NULL; 1135 rx->tail = NULL; 1136 } 1137 1138 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); 1139 return completed; 1140} 1141 1142irqreturn_t cppi_interrupt(int irq, void *dev_id) 1143{ 1144 struct musb *musb = dev_id; 1145 struct cppi *cppi; 1146 void __iomem *tibase; 1147 struct musb_hw_ep *hw_ep = NULL; 1148 u32 rx, tx; 1149 int i, index; 1150 unsigned long uninitialized_var(flags); 1151 1152 cppi = container_of(musb->dma_controller, struct cppi, controller); 1153 if (cppi->irq) 1154 spin_lock_irqsave(&musb->lock, flags); 1155 1156 tibase = musb->ctrl_base; 1157 1158 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); 1159 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); 1160 1161 if (!tx && !rx) { 1162 if (cppi->irq) 1163 spin_unlock_irqrestore(&musb->lock, flags); 1164 return IRQ_NONE; 1165 } 1166 1167 dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx); 1168 1169 /* process TX channels */ 1170 for (index = 0; tx; tx = tx >> 1, index++) { 1171 struct cppi_channel *tx_ch; 1172 struct cppi_tx_stateram __iomem *tx_ram; 1173 bool completed = false; 1174 struct cppi_descriptor *bd; 1175 1176 if (!(tx & 1)) 1177 continue; 1178 1179 tx_ch = cppi->tx + index; 1180 tx_ram = tx_ch->state_ram; 1181 1182 /* FIXME need a cppi_tx_scan() routine, which 1183 * can also be called from abort code 1184 */ 1185 1186 cppi_dump_tx(5, tx_ch, "/E"); 1187 1188 bd = tx_ch->head; 1189 1190 /* 1191 * If Head is null then this could mean that a abort interrupt 1192 * that needs to be acknowledged. 1193 */ 1194 if (NULL == bd) { 1195 dev_dbg(musb->controller, "null BD\n"); 1196 musb_writel(&tx_ram->tx_complete, 0, 0); 1197 continue; 1198 } 1199 1200 /* run through all completed BDs */ 1201 for (i = 0; !completed && bd && i < NUM_TXCHAN_BD; 1202 i++, bd = bd->next) { 1203 u16 len; 1204 1205 /* catch latest BD writes from CPPI */ 1206 rmb(); 1207 if (bd->hw_options & CPPI_OWN_SET) 1208 break; 1209 1210 dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n", 1211 bd, bd->hw_next, bd->hw_bufp, 1212 bd->hw_off_len, bd->hw_options); 1213 1214 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; 1215 tx_ch->channel.actual_len += len; 1216 1217 tx_ch->last_processed = bd; 1218 1219 /* write completion register to acknowledge 1220 * processing of completed BDs, and possibly 1221 * release the IRQ; EOQ might not be set ... 1222 * 1223 * REVISIT use the same ack strategy as rx 1224 * 1225 * REVISIT have observed bit 18 set; huh?? 1226 */ 1227 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ 1228 musb_writel(&tx_ram->tx_complete, 0, bd->dma); 1229 1230 /* stop scanning on end-of-segment */ 1231 if (bd->hw_next == 0) 1232 completed = true; 1233 } 1234 1235 /* on end of segment, maybe go to next one */ 1236 if (completed) { 1237 /* cppi_dump_tx(4, tx_ch, "/complete"); */ 1238 1239 /* transfer more, or report completion */ 1240 if (tx_ch->offset >= tx_ch->buf_len) { 1241 tx_ch->head = NULL; 1242 tx_ch->tail = NULL; 1243 tx_ch->channel.status = MUSB_DMA_STATUS_FREE; 1244 1245 hw_ep = tx_ch->hw_ep; 1246 1247 musb_dma_completion(musb, index + 1, 1); 1248 1249 } else { 1250 /* Bigger transfer than we could fit in 1251 * that first batch of descriptors... 1252 */ 1253 cppi_next_tx_segment(musb, tx_ch); 1254 } 1255 } else 1256 tx_ch->head = bd; 1257 } 1258 1259 /* Start processing the RX block */ 1260 for (index = 0; rx; rx = rx >> 1, index++) { 1261 1262 if (rx & 1) { 1263 struct cppi_channel *rx_ch; 1264 1265 rx_ch = cppi->rx + index; 1266 1267 /* let incomplete dma segments finish */ 1268 if (!cppi_rx_scan(cppi, index)) 1269 continue; 1270 1271 /* start another dma segment if needed */ 1272 if (rx_ch->channel.actual_len != rx_ch->buf_len 1273 && rx_ch->channel.actual_len 1274 == rx_ch->offset) { 1275 cppi_next_rx_segment(musb, rx_ch, 1); 1276 continue; 1277 } 1278 1279 /* all segments completed! */ 1280 rx_ch->channel.status = MUSB_DMA_STATUS_FREE; 1281 1282 hw_ep = rx_ch->hw_ep; 1283 1284 core_rxirq_disable(tibase, index + 1); 1285 musb_dma_completion(musb, index + 1, 0); 1286 } 1287 } 1288 1289 /* write to CPPI EOI register to re-enable interrupts */ 1290 musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0); 1291 1292 if (cppi->irq) 1293 spin_unlock_irqrestore(&musb->lock, flags); 1294 1295 return IRQ_HANDLED; 1296} 1297EXPORT_SYMBOL_GPL(cppi_interrupt); 1298 1299/* Instantiate a software object representing a DMA controller. */ 1300struct dma_controller *dma_controller_create(struct musb *musb, void __iomem *mregs) 1301{ 1302 struct cppi *controller; 1303 struct device *dev = musb->controller; 1304 struct platform_device *pdev = to_platform_device(dev); 1305 int irq = platform_get_irq_byname(pdev, "dma"); 1306 1307 controller = kzalloc(sizeof *controller, GFP_KERNEL); 1308 if (!controller) 1309 return NULL; 1310 1311 controller->mregs = mregs; 1312 controller->tibase = mregs - DAVINCI_BASE_OFFSET; 1313 1314 controller->musb = musb; 1315 controller->controller.channel_alloc = cppi_channel_allocate; 1316 controller->controller.channel_release = cppi_channel_release; 1317 controller->controller.channel_program = cppi_channel_program; 1318 controller->controller.channel_abort = cppi_channel_abort; 1319 1320 /* NOTE: allocating from on-chip SRAM would give the least 1321 * contention for memory access, if that ever matters here. 1322 */ 1323 1324 /* setup BufferPool */ 1325 controller->pool = dma_pool_create("cppi", 1326 controller->musb->controller, 1327 sizeof(struct cppi_descriptor), 1328 CPPI_DESCRIPTOR_ALIGN, 0); 1329 if (!controller->pool) { 1330 kfree(controller); 1331 return NULL; 1332 } 1333 1334 if (irq > 0) { 1335 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { 1336 dev_err(dev, "request_irq %d failed!\n", irq); 1337 dma_controller_destroy(&controller->controller); 1338 return NULL; 1339 } 1340 controller->irq = irq; 1341 } 1342 1343 cppi_controller_start(controller); 1344 return &controller->controller; 1345} 1346 1347/* 1348 * Destroy a previously-instantiated DMA controller. 1349 */ 1350void dma_controller_destroy(struct dma_controller *c) 1351{ 1352 struct cppi *cppi; 1353 1354 cppi = container_of(c, struct cppi, controller); 1355 1356 cppi_controller_stop(cppi); 1357 1358 if (cppi->irq) 1359 free_irq(cppi->irq, cppi->musb); 1360 1361 /* assert: caller stopped the controller first */ 1362 dma_pool_destroy(cppi->pool); 1363 1364 kfree(cppi); 1365} 1366 1367/* 1368 * Context: controller irqlocked, endpoint selected 1369 */ 1370static int cppi_channel_abort(struct dma_channel *channel) 1371{ 1372 struct cppi_channel *cppi_ch; 1373 struct cppi *controller; 1374 void __iomem *mbase; 1375 void __iomem *tibase; 1376 void __iomem *regs; 1377 u32 value; 1378 struct cppi_descriptor *queue; 1379 1380 cppi_ch = container_of(channel, struct cppi_channel, channel); 1381 1382 controller = cppi_ch->controller; 1383 1384 switch (channel->status) { 1385 case MUSB_DMA_STATUS_BUS_ABORT: 1386 case MUSB_DMA_STATUS_CORE_ABORT: 1387 /* from RX or TX fault irq handler */ 1388 case MUSB_DMA_STATUS_BUSY: 1389 /* the hardware needs shutting down */ 1390 regs = cppi_ch->hw_ep->regs; 1391 break; 1392 case MUSB_DMA_STATUS_UNKNOWN: 1393 case MUSB_DMA_STATUS_FREE: 1394 return 0; 1395 default: 1396 return -EINVAL; 1397 } 1398 1399 if (!cppi_ch->transmit && cppi_ch->head) 1400 cppi_dump_rxq(3, "/abort", cppi_ch); 1401 1402 mbase = controller->mregs; 1403 tibase = controller->tibase; 1404 1405 queue = cppi_ch->head; 1406 cppi_ch->head = NULL; 1407 cppi_ch->tail = NULL; 1408 1409 /* REVISIT should rely on caller having done this, 1410 * and caller should rely on us not changing it. 1411 * peripheral code is safe ... check host too. 1412 */ 1413 musb_ep_select(mbase, cppi_ch->index + 1); 1414 1415 if (cppi_ch->transmit) { 1416 struct cppi_tx_stateram __iomem *tx_ram; 1417 /* REVISIT put timeouts on these controller handshakes */ 1418 1419 cppi_dump_tx(6, cppi_ch, " (teardown)"); 1420 1421 /* teardown DMA engine then usb core */ 1422 do { 1423 value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); 1424 } while (!(value & CPPI_TEAR_READY)); 1425 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); 1426 1427 tx_ram = cppi_ch->state_ram; 1428 do { 1429 value = musb_readl(&tx_ram->tx_complete, 0); 1430 } while (0xFFFFFFFC != value); 1431 1432 /* FIXME clean up the transfer state ... here? 1433 * the completion routine should get called with 1434 * an appropriate status code. 1435 */ 1436 1437 value = musb_readw(regs, MUSB_TXCSR); 1438 value &= ~MUSB_TXCSR_DMAENAB; 1439 value |= MUSB_TXCSR_FLUSHFIFO; 1440 musb_writew(regs, MUSB_TXCSR, value); 1441 musb_writew(regs, MUSB_TXCSR, value); 1442 1443 /* 1444 * 1. Write to completion Ptr value 0x1(bit 0 set) 1445 * (write back mode) 1446 * 2. Wait for abort interrupt and then put the channel in 1447 * compare mode by writing 1 to the tx_complete register. 1448 */ 1449 cppi_reset_tx(tx_ram, 1); 1450 cppi_ch->head = NULL; 1451 musb_writel(&tx_ram->tx_complete, 0, 1); 1452 cppi_dump_tx(5, cppi_ch, " (done teardown)"); 1453 1454 /* REVISIT tx side _should_ clean up the same way 1455 * as the RX side ... this does no cleanup at all! 1456 */ 1457 1458 } else /* RX */ { 1459 u16 csr; 1460 1461 /* NOTE: docs don't guarantee any of this works ... we 1462 * expect that if the usb core stops telling the cppi core 1463 * to pull more data from it, then it'll be safe to flush 1464 * current RX DMA state iff any pending fifo transfer is done. 1465 */ 1466 1467 core_rxirq_disable(tibase, cppi_ch->index + 1); 1468 1469 /* for host, ensure ReqPkt is never set again */ 1470 if (is_host_active(cppi_ch->controller->musb)) { 1471 value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); 1472 value &= ~((0x3) << (cppi_ch->index * 2)); 1473 musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); 1474 } 1475 1476 csr = musb_readw(regs, MUSB_RXCSR); 1477 1478 /* for host, clear (just) ReqPkt at end of current packet(s) */ 1479 if (is_host_active(cppi_ch->controller->musb)) { 1480 csr |= MUSB_RXCSR_H_WZC_BITS; 1481 csr &= ~MUSB_RXCSR_H_REQPKT; 1482 } else 1483 csr |= MUSB_RXCSR_P_WZC_BITS; 1484 1485 /* clear dma enable */ 1486 csr &= ~(MUSB_RXCSR_DMAENAB); 1487 musb_writew(regs, MUSB_RXCSR, csr); 1488 csr = musb_readw(regs, MUSB_RXCSR); 1489 1490 /* Quiesce: wait for current dma to finish (if not cleanup). 1491 * We can't use bit zero of stateram->rx_sop, since that 1492 * refers to an entire "DMA packet" not just emptying the 1493 * current fifo. Most segments need multiple usb packets. 1494 */ 1495 if (channel->status == MUSB_DMA_STATUS_BUSY) 1496 udelay(50); 1497 1498 /* scan the current list, reporting any data that was 1499 * transferred and acking any IRQ 1500 */ 1501 cppi_rx_scan(controller, cppi_ch->index); 1502 1503 /* clobber the existing state once it's idle 1504 * 1505 * NOTE: arguably, we should also wait for all the other 1506 * RX channels to quiesce (how??) and then temporarily 1507 * disable RXCPPI_CTRL_REG ... but it seems that we can 1508 * rely on the controller restarting from state ram, with 1509 * only RXCPPI_BUFCNT state being bogus. BUFCNT will 1510 * correct itself after the next DMA transfer though. 1511 * 1512 * REVISIT does using rndis mode change that? 1513 */ 1514 cppi_reset_rx(cppi_ch->state_ram); 1515 1516 /* next DMA request _should_ load cppi head ptr */ 1517 1518 /* ... we don't "free" that list, only mutate it in place. */ 1519 cppi_dump_rx(5, cppi_ch, " (done abort)"); 1520 1521 /* clean up previously pending bds */ 1522 cppi_bd_free(cppi_ch, cppi_ch->last_processed); 1523 cppi_ch->last_processed = NULL; 1524 1525 while (queue) { 1526 struct cppi_descriptor *tmp = queue->next; 1527 1528 cppi_bd_free(cppi_ch, queue); 1529 queue = tmp; 1530 } 1531 } 1532 1533 channel->status = MUSB_DMA_STATUS_FREE; 1534 cppi_ch->buf_dma = 0; 1535 cppi_ch->offset = 0; 1536 cppi_ch->buf_len = 0; 1537 cppi_ch->maxpacket = 0; 1538 return 0; 1539} 1540 1541/* TBD Queries: 1542 * 1543 * Power Management ... probably turn off cppi during suspend, restart; 1544 * check state ram? Clocking is presumably shared with usb core. 1545 */ 1546