diff options
Diffstat (limited to 'kernel/drivers/usb/musb/cppi_dma.c')
| -rw-r--r-- | kernel/drivers/usb/musb/cppi_dma.c | 1545 |
1 files changed, 1545 insertions, 0 deletions
diff --git a/kernel/drivers/usb/musb/cppi_dma.c b/kernel/drivers/usb/musb/cppi_dma.c new file mode 100644 index 000000000..904fb85d8 --- /dev/null +++ b/kernel/drivers/usb/musb/cppi_dma.c @@ -0,0 +1,1545 @@ +/* + * Copyright (C) 2005-2006 by Texas Instruments + * + * This file implements a DMA interface using TI's CPPI DMA. + * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB. + * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci. + */ + +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/usb.h> + +#include "musb_core.h" +#include "musb_debug.h" +#include "cppi_dma.h" + + +/* CPPI DMA status 7-mar-2006: + * + * - See musb_{host,gadget}.c for more info + * + * - Correct RX DMA generally forces the engine into irq-per-packet mode, + * which can easily saturate the CPU under non-mass-storage loads. + * + * NOTES 24-aug-2006 (2.6.18-rc4): + * + * - peripheral RXDMA wedged in a test with packets of length 512/512/1. + * evidently after the 1 byte packet was received and acked, the queue + * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003, + * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401 + * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx + * of its next (512 byte) packet. IRQ issues? + * + * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will + * evidently also directly update the RX and TX CSRs ... so audit all + * host and peripheral side DMA code to avoid CSR access after DMA has + * been started. + */ + +/* REVISIT now we can avoid preallocating these descriptors; or + * more simply, switch to a global freelist not per-channel ones. + * Note: at full speed, 64 descriptors == 4K bulk data. + */ +#define NUM_TXCHAN_BD 64 +#define NUM_RXCHAN_BD 64 + +static inline void cpu_drain_writebuffer(void) +{ + wmb(); +#ifdef CONFIG_CPU_ARM926T + /* REVISIT this "should not be needed", + * but lack of it sure seemed to hurt ... + */ + asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n"); +#endif +} + +static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c) +{ + struct cppi_descriptor *bd = c->freelist; + + if (bd) + c->freelist = bd->next; + return bd; +} + +static inline void +cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd) +{ + if (!bd) + return; + bd->next = c->freelist; + c->freelist = bd; +} + +/* + * Start DMA controller + * + * Initialize the DMA controller as necessary. + */ + +/* zero out entire rx state RAM entry for the channel */ +static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) +{ + musb_writel(&rx->rx_skipbytes, 0, 0); + musb_writel(&rx->rx_head, 0, 0); + musb_writel(&rx->rx_sop, 0, 0); + musb_writel(&rx->rx_current, 0, 0); + musb_writel(&rx->rx_buf_current, 0, 0); + musb_writel(&rx->rx_len_len, 0, 0); + musb_writel(&rx->rx_cnt_cnt, 0, 0); +} + +/* zero out entire tx state RAM entry for the channel */ +static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) +{ + musb_writel(&tx->tx_head, 0, 0); + musb_writel(&tx->tx_buf, 0, 0); + musb_writel(&tx->tx_current, 0, 0); + musb_writel(&tx->tx_buf_current, 0, 0); + musb_writel(&tx->tx_info, 0, 0); + musb_writel(&tx->tx_rem_len, 0, 0); + /* musb_writel(&tx->tx_dummy, 0, 0); */ + musb_writel(&tx->tx_complete, 0, ptr); +} + +static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c) +{ + int j; + + /* initialize channel fields */ + c->head = NULL; + c->tail = NULL; + c->last_processed = NULL; + c->channel.status = MUSB_DMA_STATUS_UNKNOWN; + c->controller = cppi; + c->is_rndis = 0; + c->freelist = NULL; + + /* build the BD Free list for the channel */ + for (j = 0; j < NUM_TXCHAN_BD + 1; j++) { + struct cppi_descriptor *bd; + dma_addr_t dma; + + bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); + bd->dma = dma; + cppi_bd_free(c, bd); + } +} + +static int cppi_channel_abort(struct dma_channel *); + +static void cppi_pool_free(struct cppi_channel *c) +{ + struct cppi *cppi = c->controller; + struct cppi_descriptor *bd; + + (void) cppi_channel_abort(&c->channel); + c->channel.status = MUSB_DMA_STATUS_UNKNOWN; + c->controller = NULL; + + /* free all its bds */ + bd = c->last_processed; + do { + if (bd) + dma_pool_free(cppi->pool, bd, bd->dma); + bd = cppi_bd_alloc(c); + } while (bd); + c->last_processed = NULL; +} + +static void cppi_controller_start(struct cppi *controller) +{ + void __iomem *tibase; + int i; + + /* do whatever is necessary to start controller */ + for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { + controller->tx[i].transmit = true; + controller->tx[i].index = i; + } + for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { + controller->rx[i].transmit = false; + controller->rx[i].index = i; + } + + /* setup BD list on a per channel basis */ + for (i = 0; i < ARRAY_SIZE(controller->tx); i++) + cppi_pool_init(controller, controller->tx + i); + for (i = 0; i < ARRAY_SIZE(controller->rx); i++) + cppi_pool_init(controller, controller->rx + i); + + tibase = controller->tibase; + INIT_LIST_HEAD(&controller->tx_complete); + + /* initialise tx/rx channel head pointers to zero */ + for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { + struct cppi_channel *tx_ch = controller->tx + i; + struct cppi_tx_stateram __iomem *tx; + + INIT_LIST_HEAD(&tx_ch->tx_complete); + + tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); + tx_ch->state_ram = tx; + cppi_reset_tx(tx, 0); + } + for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { + struct cppi_channel *rx_ch = controller->rx + i; + struct cppi_rx_stateram __iomem *rx; + + INIT_LIST_HEAD(&rx_ch->tx_complete); + + rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); + rx_ch->state_ram = rx; + cppi_reset_rx(rx); + } + + /* enable individual cppi channels */ + musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, + DAVINCI_DMA_ALL_CHANNELS_ENABLE); + musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG, + DAVINCI_DMA_ALL_CHANNELS_ENABLE); + + /* enable tx/rx CPPI control */ + musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); + musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); + + /* disable RNDIS mode, also host rx RNDIS autorequest */ + musb_writel(tibase, DAVINCI_RNDIS_REG, 0); + musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0); +} + +/* + * Stop DMA controller + * + * De-Init the DMA controller as necessary. + */ + +static void cppi_controller_stop(struct cppi *controller) +{ + void __iomem *tibase; + int i; + struct musb *musb; + + musb = controller->musb; + + tibase = controller->tibase; + /* DISABLE INDIVIDUAL CHANNEL Interrupts */ + musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, + DAVINCI_DMA_ALL_CHANNELS_ENABLE); + musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG, + DAVINCI_DMA_ALL_CHANNELS_ENABLE); + + dev_dbg(musb->controller, "Tearing down RX and TX Channels\n"); + for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { + /* FIXME restructure of txdma to use bds like rxdma */ + controller->tx[i].last_processed = NULL; + cppi_pool_free(controller->tx + i); + } + for (i = 0; i < ARRAY_SIZE(controller->rx); i++) + cppi_pool_free(controller->rx + i); + + /* in Tx Case proper teardown is supported. We resort to disabling + * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is + * complete TX CPPI cannot be disabled. + */ + /*disable tx/rx cppi */ + musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); + musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); +} + +/* While dma channel is allocated, we only want the core irqs active + * for fault reports, otherwise we'd get irqs that we don't care about. + * Except for TX irqs, where dma done != fifo empty and reusable ... + * + * NOTE: docs don't say either way, but irq masking **enables** irqs. + * + * REVISIT same issue applies to pure PIO usage too, and non-cppi dma... + */ +static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum) +{ + musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8)); +} + +static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum) +{ + musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8)); +} + + +/* + * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to + * each transfer direction of a non-control endpoint, so allocating + * (and deallocating) is mostly a way to notice bad housekeeping on + * the software side. We assume the irqs are always active. + */ +static struct dma_channel * +cppi_channel_allocate(struct dma_controller *c, + struct musb_hw_ep *ep, u8 transmit) +{ + struct cppi *controller; + u8 index; + struct cppi_channel *cppi_ch; + void __iomem *tibase; + struct musb *musb; + + controller = container_of(c, struct cppi, controller); + tibase = controller->tibase; + musb = controller->musb; + + /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ + index = ep->epnum - 1; + + /* return the corresponding CPPI Channel Handle, and + * probably disable the non-CPPI irq until we need it. + */ + if (transmit) { + if (index >= ARRAY_SIZE(controller->tx)) { + dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index); + return NULL; + } + cppi_ch = controller->tx + index; + } else { + if (index >= ARRAY_SIZE(controller->rx)) { + dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index); + return NULL; + } + cppi_ch = controller->rx + index; + core_rxirq_disable(tibase, ep->epnum); + } + + /* REVISIT make this an error later once the same driver code works + * with the other DMA engine too + */ + if (cppi_ch->hw_ep) + dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n", + index, transmit ? 'T' : 'R', cppi_ch); + cppi_ch->hw_ep = ep; + cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; + cppi_ch->channel.max_len = 0x7fffffff; + + dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R'); + return &cppi_ch->channel; +} + +/* Release a CPPI Channel. */ +static void cppi_channel_release(struct dma_channel *channel) +{ + struct cppi_channel *c; + void __iomem *tibase; + + /* REVISIT: for paranoia, check state and abort if needed... */ + + c = container_of(channel, struct cppi_channel, channel); + tibase = c->controller->tibase; + if (!c->hw_ep) + dev_dbg(c->controller->musb->controller, + "releasing idle DMA channel %p\n", c); + else if (!c->transmit) + core_rxirq_enable(tibase, c->index + 1); + + /* for now, leave its cppi IRQ enabled (we won't trigger it) */ + c->hw_ep = NULL; + channel->status = MUSB_DMA_STATUS_UNKNOWN; +} + +/* Context: controller irqlocked */ +static void +cppi_dump_rx(int level, struct cppi_channel *c, const char *tag) +{ + void __iomem *base = c->controller->mregs; + struct cppi_rx_stateram __iomem *rx = c->state_ram; + + musb_ep_select(base, c->index + 1); + + dev_dbg(c->controller->musb->controller, + "RX DMA%d%s: %d left, csr %04x, " + "%08x H%08x S%08x C%08x, " + "B%08x L%08x %08x .. %08x" + "\n", + c->index, tag, + musb_readl(c->controller->tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), + musb_readw(c->hw_ep->regs, MUSB_RXCSR), + + musb_readl(&rx->rx_skipbytes, 0), + musb_readl(&rx->rx_head, 0), + musb_readl(&rx->rx_sop, 0), + musb_readl(&rx->rx_current, 0), + + musb_readl(&rx->rx_buf_current, 0), + musb_readl(&rx->rx_len_len, 0), + musb_readl(&rx->rx_cnt_cnt, 0), + musb_readl(&rx->rx_complete, 0) + ); +} + +/* Context: controller irqlocked */ +static void +cppi_dump_tx(int level, struct cppi_channel *c, const char *tag) +{ + void __iomem *base = c->controller->mregs; + struct cppi_tx_stateram __iomem *tx = c->state_ram; + + musb_ep_select(base, c->index + 1); + + dev_dbg(c->controller->musb->controller, + "TX DMA%d%s: csr %04x, " + "H%08x S%08x C%08x %08x, " + "F%08x L%08x .. %08x" + "\n", + c->index, tag, + musb_readw(c->hw_ep->regs, MUSB_TXCSR), + + musb_readl(&tx->tx_head, 0), + musb_readl(&tx->tx_buf, 0), + musb_readl(&tx->tx_current, 0), + musb_readl(&tx->tx_buf_current, 0), + + musb_readl(&tx->tx_info, 0), + musb_readl(&tx->tx_rem_len, 0), + /* dummy/unused word 6 */ + musb_readl(&tx->tx_complete, 0) + ); +} + +/* Context: controller irqlocked */ +static inline void +cppi_rndis_update(struct cppi_channel *c, int is_rx, + void __iomem *tibase, int is_rndis) +{ + /* we may need to change the rndis flag for this cppi channel */ + if (c->is_rndis != is_rndis) { + u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG); + u32 temp = 1 << (c->index); + + if (is_rx) + temp <<= 16; + if (is_rndis) + value |= temp; + else + value &= ~temp; + musb_writel(tibase, DAVINCI_RNDIS_REG, value); + c->is_rndis = is_rndis; + } +} + +static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd) +{ + pr_debug("RXBD/%s %08x: " + "nxt %08x buf %08x off.blen %08x opt.plen %08x\n", + tag, bd->dma, + bd->hw_next, bd->hw_bufp, bd->hw_off_len, + bd->hw_options); +} + +static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) +{ + struct cppi_descriptor *bd; + + cppi_dump_rx(level, rx, tag); + if (rx->last_processed) + cppi_dump_rxbd("last", rx->last_processed); + for (bd = rx->head; bd; bd = bd->next) + cppi_dump_rxbd("active", bd); +} + + +/* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX; + * so we won't ever use it (see "CPPI RX Woes" below). + */ +static inline int cppi_autoreq_update(struct cppi_channel *rx, + void __iomem *tibase, int onepacket, unsigned n_bds) +{ + u32 val; + +#ifdef RNDIS_RX_IS_USABLE + u32 tmp; + /* assert(is_host_active(musb)) */ + + /* start from "AutoReq never" */ + tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); + val = tmp & ~((0x3) << (rx->index * 2)); + + /* HCD arranged reqpkt for packet #1. we arrange int + * for all but the last one, maybe in two segments. + */ + if (!onepacket) { +#if 0 + /* use two segments, autoreq "all" then the last "never" */ + val |= ((0x3) << (rx->index * 2)); + n_bds--; +#else + /* one segment, autoreq "all-but-last" */ + val |= ((0x1) << (rx->index * 2)); +#endif + } + + if (val != tmp) { + int n = 100; + + /* make sure that autoreq is updated before continuing */ + musb_writel(tibase, DAVINCI_AUTOREQ_REG, val); + do { + tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); + if (tmp == val) + break; + cpu_relax(); + } while (n-- > 0); + } +#endif + + /* REQPKT is turned off after each segment */ + if (n_bds && rx->channel.actual_len) { + void __iomem *regs = rx->hw_ep->regs; + + val = musb_readw(regs, MUSB_RXCSR); + if (!(val & MUSB_RXCSR_H_REQPKT)) { + val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS; + musb_writew(regs, MUSB_RXCSR, val); + /* flush writebuffer */ + val = musb_readw(regs, MUSB_RXCSR); + } + } + return n_bds; +} + + +/* Buffer enqueuing Logic: + * + * - RX builds new queues each time, to help handle routine "early + * termination" cases (faults, including errors and short reads) + * more correctly. + * + * - for now, TX reuses the same queue of BDs every time + * + * REVISIT long term, we want a normal dynamic model. + * ... the goal will be to append to the + * existing queue, processing completed "dma buffers" (segments) on the fly. + * + * Otherwise we force an IRQ latency between requests, which slows us a lot + * (especially in "transparent" dma). Unfortunately that model seems to be + * inherent in the DMA model from the Mentor code, except in the rare case + * of transfers big enough (~128+ KB) that we could append "middle" segments + * in the TX paths. (RX can't do this, see below.) + * + * That's true even in the CPPI- friendly iso case, where most urbs have + * several small segments provided in a group and where the "packet at a time" + * "transparent" DMA model is always correct, even on the RX side. + */ + +/* + * CPPI TX: + * ======== + * TX is a lot more reasonable than RX; it doesn't need to run in + * irq-per-packet mode very often. RNDIS mode seems to behave too + * (except how it handles the exactly-N-packets case). Building a + * txdma queue with multiple requests (urb or usb_request) looks + * like it would work ... but fault handling would need much testing. + * + * The main issue with TX mode RNDIS relates to transfer lengths that + * are an exact multiple of the packet length. It appears that there's + * a hiccup in that case (maybe the DMA completes before the ZLP gets + * written?) boiling down to not being able to rely on CPPI writing any + * terminating zero length packet before the next transfer is written. + * So that's punted to PIO; better yet, gadget drivers can avoid it. + * + * Plus, there's allegedly an undocumented constraint that rndis transfer + * length be a multiple of 64 bytes ... but the chip doesn't act that + * way, and we really don't _want_ that behavior anyway. + * + * On TX, "transparent" mode works ... although experiments have shown + * problems trying to use the SOP/EOP bits in different USB packets. + * + * REVISIT try to handle terminating zero length packets using CPPI + * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet + * links avoid that issue by forcing them to avoid zlps.) + */ +static void +cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) +{ + unsigned maxpacket = tx->maxpacket; + dma_addr_t addr = tx->buf_dma + tx->offset; + size_t length = tx->buf_len - tx->offset; + struct cppi_descriptor *bd; + unsigned n_bds; + unsigned i; + struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; + int rndis; + + /* TX can use the CPPI "rndis" mode, where we can probably fit this + * transfer in one BD and one IRQ. The only time we would NOT want + * to use it is when hardware constraints prevent it, or if we'd + * trigger the "send a ZLP?" confusion. + */ + rndis = (maxpacket & 0x3f) == 0 + && length > maxpacket + && length < 0xffff + && (length % maxpacket) != 0; + + if (rndis) { + maxpacket = length; + n_bds = 1; + } else { + n_bds = length / maxpacket; + if (!length || (length % maxpacket)) + n_bds++; + n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD); + length = min(n_bds * maxpacket, length); + } + + dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n", + tx->index, + maxpacket, + rndis ? "rndis" : "transparent", + n_bds, + (unsigned long long)addr, length); + + cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); + + /* assuming here that channel_program is called during + * transfer initiation ... current code maintains state + * for one outstanding request only (no queues, not even + * the implicit ones of an iso urb). + */ + + bd = tx->freelist; + tx->head = bd; + tx->last_processed = NULL; + + /* FIXME use BD pool like RX side does, and just queue + * the minimum number for this request. + */ + + /* Prepare queue of BDs first, then hand it to hardware. + * All BDs except maybe the last should be of full packet + * size; for RNDIS there _is_ only that last packet. + */ + for (i = 0; i < n_bds; ) { + if (++i < n_bds && bd->next) + bd->hw_next = bd->next->dma; + else + bd->hw_next = 0; + + bd->hw_bufp = tx->buf_dma + tx->offset; + + /* FIXME set EOP only on the last packet, + * SOP only on the first ... avoid IRQs + */ + if ((tx->offset + maxpacket) <= tx->buf_len) { + tx->offset += maxpacket; + bd->hw_off_len = maxpacket; + bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET + | CPPI_OWN_SET | maxpacket; + } else { + /* only this one may be a partial USB Packet */ + u32 partial_len; + + partial_len = tx->buf_len - tx->offset; + tx->offset = tx->buf_len; + bd->hw_off_len = partial_len; + + bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET + | CPPI_OWN_SET | partial_len; + if (partial_len == 0) + bd->hw_options |= CPPI_ZERO_SET; + } + + dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n", + bd, bd->hw_next, bd->hw_bufp, + bd->hw_off_len, bd->hw_options); + + /* update the last BD enqueued to the list */ + tx->tail = bd; + bd = bd->next; + } + + /* BDs live in DMA-coherent memory, but writes might be pending */ + cpu_drain_writebuffer(); + + /* Write to the HeadPtr in state RAM to trigger */ + musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); + + cppi_dump_tx(5, tx, "/S"); +} + +/* + * CPPI RX Woes: + * ============= + * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte + * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back. + * (Full speed transfers have similar scenarios.) + * + * The correct behavior for Linux is that (a) fills the buffer with 300 bytes, + * and the next packet goes into a buffer that's queued later; while (b) fills + * the buffer with 1024 bytes. How to do that with CPPI? + * + * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but + * (b) loses **BADLY** because nothing (!) happens when that second packet + * fills the buffer, much less when a third one arrives. (Which makes this + * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination + * is optional, and it's fine if peripherals -- not hosts! -- pad messages + * out to end-of-buffer. Standard PCI host controller DMA descriptors + * implement that mode by default ... which is no accident.) + * + * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have + * converse problems: (b) is handled right, but (a) loses badly. CPPI RX + * ignores SOP/EOP markings and processes both of those BDs; so both packets + * are loaded into the buffer (with a 212 byte gap between them), and the next + * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP + * are intended as outputs for RX queues, not inputs...) + * + * - A variant of "transparent" mode -- one BD at a time -- is the only way to + * reliably make both cases work, with software handling both cases correctly + * and at the significant penalty of needing an IRQ per packet. (The lack of + * I/O overlap can be slightly ameliorated by enabling double buffering.) + * + * So how to get rid of IRQ-per-packet? The transparent multi-BD case could + * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK + * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors + * with guaranteed driver level fault recovery and scrubbing out what's left + * of that garbaged datastream. + * + * But there seems to be no way to identify the cases where CPPI RNDIS mode + * is appropriate -- which do NOT include RNDIS host drivers, but do include + * the CDC Ethernet driver! -- and the documentation is incomplete/wrong. + * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic + * that applies best on the peripheral side (and which could fail rudely). + * + * Leaving only "transparent" mode; we avoid multi-bd modes in almost all + * cases other than mass storage class. Otherwise we're correct but slow, + * since CPPI penalizes our need for a "true RNDIS" default mode. + */ + + +/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY + * + * IFF + * (a) peripheral mode ... since rndis peripherals could pad their + * writes to hosts, causing i/o failure; or we'd have to cope with + * a largely unknowable variety of host side protocol variants + * (b) and short reads are NOT errors ... since full reads would + * cause those same i/o failures + * (c) and read length is + * - less than 64KB (max per cppi descriptor) + * - not a multiple of 4096 (g_zero default, full reads typical) + * - N (>1) packets long, ditto (full reads not EXPECTED) + * THEN + * try rx rndis mode + * + * Cost of heuristic failing: RXDMA wedges at the end of transfers that + * fill out the whole buffer. Buggy host side usb network drivers could + * trigger that, but "in the field" such bugs seem to be all but unknown. + * + * So this module parameter lets the heuristic be disabled. When using + * gadgetfs, the heuristic will probably need to be disabled. + */ +static bool cppi_rx_rndis = 1; + +module_param(cppi_rx_rndis, bool, 0); +MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic"); + + +/** + * cppi_next_rx_segment - dma read for the next chunk of a buffer + * @musb: the controller + * @rx: dma channel + * @onepacket: true unless caller treats short reads as errors, and + * performs fault recovery above usbcore. + * Context: controller irqlocked + * + * See above notes about why we can't use multi-BD RX queues except in + * rare cases (mass storage class), and can never use the hardware "rndis" + * mode (since it's not a "true" RNDIS mode) with complete safety.. + * + * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in + * code to recover from corrupted datastreams after each short transfer. + */ +static void +cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) +{ + unsigned maxpacket = rx->maxpacket; + dma_addr_t addr = rx->buf_dma + rx->offset; + size_t length = rx->buf_len - rx->offset; + struct cppi_descriptor *bd, *tail; + unsigned n_bds; + unsigned i; + void __iomem *tibase = musb->ctrl_base; + int is_rndis = 0; + struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; + struct cppi_descriptor *d; + + if (onepacket) { + /* almost every USB driver, host or peripheral side */ + n_bds = 1; + + /* maybe apply the heuristic above */ + if (cppi_rx_rndis + && is_peripheral_active(musb) + && length > maxpacket + && (length & ~0xffff) == 0 + && (length & 0x0fff) != 0 + && (length & (maxpacket - 1)) == 0) { + maxpacket = length; + is_rndis = 1; + } + } else { + /* virtually nothing except mass storage class */ + if (length > 0xffff) { + n_bds = 0xffff / maxpacket; + length = n_bds * maxpacket; + } else { + n_bds = length / maxpacket; + if (length % maxpacket) + n_bds++; + } + if (n_bds == 1) + onepacket = 1; + else + n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); + } + + /* In host mode, autorequest logic can generate some IN tokens; it's + * tricky since we can't leave REQPKT set in RXCSR after the transfer + * finishes. So: multipacket transfers involve two or more segments. + * And always at least two IRQs ... RNDIS mode is not an option. + */ + if (is_host_active(musb)) + n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); + + cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); + + length = min(n_bds * maxpacket, length); + + dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " + "dma 0x%llx len %u %u/%u\n", + rx->index, maxpacket, + onepacket + ? (is_rndis ? "rndis" : "onepacket") + : "multipacket", + n_bds, + musb_readl(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) + & 0xffff, + (unsigned long long)addr, length, + rx->channel.actual_len, rx->buf_len); + + /* only queue one segment at a time, since the hardware prevents + * correct queue shutdown after unexpected short packets + */ + bd = cppi_bd_alloc(rx); + rx->head = bd; + + /* Build BDs for all packets in this segment */ + for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { + u32 bd_len; + + if (i) { + bd = cppi_bd_alloc(rx); + if (!bd) + break; + tail->next = bd; + tail->hw_next = bd->dma; + } + bd->hw_next = 0; + + /* all but the last packet will be maxpacket size */ + if (maxpacket < length) + bd_len = maxpacket; + else + bd_len = length; + + bd->hw_bufp = addr; + addr += bd_len; + rx->offset += bd_len; + + bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; + bd->buflen = bd_len; + + bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); + length -= bd_len; + } + + /* we always expect at least one reusable BD! */ + if (!tail) { + WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); + return; + } else if (i < n_bds) + WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); + + tail->next = NULL; + tail->hw_next = 0; + + bd = rx->head; + rx->tail = tail; + + /* short reads and other faults should terminate this entire + * dma segment. we want one "dma packet" per dma segment, not + * one per USB packet, terminating the whole queue at once... + * NOTE that current hardware seems to ignore SOP and EOP. + */ + bd->hw_options |= CPPI_SOP_SET; + tail->hw_options |= CPPI_EOP_SET; + + for (d = rx->head; d; d = d->next) + cppi_dump_rxbd("S", d); + + /* in case the preceding transfer left some state... */ + tail = rx->last_processed; + if (tail) { + tail->next = bd; + tail->hw_next = bd->dma; + } + + core_rxirq_enable(tibase, rx->index + 1); + + /* BDs live in DMA-coherent memory, but writes might be pending */ + cpu_drain_writebuffer(); + + /* REVISIT specs say to write this AFTER the BUFCNT register + * below ... but that loses badly. + */ + musb_writel(&rx_ram->rx_head, 0, bd->dma); + + /* bufferCount must be at least 3, and zeroes on completion + * unless it underflows below zero, or stops at two, or keeps + * growing ... grr. + */ + i = musb_readl(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) + & 0xffff; + + if (!i) + musb_writel(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), + n_bds + 2); + else if (n_bds > (i - 3)) + musb_writel(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), + n_bds - (i - 3)); + + i = musb_readl(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) + & 0xffff; + if (i < (2 + n_bds)) { + dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n", + rx->index, i, n_bds); + musb_writel(tibase, + DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), + n_bds + 2); + } + + cppi_dump_rx(4, rx, "/S"); +} + +/** + * cppi_channel_program - program channel for data transfer + * @ch: the channel + * @maxpacket: max packet size + * @mode: For RX, 1 unless the usb protocol driver promised to treat + * all short reads as errors and kick in high level fault recovery. + * For TX, ignored because of RNDIS mode races/glitches. + * @dma_addr: dma address of buffer + * @len: length of buffer + * Context: controller irqlocked + */ +static int cppi_channel_program(struct dma_channel *ch, + u16 maxpacket, u8 mode, + dma_addr_t dma_addr, u32 len) +{ + struct cppi_channel *cppi_ch; + struct cppi *controller; + struct musb *musb; + + cppi_ch = container_of(ch, struct cppi_channel, channel); + controller = cppi_ch->controller; + musb = controller->musb; + + switch (ch->status) { + case MUSB_DMA_STATUS_BUS_ABORT: + case MUSB_DMA_STATUS_CORE_ABORT: + /* fault irq handler should have handled cleanup */ + WARNING("%cX DMA%d not cleaned up after abort!\n", + cppi_ch->transmit ? 'T' : 'R', + cppi_ch->index); + /* WARN_ON(1); */ + break; + case MUSB_DMA_STATUS_BUSY: + WARNING("program active channel? %cX DMA%d\n", + cppi_ch->transmit ? 'T' : 'R', + cppi_ch->index); + /* WARN_ON(1); */ + break; + case MUSB_DMA_STATUS_UNKNOWN: + dev_dbg(musb->controller, "%cX DMA%d not allocated!\n", + cppi_ch->transmit ? 'T' : 'R', + cppi_ch->index); + /* FALLTHROUGH */ + case MUSB_DMA_STATUS_FREE: + break; + } + + ch->status = MUSB_DMA_STATUS_BUSY; + + /* set transfer parameters, then queue up its first segment */ + cppi_ch->buf_dma = dma_addr; + cppi_ch->offset = 0; + cppi_ch->maxpacket = maxpacket; + cppi_ch->buf_len = len; + cppi_ch->channel.actual_len = 0; + + /* TX channel? or RX? */ + if (cppi_ch->transmit) + cppi_next_tx_segment(musb, cppi_ch); + else + cppi_next_rx_segment(musb, cppi_ch, mode); + + return true; +} + +static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) +{ + struct cppi_channel *rx = &cppi->rx[ch]; + struct cppi_rx_stateram __iomem *state = rx->state_ram; + struct cppi_descriptor *bd; + struct cppi_descriptor *last = rx->last_processed; + bool completed = false; + bool acked = false; + int i; + dma_addr_t safe2ack; + void __iomem *regs = rx->hw_ep->regs; + struct musb *musb = cppi->musb; + + cppi_dump_rx(6, rx, "/K"); + + bd = last ? last->next : rx->head; + if (!bd) + return false; + + /* run through all completed BDs */ + for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); + (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; + i++, bd = bd->next) { + u16 len; + + /* catch latest BD writes from CPPI */ + rmb(); + if (!completed && (bd->hw_options & CPPI_OWN_SET)) + break; + + dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x " + "off.len %08x opt.len %08x (%d)\n", + (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, + bd->hw_off_len, bd->hw_options, + rx->channel.actual_len); + + /* actual packet received length */ + if ((bd->hw_options & CPPI_SOP_SET) && !completed) + len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; + else + len = 0; + + if (bd->hw_options & CPPI_EOQ_MASK) + completed = true; + + if (!completed && len < bd->buflen) { + /* NOTE: when we get a short packet, RXCSR_H_REQPKT + * must have been cleared, and no more DMA packets may + * active be in the queue... TI docs didn't say, but + * CPPI ignores those BDs even though OWN is still set. + */ + completed = true; + dev_dbg(musb->controller, "rx short %d/%d (%d)\n", + len, bd->buflen, + rx->channel.actual_len); + } + + /* If we got here, we expect to ack at least one BD; meanwhile + * CPPI may completing other BDs while we scan this list... + * + * RACE: we can notice OWN cleared before CPPI raises the + * matching irq by writing that BD as the completion pointer. + * In such cases, stop scanning and wait for the irq, avoiding + * lost acks and states where BD ownership is unclear. + */ + if (bd->dma == safe2ack) { + musb_writel(&state->rx_complete, 0, safe2ack); + safe2ack = musb_readl(&state->rx_complete, 0); + acked = true; + if (bd->dma == safe2ack) + safe2ack = 0; + } + + rx->channel.actual_len += len; + + cppi_bd_free(rx, last); + last = bd; + + /* stop scanning on end-of-segment */ + if (bd->hw_next == 0) + completed = true; + } + rx->last_processed = last; + + /* dma abort, lost ack, or ... */ + if (!acked && last) { + int csr; + + if (safe2ack == 0 || safe2ack == rx->last_processed->dma) + musb_writel(&state->rx_complete, 0, safe2ack); + if (safe2ack == 0) { + cppi_bd_free(rx, last); + rx->last_processed = NULL; + + /* if we land here on the host side, H_REQPKT will + * be clear and we need to restart the queue... + */ + WARN_ON(rx->head); + } + musb_ep_select(cppi->mregs, rx->index + 1); + csr = musb_readw(regs, MUSB_RXCSR); + if (csr & MUSB_RXCSR_DMAENAB) { + dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n", + rx->index, + rx->head, rx->tail, + rx->last_processed + ? (unsigned long long) + rx->last_processed->dma + : 0, + completed ? ", completed" : "", + csr); + cppi_dump_rxq(4, "/what?", rx); + } + } + if (!completed) { + int csr; + + rx->head = bd; + + /* REVISIT seems like "autoreq all but EOP" doesn't... + * setting it here "should" be racey, but seems to work + */ + csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); + if (is_host_active(cppi->musb) + && bd + && !(csr & MUSB_RXCSR_H_REQPKT)) { + csr |= MUSB_RXCSR_H_REQPKT; + musb_writew(regs, MUSB_RXCSR, + MUSB_RXCSR_H_WZC_BITS | csr); + csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); + } + } else { + rx->head = NULL; + rx->tail = NULL; + } + + cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); + return completed; +} + +irqreturn_t cppi_interrupt(int irq, void *dev_id) +{ + struct musb *musb = dev_id; + struct cppi *cppi; + void __iomem *tibase; + struct musb_hw_ep *hw_ep = NULL; + u32 rx, tx; + int i, index; + unsigned long uninitialized_var(flags); + + cppi = container_of(musb->dma_controller, struct cppi, controller); + if (cppi->irq) + spin_lock_irqsave(&musb->lock, flags); + + tibase = musb->ctrl_base; + + tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); + rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); + + if (!tx && !rx) { + if (cppi->irq) + spin_unlock_irqrestore(&musb->lock, flags); + return IRQ_NONE; + } + + dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx); + + /* process TX channels */ + for (index = 0; tx; tx = tx >> 1, index++) { + struct cppi_channel *tx_ch; + struct cppi_tx_stateram __iomem *tx_ram; + bool completed = false; + struct cppi_descriptor *bd; + + if (!(tx & 1)) + continue; + + tx_ch = cppi->tx + index; + tx_ram = tx_ch->state_ram; + + /* FIXME need a cppi_tx_scan() routine, which + * can also be called from abort code + */ + + cppi_dump_tx(5, tx_ch, "/E"); + + bd = tx_ch->head; + + /* + * If Head is null then this could mean that a abort interrupt + * that needs to be acknowledged. + */ + if (NULL == bd) { + dev_dbg(musb->controller, "null BD\n"); + musb_writel(&tx_ram->tx_complete, 0, 0); + continue; + } + + /* run through all completed BDs */ + for (i = 0; !completed && bd && i < NUM_TXCHAN_BD; + i++, bd = bd->next) { + u16 len; + + /* catch latest BD writes from CPPI */ + rmb(); + if (bd->hw_options & CPPI_OWN_SET) + break; + + dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n", + bd, bd->hw_next, bd->hw_bufp, + bd->hw_off_len, bd->hw_options); + + len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; + tx_ch->channel.actual_len += len; + + tx_ch->last_processed = bd; + + /* write completion register to acknowledge + * processing of completed BDs, and possibly + * release the IRQ; EOQ might not be set ... + * + * REVISIT use the same ack strategy as rx + * + * REVISIT have observed bit 18 set; huh?? + */ + /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ + musb_writel(&tx_ram->tx_complete, 0, bd->dma); + + /* stop scanning on end-of-segment */ + if (bd->hw_next == 0) + completed = true; + } + + /* on end of segment, maybe go to next one */ + if (completed) { + /* cppi_dump_tx(4, tx_ch, "/complete"); */ + + /* transfer more, or report completion */ + if (tx_ch->offset >= tx_ch->buf_len) { + tx_ch->head = NULL; + tx_ch->tail = NULL; + tx_ch->channel.status = MUSB_DMA_STATUS_FREE; + + hw_ep = tx_ch->hw_ep; + + musb_dma_completion(musb, index + 1, 1); + + } else { + /* Bigger transfer than we could fit in + * that first batch of descriptors... + */ + cppi_next_tx_segment(musb, tx_ch); + } + } else + tx_ch->head = bd; + } + + /* Start processing the RX block */ + for (index = 0; rx; rx = rx >> 1, index++) { + + if (rx & 1) { + struct cppi_channel *rx_ch; + + rx_ch = cppi->rx + index; + + /* let incomplete dma segments finish */ + if (!cppi_rx_scan(cppi, index)) + continue; + + /* start another dma segment if needed */ + if (rx_ch->channel.actual_len != rx_ch->buf_len + && rx_ch->channel.actual_len + == rx_ch->offset) { + cppi_next_rx_segment(musb, rx_ch, 1); + continue; + } + + /* all segments completed! */ + rx_ch->channel.status = MUSB_DMA_STATUS_FREE; + + hw_ep = rx_ch->hw_ep; + + core_rxirq_disable(tibase, index + 1); + musb_dma_completion(musb, index + 1, 0); + } + } + + /* write to CPPI EOI register to re-enable interrupts */ + musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0); + + if (cppi->irq) + spin_unlock_irqrestore(&musb->lock, flags); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(cppi_interrupt); + +/* Instantiate a software object representing a DMA controller. */ +struct dma_controller *dma_controller_create(struct musb *musb, void __iomem *mregs) +{ + struct cppi *controller; + struct device *dev = musb->controller; + struct platform_device *pdev = to_platform_device(dev); + int irq = platform_get_irq_byname(pdev, "dma"); + + controller = kzalloc(sizeof *controller, GFP_KERNEL); + if (!controller) + return NULL; + + controller->mregs = mregs; + controller->tibase = mregs - DAVINCI_BASE_OFFSET; + + controller->musb = musb; + controller->controller.channel_alloc = cppi_channel_allocate; + controller->controller.channel_release = cppi_channel_release; + controller->controller.channel_program = cppi_channel_program; + controller->controller.channel_abort = cppi_channel_abort; + + /* NOTE: allocating from on-chip SRAM would give the least + * contention for memory access, if that ever matters here. + */ + + /* setup BufferPool */ + controller->pool = dma_pool_create("cppi", + controller->musb->controller, + sizeof(struct cppi_descriptor), + CPPI_DESCRIPTOR_ALIGN, 0); + if (!controller->pool) { + kfree(controller); + return NULL; + } + + if (irq > 0) { + if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { + dev_err(dev, "request_irq %d failed!\n", irq); + dma_controller_destroy(&controller->controller); + return NULL; + } + controller->irq = irq; + } + + cppi_controller_start(controller); + return &controller->controller; +} + +/* + * Destroy a previously-instantiated DMA controller. + */ +void dma_controller_destroy(struct dma_controller *c) +{ + struct cppi *cppi; + + cppi = container_of(c, struct cppi, controller); + + cppi_controller_stop(cppi); + + if (cppi->irq) + free_irq(cppi->irq, cppi->musb); + + /* assert: caller stopped the controller first */ + dma_pool_destroy(cppi->pool); + + kfree(cppi); +} + +/* + * Context: controller irqlocked, endpoint selected + */ +static int cppi_channel_abort(struct dma_channel *channel) +{ + struct cppi_channel *cppi_ch; + struct cppi *controller; + void __iomem *mbase; + void __iomem *tibase; + void __iomem *regs; + u32 value; + struct cppi_descriptor *queue; + + cppi_ch = container_of(channel, struct cppi_channel, channel); + + controller = cppi_ch->controller; + + switch (channel->status) { + case MUSB_DMA_STATUS_BUS_ABORT: + case MUSB_DMA_STATUS_CORE_ABORT: + /* from RX or TX fault irq handler */ + case MUSB_DMA_STATUS_BUSY: + /* the hardware needs shutting down */ + regs = cppi_ch->hw_ep->regs; + break; + case MUSB_DMA_STATUS_UNKNOWN: + case MUSB_DMA_STATUS_FREE: + return 0; + default: + return -EINVAL; + } + + if (!cppi_ch->transmit && cppi_ch->head) + cppi_dump_rxq(3, "/abort", cppi_ch); + + mbase = controller->mregs; + tibase = controller->tibase; + + queue = cppi_ch->head; + cppi_ch->head = NULL; + cppi_ch->tail = NULL; + + /* REVISIT should rely on caller having done this, + * and caller should rely on us not changing it. + * peripheral code is safe ... check host too. + */ + musb_ep_select(mbase, cppi_ch->index + 1); + + if (cppi_ch->transmit) { + struct cppi_tx_stateram __iomem *tx_ram; + /* REVISIT put timeouts on these controller handshakes */ + + cppi_dump_tx(6, cppi_ch, " (teardown)"); + + /* teardown DMA engine then usb core */ + do { + value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); + } while (!(value & CPPI_TEAR_READY)); + musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); + + tx_ram = cppi_ch->state_ram; + do { + value = musb_readl(&tx_ram->tx_complete, 0); + } while (0xFFFFFFFC != value); + + /* FIXME clean up the transfer state ... here? + * the completion routine should get called with + * an appropriate status code. + */ + + value = musb_readw(regs, MUSB_TXCSR); + value &= ~MUSB_TXCSR_DMAENAB; + value |= MUSB_TXCSR_FLUSHFIFO; + musb_writew(regs, MUSB_TXCSR, value); + musb_writew(regs, MUSB_TXCSR, value); + + /* + * 1. Write to completion Ptr value 0x1(bit 0 set) + * (write back mode) + * 2. Wait for abort interrupt and then put the channel in + * compare mode by writing 1 to the tx_complete register. + */ + cppi_reset_tx(tx_ram, 1); + cppi_ch->head = NULL; + musb_writel(&tx_ram->tx_complete, 0, 1); + cppi_dump_tx(5, cppi_ch, " (done teardown)"); + + /* REVISIT tx side _should_ clean up the same way + * as the RX side ... this does no cleanup at all! + */ + + } else /* RX */ { + u16 csr; + + /* NOTE: docs don't guarantee any of this works ... we + * expect that if the usb core stops telling the cppi core + * to pull more data from it, then it'll be safe to flush + * current RX DMA state iff any pending fifo transfer is done. + */ + + core_rxirq_disable(tibase, cppi_ch->index + 1); + + /* for host, ensure ReqPkt is never set again */ + if (is_host_active(cppi_ch->controller->musb)) { + value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); + value &= ~((0x3) << (cppi_ch->index * 2)); + musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); + } + + csr = musb_readw(regs, MUSB_RXCSR); + + /* for host, clear (just) ReqPkt at end of current packet(s) */ + if (is_host_active(cppi_ch->controller->musb)) { + csr |= MUSB_RXCSR_H_WZC_BITS; + csr &= ~MUSB_RXCSR_H_REQPKT; + } else + csr |= MUSB_RXCSR_P_WZC_BITS; + + /* clear dma enable */ + csr &= ~(MUSB_RXCSR_DMAENAB); + musb_writew(regs, MUSB_RXCSR, csr); + csr = musb_readw(regs, MUSB_RXCSR); + + /* Quiesce: wait for current dma to finish (if not cleanup). + * We can't use bit zero of stateram->rx_sop, since that + * refers to an entire "DMA packet" not just emptying the + * current fifo. Most segments need multiple usb packets. + */ + if (channel->status == MUSB_DMA_STATUS_BUSY) + udelay(50); + + /* scan the current list, reporting any data that was + * transferred and acking any IRQ + */ + cppi_rx_scan(controller, cppi_ch->index); + + /* clobber the existing state once it's idle + * + * NOTE: arguably, we should also wait for all the other + * RX channels to quiesce (how??) and then temporarily + * disable RXCPPI_CTRL_REG ... but it seems that we can + * rely on the controller restarting from state ram, with + * only RXCPPI_BUFCNT state being bogus. BUFCNT will + * correct itself after the next DMA transfer though. + * + * REVISIT does using rndis mode change that? + */ + cppi_reset_rx(cppi_ch->state_ram); + + /* next DMA request _should_ load cppi head ptr */ + + /* ... we don't "free" that list, only mutate it in place. */ + cppi_dump_rx(5, cppi_ch, " (done abort)"); + + /* clean up previously pending bds */ + cppi_bd_free(cppi_ch, cppi_ch->last_processed); + cppi_ch->last_processed = NULL; + + while (queue) { + struct cppi_descriptor *tmp = queue->next; + + cppi_bd_free(cppi_ch, queue); + queue = tmp; + } + } + + channel->status = MUSB_DMA_STATUS_FREE; + cppi_ch->buf_dma = 0; + cppi_ch->offset = 0; + cppi_ch->buf_len = 0; + cppi_ch->maxpacket = 0; + return 0; +} + +/* TBD Queries: + * + * Power Management ... probably turn off cppi during suspend, restart; + * check state ram? Clocking is presumably shared with usb core. + */ |