diff options
Diffstat (limited to 'kernel/drivers/mmc/host/dw_mmc.c')
| -rw-r--r-- | kernel/drivers/mmc/host/dw_mmc.c | 635 |
1 files changed, 444 insertions, 191 deletions
diff --git a/kernel/drivers/mmc/host/dw_mmc.c b/kernel/drivers/mmc/host/dw_mmc.c index 5f5adafb2..7a6cedbe4 100644 --- a/kernel/drivers/mmc/host/dw_mmc.c +++ b/kernel/drivers/mmc/host/dw_mmc.c @@ -56,7 +56,6 @@ #define DW_MCI_FREQ_MAX 200000000 /* unit: HZ */ #define DW_MCI_FREQ_MIN 400000 /* unit: HZ */ -#ifdef CONFIG_MMC_DW_IDMAC #define IDMAC_INT_CLR (SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \ SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \ SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \ @@ -99,7 +98,9 @@ struct idmac_desc { __le32 des3; /* buffer 2 physical address */ }; -#endif /* CONFIG_MMC_DW_IDMAC */ + +/* Each descriptor can transfer up to 4KB of data in chained mode */ +#define DW_MCI_DESC_DATA_LENGTH 0x1000 static bool dw_mci_reset(struct dw_mci *host); static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset); @@ -235,8 +236,8 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd) struct dw_mci *host = slot->host; const struct dw_mci_drv_data *drv_data = slot->host->drv_data; u32 cmdr; - cmd->error = -EINPROGRESS; + cmd->error = -EINPROGRESS; cmdr = cmd->opcode; if (cmd->opcode == MMC_STOP_TRANSMISSION || @@ -371,7 +372,7 @@ static void dw_mci_start_command(struct dw_mci *host, cmd->arg, cmd_flags); mci_writel(host, CMDARG, cmd->arg); - wmb(); + wmb(); /* drain writebuffer */ dw_mci_wait_while_busy(host, cmd_flags); mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START); @@ -380,6 +381,7 @@ static void dw_mci_start_command(struct dw_mci *host, static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data) { struct mmc_command *stop = data->stop ? data->stop : &host->stop_abort; + dw_mci_start_command(host, stop, host->stop_cmdr); } @@ -403,7 +405,6 @@ static int dw_mci_get_dma_dir(struct mmc_data *data) return DMA_FROM_DEVICE; } -#ifdef CONFIG_MMC_DW_IDMAC static void dw_mci_dma_cleanup(struct dw_mci *host) { struct mmc_data *data = host->data; @@ -441,12 +442,21 @@ static void dw_mci_idmac_stop_dma(struct dw_mci *host) mci_writel(host, BMOD, temp); } -static void dw_mci_idmac_complete_dma(struct dw_mci *host) +static void dw_mci_dmac_complete_dma(void *arg) { + struct dw_mci *host = arg; struct mmc_data *data = host->data; dev_vdbg(host->dev, "DMA complete\n"); + if ((host->use_dma == TRANS_MODE_EDMAC) && + data && (data->flags & MMC_DATA_READ)) + /* Invalidate cache after read */ + dma_sync_sg_for_cpu(mmc_dev(host->cur_slot->mmc), + data->sg, + data->sg_len, + DMA_FROM_DEVICE); + host->dma_ops->cleanup(host); /* @@ -462,72 +472,105 @@ static void dw_mci_idmac_complete_dma(struct dw_mci *host) static void dw_mci_translate_sglist(struct dw_mci *host, struct mmc_data *data, unsigned int sg_len) { + unsigned int desc_len; int i; + if (host->dma_64bit_address == 1) { - struct idmac_desc_64addr *desc = host->sg_cpu; + struct idmac_desc_64addr *desc_first, *desc_last, *desc; + + desc_first = desc_last = desc = host->sg_cpu; - for (i = 0; i < sg_len; i++, desc++) { + for (i = 0; i < sg_len; i++) { unsigned int length = sg_dma_len(&data->sg[i]); + u64 mem_addr = sg_dma_address(&data->sg[i]); - /* - * Set the OWN bit and disable interrupts for this - * descriptor - */ - desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | - IDMAC_DES0_CH; - /* Buffer length */ - IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, length); - - /* Physical address to DMA to/from */ - desc->des4 = mem_addr & 0xffffffff; - desc->des5 = mem_addr >> 32; + for ( ; length ; desc++) { + desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ? + length : DW_MCI_DESC_DATA_LENGTH; + + length -= desc_len; + + /* + * Set the OWN bit and disable interrupts + * for this descriptor + */ + desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | + IDMAC_DES0_CH; + + /* Buffer length */ + IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len); + + /* Physical address to DMA to/from */ + desc->des4 = mem_addr & 0xffffffff; + desc->des5 = mem_addr >> 32; + + /* Update physical address for the next desc */ + mem_addr += desc_len; + + /* Save pointer to the last descriptor */ + desc_last = desc; + } } /* Set first descriptor */ - desc = host->sg_cpu; - desc->des0 |= IDMAC_DES0_FD; + desc_first->des0 |= IDMAC_DES0_FD; /* Set last descriptor */ - desc = host->sg_cpu + (i - 1) * - sizeof(struct idmac_desc_64addr); - desc->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC); - desc->des0 |= IDMAC_DES0_LD; + desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC); + desc_last->des0 |= IDMAC_DES0_LD; } else { - struct idmac_desc *desc = host->sg_cpu; + struct idmac_desc *desc_first, *desc_last, *desc; + + desc_first = desc_last = desc = host->sg_cpu; - for (i = 0; i < sg_len; i++, desc++) { + for (i = 0; i < sg_len; i++) { unsigned int length = sg_dma_len(&data->sg[i]); + u32 mem_addr = sg_dma_address(&data->sg[i]); - /* - * Set the OWN bit and disable interrupts for this - * descriptor - */ - desc->des0 = cpu_to_le32(IDMAC_DES0_OWN | - IDMAC_DES0_DIC | IDMAC_DES0_CH); - /* Buffer length */ - IDMAC_SET_BUFFER1_SIZE(desc, length); + for ( ; length ; desc++) { + desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ? + length : DW_MCI_DESC_DATA_LENGTH; + + length -= desc_len; + + /* + * Set the OWN bit and disable interrupts + * for this descriptor + */ + desc->des0 = cpu_to_le32(IDMAC_DES0_OWN | + IDMAC_DES0_DIC | + IDMAC_DES0_CH); + + /* Buffer length */ + IDMAC_SET_BUFFER1_SIZE(desc, desc_len); - /* Physical address to DMA to/from */ - desc->des2 = cpu_to_le32(mem_addr); + /* Physical address to DMA to/from */ + desc->des2 = cpu_to_le32(mem_addr); + + /* Update physical address for the next desc */ + mem_addr += desc_len; + + /* Save pointer to the last descriptor */ + desc_last = desc; + } } /* Set first descriptor */ - desc = host->sg_cpu; - desc->des0 |= cpu_to_le32(IDMAC_DES0_FD); + desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD); /* Set last descriptor */ - desc = host->sg_cpu + (i - 1) * sizeof(struct idmac_desc); - desc->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC)); - desc->des0 |= cpu_to_le32(IDMAC_DES0_LD); + desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | + IDMAC_DES0_DIC)); + desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD); } - wmb(); + wmb(); /* drain writebuffer */ } -static void dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) +static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) { u32 temp; @@ -542,6 +585,7 @@ static void dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) temp |= SDMMC_CTRL_USE_IDMAC; mci_writel(host, CTRL, temp); + /* drain writebuffer */ wmb(); /* Enable the IDMAC */ @@ -551,6 +595,8 @@ static void dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) /* Start it running */ mci_writel(host, PLDMND, 1); + + return 0; } static int dw_mci_idmac_init(struct dw_mci *host) @@ -589,7 +635,9 @@ static int dw_mci_idmac_init(struct dw_mci *host) host->ring_size = PAGE_SIZE / sizeof(struct idmac_desc); /* Forward link the descriptor list */ - for (i = 0, p = host->sg_cpu; i < host->ring_size - 1; i++, p++) { + for (i = 0, p = host->sg_cpu; + i < host->ring_size - 1; + i++, p++) { p->des3 = cpu_to_le32(host->sg_dma + (sizeof(struct idmac_desc) * (i + 1))); p->des1 = 0; @@ -629,10 +677,110 @@ static const struct dw_mci_dma_ops dw_mci_idmac_ops = { .init = dw_mci_idmac_init, .start = dw_mci_idmac_start_dma, .stop = dw_mci_idmac_stop_dma, - .complete = dw_mci_idmac_complete_dma, + .complete = dw_mci_dmac_complete_dma, + .cleanup = dw_mci_dma_cleanup, +}; + +static void dw_mci_edmac_stop_dma(struct dw_mci *host) +{ + dmaengine_terminate_all(host->dms->ch); +} + +static int dw_mci_edmac_start_dma(struct dw_mci *host, + unsigned int sg_len) +{ + struct dma_slave_config cfg; + struct dma_async_tx_descriptor *desc = NULL; + struct scatterlist *sgl = host->data->sg; + const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; + u32 sg_elems = host->data->sg_len; + u32 fifoth_val; + u32 fifo_offset = host->fifo_reg - host->regs; + int ret = 0; + + /* Set external dma config: burst size, burst width */ + cfg.dst_addr = (dma_addr_t)(host->phy_regs + fifo_offset); + cfg.src_addr = cfg.dst_addr; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + /* Match burst msize with external dma config */ + fifoth_val = mci_readl(host, FIFOTH); + cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7]; + cfg.src_maxburst = cfg.dst_maxburst; + + if (host->data->flags & MMC_DATA_WRITE) + cfg.direction = DMA_MEM_TO_DEV; + else + cfg.direction = DMA_DEV_TO_MEM; + + ret = dmaengine_slave_config(host->dms->ch, &cfg); + if (ret) { + dev_err(host->dev, "Failed to config edmac.\n"); + return -EBUSY; + } + + desc = dmaengine_prep_slave_sg(host->dms->ch, sgl, + sg_len, cfg.direction, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) { + dev_err(host->dev, "Can't prepare slave sg.\n"); + return -EBUSY; + } + + /* Set dw_mci_dmac_complete_dma as callback */ + desc->callback = dw_mci_dmac_complete_dma; + desc->callback_param = (void *)host; + dmaengine_submit(desc); + + /* Flush cache before write */ + if (host->data->flags & MMC_DATA_WRITE) + dma_sync_sg_for_device(mmc_dev(host->cur_slot->mmc), sgl, + sg_elems, DMA_TO_DEVICE); + + dma_async_issue_pending(host->dms->ch); + + return 0; +} + +static int dw_mci_edmac_init(struct dw_mci *host) +{ + /* Request external dma channel */ + host->dms = kzalloc(sizeof(struct dw_mci_dma_slave), GFP_KERNEL); + if (!host->dms) + return -ENOMEM; + + host->dms->ch = dma_request_slave_channel(host->dev, "rx-tx"); + if (!host->dms->ch) { + dev_err(host->dev, "Failed to get external DMA channel.\n"); + kfree(host->dms); + host->dms = NULL; + return -ENXIO; + } + + return 0; +} + +static void dw_mci_edmac_exit(struct dw_mci *host) +{ + if (host->dms) { + if (host->dms->ch) { + dma_release_channel(host->dms->ch); + host->dms->ch = NULL; + } + kfree(host->dms); + host->dms = NULL; + } +} + +static const struct dw_mci_dma_ops dw_mci_edmac_ops = { + .init = dw_mci_edmac_init, + .exit = dw_mci_edmac_exit, + .start = dw_mci_edmac_start_dma, + .stop = dw_mci_edmac_stop_dma, + .complete = dw_mci_dmac_complete_dma, .cleanup = dw_mci_dma_cleanup, }; -#endif /* CONFIG_MMC_DW_IDMAC */ static int dw_mci_pre_dma_transfer(struct dw_mci *host, struct mmc_data *data, @@ -712,13 +860,16 @@ static void dw_mci_post_req(struct mmc_host *mmc, static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data) { -#ifdef CONFIG_MMC_DW_IDMAC unsigned int blksz = data->blksz; const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; u32 fifo_width = 1 << host->data_shift; u32 blksz_depth = blksz / fifo_width, fifoth_val; u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers; - int idx = (sizeof(mszs) / sizeof(mszs[0])) - 1; + int idx = ARRAY_SIZE(mszs) - 1; + + /* pio should ship this scenario */ + if (!host->use_dma) + return; tx_wmark = (host->fifo_depth) / 2; tx_wmark_invers = host->fifo_depth - tx_wmark; @@ -748,7 +899,6 @@ static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data) done: fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark); mci_writel(host, FIFOTH, fifoth_val); -#endif } static void dw_mci_ctrl_rd_thld(struct dw_mci *host, struct mmc_data *data) @@ -810,10 +960,12 @@ static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) host->using_dma = 1; - dev_vdbg(host->dev, - "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n", - (unsigned long)host->sg_cpu, (unsigned long)host->sg_dma, - sg_len); + if (host->use_dma == TRANS_MODE_IDMAC) + dev_vdbg(host->dev, + "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n", + (unsigned long)host->sg_cpu, + (unsigned long)host->sg_dma, + sg_len); /* * Decide the MSIZE and RX/TX Watermark. @@ -835,7 +987,11 @@ static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) mci_writel(host, INTMASK, temp); spin_unlock_irqrestore(&host->irq_lock, irqflags); - host->dma_ops->start(host, sg_len); + if (host->dma_ops->start(host, sg_len)) { + /* We can't do DMA */ + dev_err(host->dev, "%s: failed to start DMA.\n", __func__); + return -ENODEV; + } return 0; } @@ -843,6 +999,7 @@ static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data) { unsigned long irqflags; + int flags = SG_MITER_ATOMIC; u32 temp; data->error = -EINPROGRESS; @@ -859,7 +1016,6 @@ static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data) } if (dw_mci_submit_data_dma(host, data)) { - int flags = SG_MITER_ATOMIC; if (host->data->flags & MMC_DATA_READ) flags |= SG_MITER_TO_SG; else @@ -906,7 +1062,7 @@ static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg) unsigned int cmd_status = 0; mci_writel(host, CMDARG, arg); - wmb(); + wmb(); /* drain writebuffer */ dw_mci_wait_while_busy(host, cmd); mci_writel(host, CMD, SDMMC_CMD_START | cmd); @@ -1019,7 +1175,7 @@ static void __dw_mci_start_request(struct dw_mci *host, if (data) { dw_mci_submit_data(host, data); - wmb(); + wmb(); /* drain writebuffer */ } dw_mci_start_command(host, cmd, cmdflags); @@ -1137,6 +1293,7 @@ static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) /* DDR mode set */ if (ios->timing == MMC_TIMING_MMC_DDR52 || + ios->timing == MMC_TIMING_UHS_DDR50 || ios->timing == MMC_TIMING_MMC_HS400) regs |= ((0x1 << slot->id) << 16); else @@ -1236,33 +1393,32 @@ static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios) { struct dw_mci_slot *slot = mmc_priv(mmc); struct dw_mci *host = slot->host; + const struct dw_mci_drv_data *drv_data = host->drv_data; u32 uhs; u32 v18 = SDMMC_UHS_18V << slot->id; - int min_uv, max_uv; int ret; + if (drv_data && drv_data->switch_voltage) + return drv_data->switch_voltage(mmc, ios); + /* * Program the voltage. Note that some instances of dw_mmc may use * the UHS_REG for this. For other instances (like exynos) the UHS_REG * does no harm but you need to set the regulator directly. Try both. */ uhs = mci_readl(host, UHS_REG); - if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) { - min_uv = 2700000; - max_uv = 3600000; + if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) uhs &= ~v18; - } else { - min_uv = 1700000; - max_uv = 1950000; + else uhs |= v18; - } + if (!IS_ERR(mmc->supply.vqmmc)) { - ret = regulator_set_voltage(mmc->supply.vqmmc, min_uv, max_uv); + ret = mmc_regulator_set_vqmmc(mmc, ios); if (ret) { dev_dbg(&mmc->class_dev, - "Regulator set error %d: %d - %d\n", - ret, min_uv, max_uv); + "Regulator set error %d - %s V\n", + ret, uhs & v18 ? "1.8" : "3.3"); return ret; } } @@ -1278,10 +1434,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc) int gpio_ro = mmc_gpio_get_ro(mmc); /* Use platform get_ro function, else try on board write protect */ - if ((slot->quirks & DW_MCI_SLOT_QUIRK_NO_WRITE_PROTECT) || - (slot->host->quirks & DW_MCI_QUIRK_NO_WRITE_PROTECT)) - read_only = 0; - else if (!IS_ERR_VALUE(gpio_ro)) + if (!IS_ERR_VALUE(gpio_ro)) read_only = gpio_ro; else read_only = @@ -1383,14 +1536,15 @@ static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode) struct dw_mci_slot *slot = mmc_priv(mmc); struct dw_mci *host = slot->host; const struct dw_mci_drv_data *drv_data = host->drv_data; - int err = -ENOSYS; + int err = -EINVAL; if (drv_data && drv_data->execute_tuning) - err = drv_data->execute_tuning(slot); + err = drv_data->execute_tuning(slot, opcode); return err; } -static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios) +static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc, + struct mmc_ios *ios) { struct dw_mci_slot *slot = mmc_priv(mmc); struct dw_mci *host = slot->host; @@ -1532,6 +1686,20 @@ static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data) return data->error; } +static void dw_mci_set_drto(struct dw_mci *host) +{ + unsigned int drto_clks; + unsigned int drto_ms; + + drto_clks = mci_readl(host, TMOUT) >> 8; + drto_ms = DIV_ROUND_UP(drto_clks, host->bus_hz / 1000); + + /* add a bit spare time */ + drto_ms += 10; + + mod_timer(&host->dto_timer, jiffies + msecs_to_jiffies(drto_ms)); +} + static void dw_mci_tasklet_func(unsigned long priv) { struct dw_mci *host = (struct dw_mci *)priv; @@ -1609,8 +1777,16 @@ static void dw_mci_tasklet_func(unsigned long priv) } if (!test_and_clear_bit(EVENT_XFER_COMPLETE, - &host->pending_events)) + &host->pending_events)) { + /* + * If all data-related interrupts don't come + * within the given time in reading data state. + */ + if ((host->quirks & DW_MCI_QUIRK_BROKEN_DTO) && + (host->dir_status == DW_MCI_RECV_STATUS)) + dw_mci_set_drto(host); break; + } set_bit(EVENT_XFER_COMPLETE, &host->completed_events); @@ -1643,8 +1819,17 @@ static void dw_mci_tasklet_func(unsigned long priv) case STATE_DATA_BUSY: if (!test_and_clear_bit(EVENT_DATA_COMPLETE, - &host->pending_events)) + &host->pending_events)) { + /* + * If data error interrupt comes but data over + * interrupt doesn't come within the given time. + * in reading data state. + */ + if ((host->quirks & DW_MCI_QUIRK_BROKEN_DTO) && + (host->dir_status == DW_MCI_RECV_STATUS)) + dw_mci_set_drto(host); break; + } host->data = NULL; set_bit(EVENT_DATA_COMPLETE, &host->completed_events); @@ -1742,7 +1927,7 @@ static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt) /* pull first bytes from part_buf, only use during pull */ static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt) { - cnt = min(cnt, (int)host->part_buf_count); + cnt = min_t(int, cnt, host->part_buf_count); if (cnt) { memcpy(buf, (void *)&host->part_buf + host->part_buf_start, cnt); @@ -1768,6 +1953,7 @@ static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt) /* try and push anything in the part_buf */ if (unlikely(host->part_buf_count)) { int len = dw_mci_push_part_bytes(host, buf, cnt); + buf += len; cnt -= len; if (host->part_buf_count == 2) { @@ -1794,6 +1980,7 @@ static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt) #endif { u16 *pdata = buf; + for (; cnt >= 2; cnt -= 2) mci_fifo_writew(host->fifo_reg, *pdata++); buf = pdata; @@ -1818,6 +2005,7 @@ static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt) int len = min(cnt & -2, (int)sizeof(aligned_buf)); int items = len >> 1; int i; + for (i = 0; i < items; ++i) aligned_buf[i] = mci_fifo_readw(host->fifo_reg); /* memcpy from aligned buffer into output buffer */ @@ -1829,6 +2017,7 @@ static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt) #endif { u16 *pdata = buf; + for (; cnt >= 2; cnt -= 2) *pdata++ = mci_fifo_readw(host->fifo_reg); buf = pdata; @@ -1847,6 +2036,7 @@ static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt) /* try and push anything in the part_buf */ if (unlikely(host->part_buf_count)) { int len = dw_mci_push_part_bytes(host, buf, cnt); + buf += len; cnt -= len; if (host->part_buf_count == 4) { @@ -1873,6 +2063,7 @@ static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt) #endif { u32 *pdata = buf; + for (; cnt >= 4; cnt -= 4) mci_fifo_writel(host->fifo_reg, *pdata++); buf = pdata; @@ -1897,6 +2088,7 @@ static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt) int len = min(cnt & -4, (int)sizeof(aligned_buf)); int items = len >> 2; int i; + for (i = 0; i < items; ++i) aligned_buf[i] = mci_fifo_readl(host->fifo_reg); /* memcpy from aligned buffer into output buffer */ @@ -1908,6 +2100,7 @@ static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt) #endif { u32 *pdata = buf; + for (; cnt >= 4; cnt -= 4) *pdata++ = mci_fifo_readl(host->fifo_reg); buf = pdata; @@ -1926,6 +2119,7 @@ static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt) /* try and push anything in the part_buf */ if (unlikely(host->part_buf_count)) { int len = dw_mci_push_part_bytes(host, buf, cnt); + buf += len; cnt -= len; @@ -1953,6 +2147,7 @@ static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt) #endif { u64 *pdata = buf; + for (; cnt >= 8; cnt -= 8) mci_fifo_writeq(host->fifo_reg, *pdata++); buf = pdata; @@ -1977,6 +2172,7 @@ static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt) int len = min(cnt & -8, (int)sizeof(aligned_buf)); int items = len >> 3; int i; + for (i = 0; i < items; ++i) aligned_buf[i] = mci_fifo_readq(host->fifo_reg); @@ -1989,6 +2185,7 @@ static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt) #endif { u64 *pdata = buf; + for (; cnt >= 8; cnt -= 8) *pdata++ = mci_fifo_readq(host->fifo_reg); buf = pdata; @@ -2064,7 +2261,7 @@ static void dw_mci_read_data_pio(struct dw_mci *host, bool dto) done: sg_miter_stop(sg_miter); host->sg = NULL; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ set_bit(EVENT_XFER_COMPLETE, &host->pending_events); } @@ -2118,7 +2315,7 @@ static void dw_mci_write_data_pio(struct dw_mci *host) done: sg_miter_stop(sg_miter); host->sg = NULL; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ set_bit(EVENT_XFER_COMPLETE, &host->pending_events); } @@ -2127,7 +2324,7 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status) if (!host->cmd_status) host->cmd_status = status; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ set_bit(EVENT_CMD_COMPLETE, &host->pending_events); tasklet_schedule(&host->tasklet); @@ -2191,7 +2388,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) if (pending & DW_MCI_CMD_ERROR_FLAGS) { mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS); host->cmd_status = pending; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ set_bit(EVENT_CMD_COMPLETE, &host->pending_events); } @@ -2199,16 +2396,19 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) /* if there is an error report DATA_ERROR */ mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS); host->data_status = pending; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ set_bit(EVENT_DATA_ERROR, &host->pending_events); tasklet_schedule(&host->tasklet); } if (pending & SDMMC_INT_DATA_OVER) { + if (host->quirks & DW_MCI_QUIRK_BROKEN_DTO) + del_timer(&host->dto_timer); + mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER); if (!host->data_status) host->data_status = pending; - smp_wmb(); + smp_wmb(); /* drain writebuffer */ if (host->dir_status == DW_MCI_RECV_STATUS) { if (host->sg != NULL) dw_mci_read_data_pio(host, true); @@ -2255,15 +2455,17 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) } -#ifdef CONFIG_MMC_DW_IDMAC - /* Handle DMA interrupts */ + if (host->use_dma != TRANS_MODE_IDMAC) + return IRQ_HANDLED; + + /* Handle IDMA interrupts */ if (host->dma_64bit_address == 1) { pending = mci_readl(host, IDSTS64); if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI); mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI); - host->dma_ops->complete(host); + host->dma_ops->complete((void *)host); } } else { pending = mci_readl(host, IDSTS); @@ -2271,18 +2473,18 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI); mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI); - host->dma_ops->complete(host); + host->dma_ops->complete((void *)host); } } -#endif return IRQ_HANDLED; } #ifdef CONFIG_OF -/* given a slot id, find out the device node representing that slot */ -static struct device_node *dw_mci_of_find_slot_node(struct device *dev, u8 slot) +/* given a slot, find out the device node representing that slot */ +static struct device_node *dw_mci_of_find_slot_node(struct dw_mci_slot *slot) { + struct device *dev = slot->mmc->parent; struct device_node *np; const __be32 *addr; int len; @@ -2294,42 +2496,28 @@ static struct device_node *dw_mci_of_find_slot_node(struct device *dev, u8 slot) addr = of_get_property(np, "reg", &len); if (!addr || (len < sizeof(int))) continue; - if (be32_to_cpup(addr) == slot) + if (be32_to_cpup(addr) == slot->id) return np; } return NULL; } -static struct dw_mci_of_slot_quirks { - char *quirk; - int id; -} of_slot_quirks[] = { - { - .quirk = "disable-wp", - .id = DW_MCI_SLOT_QUIRK_NO_WRITE_PROTECT, - }, -}; - -static int dw_mci_of_get_slot_quirks(struct device *dev, u8 slot) +static void dw_mci_slot_of_parse(struct dw_mci_slot *slot) { - struct device_node *np = dw_mci_of_find_slot_node(dev, slot); - int quirks = 0; - int idx; + struct device_node *np = dw_mci_of_find_slot_node(slot); - /* get quirks */ - for (idx = 0; idx < ARRAY_SIZE(of_slot_quirks); idx++) - if (of_get_property(np, of_slot_quirks[idx].quirk, NULL)) { - dev_warn(dev, "Slot quirk %s is deprecated\n", - of_slot_quirks[idx].quirk); - quirks |= of_slot_quirks[idx].id; - } + if (!np) + return; - return quirks; + if (of_property_read_bool(np, "disable-wp")) { + slot->mmc->caps2 |= MMC_CAP2_NO_WRITE_PROTECT; + dev_warn(slot->mmc->parent, + "Slot quirk 'disable-wp' is deprecated\n"); + } } #else /* CONFIG_OF */ -static int dw_mci_of_get_slot_quirks(struct device *dev, u8 slot) +static void dw_mci_slot_of_parse(struct dw_mci_slot *slot) { - return 0; } #endif /* CONFIG_OF */ @@ -2352,8 +2540,6 @@ static int dw_mci_init_slot(struct dw_mci *host, unsigned int id) slot->host = host; host->slot[id] = slot; - slot->quirks = dw_mci_of_get_slot_quirks(host->dev, slot->id); - mmc->ops = &dw_mci_ops; if (of_property_read_u32_array(host->dev->of_node, "clock-freq-min-max", freq, 2)) { @@ -2391,31 +2577,34 @@ static int dw_mci_init_slot(struct dw_mci *host, unsigned int id) if (host->pdata->caps2) mmc->caps2 = host->pdata->caps2; + dw_mci_slot_of_parse(slot); + ret = mmc_of_parse(mmc); if (ret) goto err_host_allocated; - if (host->pdata->blk_settings) { - mmc->max_segs = host->pdata->blk_settings->max_segs; - mmc->max_blk_size = host->pdata->blk_settings->max_blk_size; - mmc->max_blk_count = host->pdata->blk_settings->max_blk_count; - mmc->max_req_size = host->pdata->blk_settings->max_req_size; - mmc->max_seg_size = host->pdata->blk_settings->max_seg_size; - } else { - /* Useful defaults if platform data is unset. */ -#ifdef CONFIG_MMC_DW_IDMAC + /* Useful defaults if platform data is unset. */ + if (host->use_dma == TRANS_MODE_IDMAC) { mmc->max_segs = host->ring_size; mmc->max_blk_size = 65536; mmc->max_seg_size = 0x1000; mmc->max_req_size = mmc->max_seg_size * host->ring_size; mmc->max_blk_count = mmc->max_req_size / 512; -#else + } else if (host->use_dma == TRANS_MODE_EDMAC) { + mmc->max_segs = 64; + mmc->max_blk_size = 65536; + mmc->max_blk_count = 65535; + mmc->max_req_size = + mmc->max_blk_size * mmc->max_blk_count; + mmc->max_seg_size = mmc->max_req_size; + } else { + /* TRANS_MODE_PIO */ mmc->max_segs = 64; mmc->max_blk_size = 65536; /* BLKSIZ is 16 bits */ mmc->max_blk_count = 512; - mmc->max_req_size = mmc->max_blk_size * mmc->max_blk_count; + mmc->max_req_size = mmc->max_blk_size * + mmc->max_blk_count; mmc->max_seg_size = mmc->max_req_size; -#endif /* CONFIG_MMC_DW_IDMAC */ } if (dw_mci_get_cd(mmc)) @@ -2449,44 +2638,80 @@ static void dw_mci_cleanup_slot(struct dw_mci_slot *slot, unsigned int id) static void dw_mci_init_dma(struct dw_mci *host) { int addr_config; - /* Check ADDR_CONFIG bit in HCON to find IDMAC address bus width */ - addr_config = (mci_readl(host, HCON) >> 27) & 0x01; - - if (addr_config == 1) { - /* host supports IDMAC in 64-bit address mode */ - host->dma_64bit_address = 1; - dev_info(host->dev, "IDMAC supports 64-bit address mode.\n"); - if (!dma_set_mask(host->dev, DMA_BIT_MASK(64))) - dma_set_coherent_mask(host->dev, DMA_BIT_MASK(64)); - } else { - /* host supports IDMAC in 32-bit address mode */ - host->dma_64bit_address = 0; - dev_info(host->dev, "IDMAC supports 32-bit address mode.\n"); - } + struct device *dev = host->dev; + struct device_node *np = dev->of_node; - /* Alloc memory for sg translation */ - host->sg_cpu = dmam_alloc_coherent(host->dev, PAGE_SIZE, - &host->sg_dma, GFP_KERNEL); - if (!host->sg_cpu) { - dev_err(host->dev, "%s: could not alloc DMA memory\n", - __func__); + /* + * Check tansfer mode from HCON[17:16] + * Clear the ambiguous description of dw_mmc databook: + * 2b'00: No DMA Interface -> Actually means using Internal DMA block + * 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block + * 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block + * 2b'11: Non DW DMA Interface -> pio only + * Compared to DesignWare DMA Interface, Generic DMA Interface has a + * simpler request/acknowledge handshake mechanism and both of them + * are regarded as external dma master for dw_mmc. + */ + host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON)); + if (host->use_dma == DMA_INTERFACE_IDMA) { + host->use_dma = TRANS_MODE_IDMAC; + } else if (host->use_dma == DMA_INTERFACE_DWDMA || + host->use_dma == DMA_INTERFACE_GDMA) { + host->use_dma = TRANS_MODE_EDMAC; + } else { goto no_dma; } /* Determine which DMA interface to use */ -#ifdef CONFIG_MMC_DW_IDMAC - host->dma_ops = &dw_mci_idmac_ops; - dev_info(host->dev, "Using internal DMA controller.\n"); -#endif + if (host->use_dma == TRANS_MODE_IDMAC) { + /* + * Check ADDR_CONFIG bit in HCON to find + * IDMAC address bus width + */ + addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON)); + + if (addr_config == 1) { + /* host supports IDMAC in 64-bit address mode */ + host->dma_64bit_address = 1; + dev_info(host->dev, + "IDMAC supports 64-bit address mode.\n"); + if (!dma_set_mask(host->dev, DMA_BIT_MASK(64))) + dma_set_coherent_mask(host->dev, + DMA_BIT_MASK(64)); + } else { + /* host supports IDMAC in 32-bit address mode */ + host->dma_64bit_address = 0; + dev_info(host->dev, + "IDMAC supports 32-bit address mode.\n"); + } - if (!host->dma_ops) - goto no_dma; + /* Alloc memory for sg translation */ + host->sg_cpu = dmam_alloc_coherent(host->dev, PAGE_SIZE, + &host->sg_dma, GFP_KERNEL); + if (!host->sg_cpu) { + dev_err(host->dev, + "%s: could not alloc DMA memory\n", + __func__); + goto no_dma; + } + + host->dma_ops = &dw_mci_idmac_ops; + dev_info(host->dev, "Using internal DMA controller.\n"); + } else { + /* TRANS_MODE_EDMAC: check dma bindings again */ + if ((of_property_count_strings(np, "dma-names") < 0) || + (!of_find_property(np, "dmas", NULL))) { + goto no_dma; + } + host->dma_ops = &dw_mci_edmac_ops; + dev_info(host->dev, "Using external DMA controller.\n"); + } if (host->dma_ops->init && host->dma_ops->start && host->dma_ops->stop && host->dma_ops->cleanup) { if (host->dma_ops->init(host)) { - dev_err(host->dev, "%s: Unable to initialize " - "DMA Controller.\n", __func__); + dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n", + __func__); goto no_dma; } } else { @@ -2494,13 +2719,11 @@ static void dw_mci_init_dma(struct dw_mci *host) goto no_dma; } - host->use_dma = 1; return; no_dma: dev_info(host->dev, "Using PIO mode.\n"); - host->use_dma = 0; - return; + host->use_dma = TRANS_MODE_PIO; } static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset) @@ -2554,6 +2777,7 @@ static bool dw_mci_reset(struct dw_mci *host) if (host->use_dma) { unsigned long timeout = jiffies + msecs_to_jiffies(500); u32 status; + do { status = mci_readl(host, STATUS); if (!(status & SDMMC_STATUS_DMA_REQ)) @@ -2563,8 +2787,8 @@ static bool dw_mci_reset(struct dw_mci *host) if (status & SDMMC_STATUS_DMA_REQ) { dev_err(host->dev, - "%s: Timeout waiting for dma_req to " - "clear during reset\n", __func__); + "%s: Timeout waiting for dma_req to clear during reset\n", + __func__); goto ciu_out; } @@ -2575,17 +2799,16 @@ static bool dw_mci_reset(struct dw_mci *host) } else { /* if the controller reset bit did clear, then set clock regs */ if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) { - dev_err(host->dev, "%s: fifo/dma reset bits didn't " - "clear but ciu was reset, doing clock update\n", + dev_err(host->dev, + "%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n", __func__); goto ciu_out; } } -#if IS_ENABLED(CONFIG_MMC_DW_IDMAC) - /* It is also recommended that we reset and reprogram idmac */ - dw_mci_idmac_reset(host); -#endif + if (host->use_dma == TRANS_MODE_IDMAC) + /* It is also recommended that we reset and reprogram idmac */ + dw_mci_idmac_reset(host); ret = true; @@ -2610,6 +2833,28 @@ static void dw_mci_cmd11_timer(unsigned long arg) tasklet_schedule(&host->tasklet); } +static void dw_mci_dto_timer(unsigned long arg) +{ + struct dw_mci *host = (struct dw_mci *)arg; + + switch (host->state) { + case STATE_SENDING_DATA: + case STATE_DATA_BUSY: + /* + * If DTO interrupt does NOT come in sending data state, + * we should notify the driver to terminate current transfer + * and report a data timeout to the core. + */ + host->data_status = SDMMC_INT_DRTO; + set_bit(EVENT_DATA_ERROR, &host->pending_events); + set_bit(EVENT_DATA_COMPLETE, &host->pending_events); + tasklet_schedule(&host->tasklet); + break; + default: + break; + } +} + #ifdef CONFIG_OF static struct dw_mci_of_quirks { char *quirk; @@ -2618,9 +2863,6 @@ static struct dw_mci_of_quirks { { .quirk = "broken-cd", .id = DW_MCI_QUIRK_BROKEN_CARD_DETECTION, - }, { - .quirk = "disable-wp", - .id = DW_MCI_QUIRK_NO_WRITE_PROTECT, }, }; @@ -2640,8 +2882,8 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host) /* find out number of slots supported */ if (of_property_read_u32(dev->of_node, "num-slots", &pdata->num_slots)) { - dev_info(dev, "num-slots property not found, " - "assuming 1 slot is available\n"); + dev_info(dev, + "num-slots property not found, assuming 1 slot is available\n"); pdata->num_slots = 1; } @@ -2651,8 +2893,8 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host) pdata->quirks |= of_quirks[idx].id; if (of_property_read_u32(np, "fifo-depth", &pdata->fifo_depth)) - dev_info(dev, "fifo-depth property not found, using " - "value of FIFOTH register as default\n"); + dev_info(dev, + "fifo-depth property not found, using value of FIFOTH register as default\n"); of_property_read_u32(np, "card-detect-delay", &pdata->detect_delay_ms); @@ -2665,8 +2907,10 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host) return ERR_PTR(ret); } - if (of_find_property(np, "supports-highspeed", NULL)) + if (of_find_property(np, "supports-highspeed", NULL)) { + dev_info(dev, "supports-highspeed property is deprecated.\n"); pdata->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED; + } return pdata; } @@ -2721,7 +2965,7 @@ int dw_mci_probe(struct dw_mci *host) } } - if (host->pdata->num_slots > 1) { + if (host->pdata->num_slots < 1) { dev_err(host->dev, "Platform data must supply num_slots.\n"); return -ENODEV; @@ -2789,6 +3033,10 @@ int dw_mci_probe(struct dw_mci *host) host->quirks = host->pdata->quirks; + if (host->quirks & DW_MCI_QUIRK_BROKEN_DTO) + setup_timer(&host->dto_timer, + dw_mci_dto_timer, (unsigned long)host); + spin_lock_init(&host->lock); spin_lock_init(&host->irq_lock); INIT_LIST_HEAD(&host->queue); @@ -2797,7 +3045,7 @@ int dw_mci_probe(struct dw_mci *host) * Get the host data width - this assumes that HCON has been set with * the correct values. */ - i = (mci_readl(host, HCON) >> 7) & 0x7; + i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON)); if (!i) { host->push_data = dw_mci_push_data16; host->pull_data = dw_mci_pull_data16; @@ -2879,7 +3127,7 @@ int dw_mci_probe(struct dw_mci *host) if (host->pdata->num_slots) host->num_slots = host->pdata->num_slots; else - host->num_slots = ((mci_readl(host, HCON) >> 1) & 0x1F) + 1; + host->num_slots = SDMMC_GET_SLOT_NUM(mci_readl(host, HCON)); /* * Enable interrupts for command done, data over, data empty, @@ -2889,11 +3137,11 @@ int dw_mci_probe(struct dw_mci *host) mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | SDMMC_INT_TXDR | SDMMC_INT_RXDR | DW_MCI_ERROR_FLAGS); - mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); /* Enable mci interrupt */ + /* Enable mci interrupt */ + mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); - dev_info(host->dev, "DW MMC controller at irq %d, " - "%d bit host data width, " - "%u deep fifo\n", + dev_info(host->dev, + "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n", host->irq, width, fifo_size); /* We need at least one slot to succeed */ @@ -2908,8 +3156,9 @@ int dw_mci_probe(struct dw_mci *host) if (init_slots) { dev_info(host->dev, "%d slots initialized\n", init_slots); } else { - dev_dbg(host->dev, "attempted to initialize %d slots, " - "but failed on all\n", host->num_slots); + dev_dbg(host->dev, + "attempted to initialize %d slots, but failed on all\n", + host->num_slots); goto err_dmaunmap; } @@ -2941,15 +3190,15 @@ void dw_mci_remove(struct dw_mci *host) { int i; - mci_writel(host, RINTSTS, 0xFFFFFFFF); - mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ - for (i = 0; i < host->num_slots; i++) { dev_dbg(host->dev, "remove slot %d\n", i); if (host->slot[i]) dw_mci_cleanup_slot(host->slot[i], i); } + mci_writel(host, RINTSTS, 0xFFFFFFFF); + mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ + /* disable clock to CIU */ mci_writel(host, CLKENA, 0); mci_writel(host, CLKSRC, 0); @@ -2973,6 +3222,9 @@ EXPORT_SYMBOL(dw_mci_remove); */ int dw_mci_suspend(struct dw_mci *host) { + if (host->use_dma && host->dma_ops->exit) + host->dma_ops->exit(host); + return 0; } EXPORT_SYMBOL(dw_mci_suspend); @@ -3007,6 +3259,7 @@ int dw_mci_resume(struct dw_mci *host) for (i = 0; i < host->num_slots; i++) { struct dw_mci_slot *slot = host->slot[i]; + if (!slot) continue; if (slot->mmc->pm_flags & MMC_PM_KEEP_POWER) { |