--- linux-4.9.37/drivers/mmc/host/sdhci-cmdq.c 1970-01-01 03:00:00.000000000 +0300 +++ linux-4.9.y/drivers/mmc/host/sdhci-cmdq.c 2021-06-07 13:01:33.000000000 +0300 @@ -0,0 +1,1108 @@ +/* + * Copyright (c) Hunan Goke,Chengdu Goke,Shandong Goke. 2021. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +//#include +#include + +#include "sdhci-cmdq.h" +#include "sdhci.h" + +#define DCMD_SLOT 31 +#define NUM_SLOTS 32 + +/* 1 sec */ +#define HALT_TIMEOUT_MS 1000 + +static int cmdq_halt_poll(struct mmc_host *mmc, bool halt); +static int cmdq_halt(struct mmc_host *mmc, bool halt); + +#ifdef CONFIG_PM_RUNTIME +static int cmdq_runtime_pm_get(struct cmdq_host *host) +{ + return pm_runtime_get_sync(host->mmc->parent); +} +static int cmdq_runtime_pm_put(struct cmdq_host *host) +{ + pm_runtime_mark_last_busy(host->mmc->parent); + return pm_runtime_put_autosuspend(host->mmc->parent); +} +#else +static inline int cmdq_runtime_pm_get(struct cmdq_host *host) +{ + return 0; +} +static inline int cmdq_runtime_pm_put(struct cmdq_host *host) +{ + return 0; +} +#endif +static inline struct mmc_request *get_req_by_tag(struct cmdq_host *cq_host, + unsigned int tag) +{ + return cq_host->mrq_slot[tag]; +} + +static inline u8 *get_desc(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->desc_base + (tag * cq_host->slot_sz); +} + +static inline u8 *get_link_desc(struct cmdq_host *cq_host, u8 tag) +{ + u8 *desc = get_desc(cq_host, tag); + + return desc + cq_host->task_desc_len; +} + +static inline dma_addr_t get_trans_desc_dma(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->trans_desc_dma_base + + (cq_host->slot_desc_sz * tag); +} + +static inline u8 *get_trans_desc(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->trans_desc_base + + (cq_host->slot_desc_sz * tag); +} + +static void setup_trans_desc(struct cmdq_host *cq_host, u8 tag) +{ + u8 *link_temp; + dma_addr_t trans_temp; + + link_temp = get_link_desc(cq_host, tag); + trans_temp = get_trans_desc_dma(cq_host, tag); + + memset(link_temp, 0, cq_host->link_desc_len); + if (cq_host->link_desc_len > 8) + *(link_temp + 8) = 0; + + if (tag == DCMD_SLOT) { + *link_temp = VALID(0) | ACT(0) | END(1); + return; + } + + *link_temp = VALID(1) | ACT(0x6) | END(0); + + if (cq_host->dma64) { + __le64 *data_addr = (__le64 __force *)(link_temp + 4); + data_addr[0] = cpu_to_le64(trans_temp); + } else { + __le32 *data_addr = (__le32 __force *)(link_temp + 4); + data_addr[0] = cpu_to_le32(trans_temp); + } +} + +static void cmdq_set_halt_irq(struct cmdq_host *cq_host, bool enable) +{ + u32 ier; + + ier = cmdq_readl(cq_host, CQISTE); + if (enable) { + cmdq_writel(cq_host, ier | HALT, CQISTE); + cmdq_writel(cq_host, ier | HALT, CQISGE); + } else { + cmdq_writel(cq_host, ier & ~HALT, CQISTE); + cmdq_writel(cq_host, ier & ~HALT, CQISGE); + } +} + +static void cmdq_clear_set_irqs(struct cmdq_host *cq_host, u32 clear, u32 set) +{ + u32 ier; + + ier = cmdq_readl(cq_host, CQISTE); + ier &= ~clear; + ier |= set; + cmdq_writel(cq_host, ier, CQISTE); + cmdq_writel(cq_host, ier, CQISGE); + /* ensure the writes are done */ + mb(); +} + + +#define DRV_NAME "cmdq-host" + +static void cmdq_dump_task_history(struct cmdq_host *cq_host) +{ + int i; + + if (likely(!cq_host->mmc->cmdq_thist_enabled)) + return; + + if (!cq_host->thist) { + pr_err("%s: %s: CMDQ task history buffer not allocated\n", + mmc_hostname(cq_host->mmc), __func__); + return; + } + + pr_err("---- Circular Task History ----\n"); + pr_err(DRV_NAME ": Last entry index: %d", cq_host->thist_idx - 1); + + for (i = 0; i < cq_host->num_slots; i++) { + pr_err(DRV_NAME ": [%02d]%s Task: 0x%08x | Args: 0x%08x\n", i, + (cq_host->thist[i].is_dcmd) ? "DCMD" : "DATA", + lower_32_bits(cq_host->thist[i].task), + upper_32_bits(cq_host->thist[i].task)); + } + pr_err("-------------------------\n"); +} + +static void cmdq_dump_adma_mem(struct cmdq_host *cq_host) +{ + struct mmc_host *mmc = cq_host->mmc; + dma_addr_t desc_dma; + int tag = 0; + unsigned long data_active_reqs = + mmc->cmdq_ctx.data_active_reqs; + unsigned long desc_size = cq_host->slot_desc_sz; + + for_each_set_bit(tag, &data_active_reqs, cq_host->num_slots) { + desc_dma = get_trans_desc_dma(cq_host, tag); + pr_err("%s: %s: tag = %d, trans_dma(phys) = %pad, trans_desc(virt) = 0x%p\n", + mmc_hostname(mmc), __func__, tag, + &desc_dma, get_trans_desc(cq_host, tag)); + print_hex_dump(KERN_ERR, "cmdq-adma:", DUMP_PREFIX_ADDRESS, + 32, 8, get_trans_desc(cq_host, tag), + (desc_size), false); + } +} + +static void cmdq_dumpregs(struct cmdq_host *cq_host) +{ + struct mmc_host *mmc = cq_host->mmc; + + pr_err(DRV_NAME ": ========== REGISTER DUMP (%s)==========\n", + mmc_hostname(mmc)); + + pr_err(DRV_NAME ": Caps: 0x%08x | Version: 0x%08x\n", + cmdq_readl(cq_host, CQCAP), + cmdq_readl(cq_host, CQVER)); + pr_err(DRV_NAME ": Queing config: 0x%08x | Queue Ctrl: 0x%08x\n", + cmdq_readl(cq_host, CQCFG), + cmdq_readl(cq_host, CQCTL)); + pr_err(DRV_NAME ": Int stat: 0x%08x | Int enab: 0x%08x\n", + cmdq_readl(cq_host, CQIS), + cmdq_readl(cq_host, CQISTE)); + pr_err(DRV_NAME ": Int sig: 0x%08x | Int Coal: 0x%08x\n", + cmdq_readl(cq_host, CQISGE), + cmdq_readl(cq_host, CQIC)); + pr_err(DRV_NAME ": TDL base: 0x%08x | TDL up32: 0x%08x\n", + cmdq_readl(cq_host, CQTDLBA), + cmdq_readl(cq_host, CQTDLBAU)); + pr_err(DRV_NAME ": Doorbell: 0x%08x | Comp Notif: 0x%08x\n", + cmdq_readl(cq_host, CQTDBR), + cmdq_readl(cq_host, CQTCN)); + pr_err(DRV_NAME ": Dev queue: 0x%08x | Dev Pend: 0x%08x\n", + cmdq_readl(cq_host, CQDQS), + cmdq_readl(cq_host, CQDPT)); + pr_err(DRV_NAME ": Task clr: 0x%08x | Send stat 1: 0x%08x\n", + cmdq_readl(cq_host, CQTCLR), + cmdq_readl(cq_host, CQSSC1)); + pr_err(DRV_NAME ": Send stat 2: 0x%08x | DCMD resp: 0x%08x\n", + cmdq_readl(cq_host, CQSSC2), + cmdq_readl(cq_host, CQCRDCT)); + pr_err(DRV_NAME ": Resp err mask: 0x%08x | Task err: 0x%08x\n", + cmdq_readl(cq_host, CQRMEM), + cmdq_readl(cq_host, CQTERRI)); + pr_err(DRV_NAME ": Resp idx 0x%08x | Resp arg: 0x%08x\n", + cmdq_readl(cq_host, CQCRI), + cmdq_readl(cq_host, CQCRA)); + pr_err(DRV_NAME": Goke cfg 0x%08x\n", + cmdq_readl(cq_host, CQ_GOKE_CFG)); + pr_err(DRV_NAME ": ===========================================\n"); + + cmdq_dump_task_history(cq_host); + if (cq_host->ops->dump_goke_regs) + cq_host->ops->dump_goke_regs(mmc); +} + +/** + * The allocated descriptor table for task, link & transfer descritors + * looks like: + * |----------| + * |task desc | |->|----------| + * |----------| | |trans desc| + * |link desc-|->| |----------| + * |----------| . + * . . + * no. of slots max-segs + * . |----------| + * |----------| + * The idea here is to create the [task+trans] table and mark & point the + * link desc to the transfer desc table on a per slot basis. + */ +static int cmdq_host_alloc_tdl(struct cmdq_host *cq_host) +{ + + size_t desc_size; + size_t data_size; + int i = 0; + + /* task descriptor can be 64/128 bit irrespective of arch */ + if (cq_host->caps & CMDQ_TASK_DESC_SZ_128) { + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCFG) | + CQ_TASK_DESC_SZ, CQCFG); + cq_host->task_desc_len = 16; + } else { + cq_host->task_desc_len = 8; + } + + /* + * 96 bits length of transfer desc instead of 128 bits which means + * ADMA would expect next valid descriptor at the 96th bit + * or 128th bit + */ + if (cq_host->dma64) { + if (cq_host->quirks & CMDQ_QUIRK_SHORT_TXFR_DESC_SZ) + cq_host->trans_desc_len = 12; + else + cq_host->trans_desc_len = 16; + cq_host->link_desc_len = 16; + } else { + cq_host->trans_desc_len = 8; + cq_host->link_desc_len = 8; + } + + /* total size of a slot: 1 task & 1 transfer (link) */ + cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len; + + desc_size = cq_host->slot_sz * cq_host->num_slots; + + cq_host->slot_desc_sz = cq_host->trans_desc_len * + cq_host->mmc->max_segs * 2; + data_size = cq_host->slot_desc_sz * (cq_host->num_slots - 1); + + pr_debug("%s: desc_size: %d data_sz: %d slot-sz: %d\n", __func__, + (int)desc_size, (int)data_size, cq_host->slot_sz); + + /* + * allocate a dma-mapped chunk of memory for the descriptors + * allocate a dma-mapped chunk of memory for link descriptors + * setup each link-desc memory offset per slot-number to + * the descriptor table. + */ + cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), + desc_size, + &cq_host->desc_dma_base, + GFP_KERNEL); + cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), + data_size, + &cq_host->trans_desc_dma_base, + GFP_KERNEL); + cq_host->thist = devm_kzalloc(mmc_dev(cq_host->mmc), + (sizeof(*cq_host->thist) * + cq_host->num_slots), + GFP_KERNEL); + if (!cq_host->desc_base || !cq_host->trans_desc_base) + return -ENOMEM; + + pr_debug("desc-base: 0x%p trans-base: 0x%p\ndesc_dma 0x%llx trans_dma: 0x%llx\n", + cq_host->desc_base, cq_host->trans_desc_base, + (unsigned long long)cq_host->desc_dma_base, + (unsigned long long) cq_host->trans_desc_dma_base); + + for (; i < (cq_host->num_slots); i++) + setup_trans_desc(cq_host, i); + + return 0; +} + +static int cmdq_enable(struct mmc_host *mmc) +{ + int err = 0; + u32 cqcfg; + bool dcmd_enable = false; + struct cmdq_host *cq_host = mmc_cmdq_private(mmc); + + if (!cq_host || !mmc->card || !mmc_card_cmdq(mmc->card)) { + err = -EINVAL; + goto out; + } + + if (cq_host->enabled) + goto out; + + cmdq_runtime_pm_get(cq_host); + cqcfg = cmdq_readl(cq_host, CQCFG); + if (cqcfg & 0x1) { + pr_info("%s: %s: cq_host is already enabled\n", + mmc_hostname(mmc), __func__); + WARN_ON(1); + goto pm_ref_count; + } + + if (cq_host->quirks & CMDQ_QUIRK_NO_DCMD) + dcmd_enable = false; + else + dcmd_enable = true; + + cqcfg = ((cq_host->caps & CMDQ_TASK_DESC_SZ_128 ? CQ_TASK_DESC_SZ : 0) | + (dcmd_enable ? CQ_DCMD : 0)); + + cmdq_writel(cq_host, cqcfg, CQCFG); + /* enable CQ_HOST */ + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCFG) | CQ_ENABLE, + CQCFG); + + if (!cq_host->desc_base || + !cq_host->trans_desc_base) { + err = cmdq_host_alloc_tdl(cq_host); + if (err) + goto pm_ref_count; + } + + cmdq_writel(cq_host, lower_32_bits(cq_host->desc_dma_base), CQTDLBA); + cmdq_writel(cq_host, upper_32_bits(cq_host->desc_dma_base), CQTDLBAU); + + /* + * disable all goke interrupts + * enable CMDQ interrupts + * enable the goke error interrupts + */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + + cmdq_clear_set_irqs(cq_host, 0x0, CQ_INT_ALL); + + /* cq_host would use this rca to address the card */ + cmdq_writel(cq_host, mmc->card->rca, CQSSC2); + + /* send QSR at lesser intervals than the default */ + cmdq_writel(cq_host, SEND_QSR_INTERVAL, CQSSC1); + + /* enable bkops exception indication */ + if (mmc_card_configured_manual_bkops(mmc->card) && + !mmc_card_configured_auto_bkops(mmc->card)) + cmdq_writel(cq_host, cmdq_readl(cq_host, CQRMEM) | CQ_EXCEPTION, + CQRMEM); + + /* ensure the writes are done before enabling CQE */ + mb(); + + cq_host->enabled = true; + mmc_host_clr_cq_disable(mmc); + + if (cq_host->ops->set_transfer_params) + cq_host->ops->set_transfer_params(mmc); + + if (cq_host->ops->set_block_size) + cq_host->ops->set_block_size(cq_host->mmc); + + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + + if (cq_host->ops->clear_set_dumpregs) + cq_host->ops->clear_set_dumpregs(mmc, 1); + + if (cq_host->ops->enhanced_strobe_mask) + cq_host->ops->enhanced_strobe_mask(mmc, true); + +pm_ref_count: + cmdq_runtime_pm_put(cq_host); +out: + return err; +} + +static void cmdq_disable_nosync(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + if (soft) { + cmdq_writel(cq_host, cmdq_readl( + cq_host, CQCFG) & ~(CQ_ENABLE), + CQCFG); + } + if (cq_host->ops->enhanced_strobe_mask) + cq_host->ops->enhanced_strobe_mask(mmc, false); + + cq_host->enabled = false; + mmc_host_set_cq_disable(mmc); +} + +static void cmdq_disable(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + cmdq_runtime_pm_get(cq_host); + cmdq_disable_nosync(mmc, soft); + cmdq_runtime_pm_put(cq_host); +} + +static void cmdq_reset(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + unsigned int cqcfg; + unsigned int tdlba; + unsigned int tdlbau; + unsigned int rca; + int ret; + + cmdq_runtime_pm_get(cq_host); + cqcfg = cmdq_readl(cq_host, CQCFG); + tdlba = cmdq_readl(cq_host, CQTDLBA); + tdlbau = cmdq_readl(cq_host, CQTDLBAU); + rca = cmdq_readl(cq_host, CQSSC2); + + cmdq_disable(mmc, true); + + if (cq_host->ops->reset) { + ret = cq_host->ops->reset(mmc); + if (ret) { + pr_crit("%s: reset CMDQ controller: failed\n", + mmc_hostname(mmc)); + BUG(); + } + } + + cmdq_writel(cq_host, tdlba, CQTDLBA); + cmdq_writel(cq_host, tdlbau, CQTDLBAU); + + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + + cmdq_clear_set_irqs(cq_host, 0x0, CQ_INT_ALL); + + /* cq_host would use this rca to address the card */ + cmdq_writel(cq_host, rca, CQSSC2); + + /* ensure the writes are done before enabling CQE */ + mb(); + + cmdq_writel(cq_host, cqcfg, CQCFG); + cmdq_runtime_pm_put(cq_host); + cq_host->enabled = true; + mmc_host_clr_cq_disable(mmc); +} + +static void cmdq_prep_task_desc(struct mmc_request *mrq, + u64 *data, bool intr, bool qbr) +{ + struct mmc_cmdq_req *cmdq_req = mrq->cmdq_req; + u32 req_flags = cmdq_req->cmdq_req_flags; + + pr_debug("%s: %s: data-tag: 0x%08x - dir: %d - prio: %d - cnt: 0x%08x - addr: 0x%llx\n", + mmc_hostname(mrq->host), __func__, + !!(req_flags & DAT_TAG), !!(req_flags & DIR), + !!(req_flags & PRIO), cmdq_req->data.blocks, + (u64)mrq->cmdq_req->blk_addr); + + *data = VALID(1) | + END(1) | + INT(intr) | + ACT(0x5) | + FORCED_PROG(!!(req_flags & FORCED_PRG)) | + CONTEXT(mrq->cmdq_req->ctx_id) | + DATA_TAG(!!(req_flags & DAT_TAG)) | + DATA_DIR(!!(req_flags & DIR)) | + PRIORITY(!!(req_flags & PRIO)) | + QBAR(qbr) | + REL_WRITE(!!(req_flags & REL_WR)) | + BLK_COUNT(mrq->cmdq_req->data.blocks) | + BLK_ADDR((u64)mrq->cmdq_req->blk_addr); +} + +static int cmdq_dma_map(struct mmc_host *host, struct mmc_request *mrq) +{ + int sg_count; + struct mmc_data *data = mrq->data; + + if (!data) + return -EINVAL; + + sg_count = dma_map_sg(mmc_dev(host), data->sg, + data->sg_len, + (data->flags & MMC_DATA_WRITE) ? + DMA_TO_DEVICE : DMA_FROM_DEVICE); + if (!sg_count) { + pr_err("%s: sg-len: %d\n", __func__, data->sg_len); + return -ENOMEM; + } + + return sg_count; +} + +static void cmdq_set_tran_desc(u8 *desc, dma_addr_t addr, int len, + bool end, bool is_dma64) +{ + __le32 *attr = (__le32 __force *)desc; + + *attr = (VALID(1) | + END(end ? 1 : 0) | + INT(0) | + ACT(0x4) | + DAT_LENGTH(len)); + + if (is_dma64) { + __le64 *dataddr = (__le64 __force *)(desc + 4); + + dataddr[0] = cpu_to_le64(addr); + } else { + __le32 *dataddr = (__le32 __force *)(desc + 4); + + dataddr[0] = cpu_to_le32(addr); + } +} + +static int cmdq_prep_tran_desc(struct mmc_request *mrq, + struct cmdq_host *cq_host, int tag) +{ + struct mmc_data *data = mrq->data; + int i, sg_count, len; + bool end = false; + dma_addr_t addr; + u8 *desc = NULL; + struct scatterlist *sg = NULL; + + sg_count = cmdq_dma_map(mrq->host, mrq); + if (sg_count < 0) { + pr_err("%s: %s: unable to map sg lists, %d\n", + mmc_hostname(mrq->host), __func__, sg_count); + return sg_count; + } + + desc = get_trans_desc(cq_host, tag); + memset(desc, 0, cq_host->slot_desc_sz); + + for_each_sg(data->sg, sg, sg_count, i) { + addr = sg_dma_address(sg); + len = sg_dma_len(sg); + + if ((i+1) == sg_count) + end = true; + /* work around for buffer across 128M boundary, split the buffer */ + if (((addr & (SDHCI_DMA_BOUNDARY_SIZE - 1)) + len) > + SDHCI_DMA_BOUNDARY_SIZE) { + int offset; + + offset = SDHCI_DMA_BOUNDARY_SIZE - + (addr & (SDHCI_DMA_BOUNDARY_SIZE - 1)); + cmdq_set_tran_desc(desc, addr, offset, false, cq_host->dma64); + desc += cq_host->trans_desc_len; + addr += offset; + len -= offset; + } + cmdq_set_tran_desc(desc, addr, len, end, cq_host->dma64); + desc += cq_host->trans_desc_len; + } + + pr_debug("%s: req: 0x%p tag: %d calc_trans_des: 0x%p sg-cnt: %d\n", + __func__, mrq->req, tag, desc, sg_count); + + return 0; +} + +static void cmdq_log_task_desc_history(struct cmdq_host *cq_host, u64 task, + bool is_dcmd) +{ + if (likely(!cq_host->mmc->cmdq_thist_enabled)) + return; + + if (!cq_host->thist) { + pr_err("%s: %s: CMDQ task history buffer not allocated\n", + mmc_hostname(cq_host->mmc), __func__); + return; + } + + if (cq_host->thist_idx >= cq_host->num_slots) + cq_host->thist_idx = 0; + + cq_host->thist[cq_host->thist_idx].is_dcmd = is_dcmd; + memcpy(&cq_host->thist[cq_host->thist_idx++].task, + &task, cq_host->task_desc_len); +} + +static void cmdq_prep_dcmd_desc(struct mmc_host *mmc, + struct mmc_request *mrq) +{ + u64 *task_desc = NULL; + u64 data = 0; + u8 resp_type; + u8 *desc; + __le64 *dataddr == NULL; + struct cmdq_host *cq_host = mmc_cmdq_private(mmc); + u8 timing; + + if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) { + resp_type = 0x0; + timing = 0x1; + } else { + if (mrq->cmd->flags & MMC_RSP_BUSY) { + resp_type = 0x3; + timing = 0x0; + } else { + resp_type = 0x2; + timing = 0x1; + } + } + + task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot); + memset(task_desc, 0, cq_host->task_desc_len); + data |= (VALID(1) | + END(1) | + INT(1) | + QBAR(1) | + ACT(0x5) | + CMD_INDEX(mrq->cmd->opcode) | + CMD_TIMING(timing) | RESP_TYPE(resp_type)); + *task_desc |= data; + desc = (u8 *)task_desc; + pr_debug("cmdq: dcmd: cmd: %d timing: %d resp: %d\n", + mrq->cmd->opcode, timing, resp_type); + dataddr = (__le64 __force *)(desc + 4); + dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg); + cmdq_log_task_desc_history(cq_host, *task_desc, true); +} + +static int cmdq_request(struct mmc_host *mmc, struct mmc_request *mrq) +{ + int err = 0; + u64 data = 0; + u64 *task_desc = NULL; + u32 tag = mrq->cmdq_req->tag; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + if (!cq_host->enabled) { + pr_err("%s: CMDQ host not enabled yet !!!\n", + mmc_hostname(mmc)); + err = -EINVAL; + goto out; + } + + cmdq_runtime_pm_get(cq_host); + + if (mrq->cmdq_req->cmdq_req_flags & DCMD) { + cmdq_prep_dcmd_desc(mmc, mrq); + cq_host->mrq_slot[DCMD_SLOT] = mrq; + /* DCMD's are always issued on a fixed slot */ + tag = DCMD_SLOT; + goto ring_doorbell; + } + + if (cq_host->ops->crypto_cfg) { + err = cq_host->ops->crypto_cfg(mmc, mrq, tag); + if (err) { + pr_err("%s: failed to configure crypto: err %d tag %d\n", + mmc_hostname(mmc), err, tag); + goto out; + } + } + + task_desc = (__le64 __force *)get_desc(cq_host, tag); + + cmdq_prep_task_desc(mrq, &data, 1, + (mrq->cmdq_req->cmdq_req_flags & QBR)); + *task_desc = cpu_to_le64(data); + cmdq_log_task_desc_history(cq_host, *task_desc, false); + + err = cmdq_prep_tran_desc(mrq, cq_host, tag); + if (err) { + pr_err("%s: %s: failed to setup tx desc: %d\n", + mmc_hostname(mmc), __func__, err); + goto out; + } + + cq_host->mrq_slot[tag] = mrq; + +ring_doorbell: + /* Ensure the task descriptor list is flushed before ringing doorbell */ + wmb(); + if (cmdq_readl(cq_host, CQTDBR) & (1 << tag)) { + cmdq_dumpregs(cq_host); + BUG_ON(1); + } + cmdq_writel(cq_host, 1 << tag, CQTDBR); + /* Commit the doorbell write immediately */ + wmb(); + +out: + return err; +} + +static void cmdq_finish_data(struct mmc_host *mmc, unsigned int tag) +{ + struct mmc_request *mrq; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + mrq = get_req_by_tag(cq_host, tag); + if (tag == cq_host->dcmd_slot) + mrq->cmd->resp[0] = cmdq_readl(cq_host, CQCRDCT); + + if (mrq->cmdq_req->cmdq_req_flags & DCMD) + cmdq_writel(cq_host, cmdq_readl(cq_host, CQ_GOKE_CFG) | + CMDQ_SEND_STATUS_TRIGGER, CQ_GOKE_CFG); + + cmdq_runtime_pm_put(cq_host); + if (cq_host->ops->crypto_cfg_reset) + cq_host->ops->crypto_cfg_reset(mmc, tag); + mrq->done(mrq); +} + +irqreturn_t cmdq_irq(struct mmc_host *mmc, int err) +{ + u32 status; + unsigned long tag = 0, comp_status; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + unsigned long err_info = 0; + struct mmc_request *mrq = NULL; + int ret; + u32 dbr_set = 0; + + status = cmdq_readl(cq_host, CQIS); + cmdq_writel(cq_host, status, CQIS); + + if (!status && !err) + return IRQ_NONE; + + if (err || (status & CQIS_RED)) { + err_info = cmdq_readl(cq_host, CQTERRI); + pr_err("%s: err: %d status: 0x%08x task-err-info (0x%08lx)\n", + mmc_hostname(mmc), err, status, err_info); + + /* + * Need to halt CQE in case of error in interrupt context itself + * otherwise CQE may proceed with sending CMD to device even if + * CQE/card is in error state. + * CMDQ error handling will make sure that it is unhalted after + * handling all the errors. + */ + ret = cmdq_halt_poll(mmc, true); + if (ret) + pr_err("%s: %s: halt failed ret=%d\n", + mmc_hostname(mmc), __func__, ret); + cmdq_dumpregs(cq_host); + + if (!err_info) { + /* + * It may so happen sometimes for few errors(like ADMA) + * that HW cannot give CQTERRI info. + * Thus below is a HW WA for recovering from such + * scenario. + * - To halt/disable CQE and do reset_all. + * Since there is no way to know which tag would + * have caused such error, so check for any first + * bit set in doorbell and proceed with an error. + */ + dbr_set = cmdq_readl(cq_host, CQTDBR); + if (!dbr_set) { + pr_err("%s: spurious/force error interrupt\n", + mmc_hostname(mmc)); + cmdq_halt_poll(mmc, false); + mmc_host_clr_halt(mmc); + return IRQ_HANDLED; + } + + tag = ffs(dbr_set) - 1; + pr_err("%s: error tag selected: tag = %lu\n", + mmc_hostname(mmc), tag); + mrq = get_req_by_tag(cq_host, tag); + if (mrq->data) + mrq->data->error = err; + else + mrq->cmd->error = err; + /* + * Get ADMA descriptor memory in case of ADMA + * error for debug. + */ + if (err == -EIO) + cmdq_dump_adma_mem(cq_host); + goto skip_cqterri; + } + + if (err_info & CQ_RMEFV) { + tag = GET_CMD_ERR_TAG(err_info); + pr_err("%s: CMD err tag: %lu\n", __func__, tag); + + mrq = get_req_by_tag(cq_host, tag); + /* CMD44/45/46/47 will not have a valid cmd */ + if (mrq->cmd) + mrq->cmd->error = err; + else + mrq->data->error = err; + } else if (err_info & CQ_DTEFV) { + tag = GET_DAT_ERR_TAG(err_info); + pr_err("%s: Dat err tag: %lu\n", __func__, tag); + mrq = get_req_by_tag(cq_host, tag); + mrq->data->error = err; + } + +skip_cqterri: + /* + * If CQE halt fails then, disable CQE + * from processing any further requests + */ + if (ret) { + cmdq_disable_nosync(mmc, true); + /* + * Enable legacy interrupts as CQE halt has failed. + * This is needed to send legacy commands like status + * cmd as part of error handling work. + */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, false); + } + + /* + * CQE detected a reponse error from device + * In most cases, this would require a reset. + */ + if (status & CQIS_RED) { + /* + * will check if the RED error is due to a bkops + * exception once the queue is empty + */ + BUG_ON(!mmc->card); + /*if (mmc_card_configured_manual_bkops(mmc->card) || + mmc_card_configured_auto_bkops(mmc->card)) + mmc->card->bkops.needs_check = true;*/ + + mrq->cmdq_req->resp_err = true; + pr_err("%s: Response error (0x%08x) from card !!!", + mmc_hostname(mmc), status); + } else { + mrq->cmdq_req->resp_idx = cmdq_readl(cq_host, CQCRI); + mrq->cmdq_req->resp_arg = cmdq_readl(cq_host, CQCRA); + } + + cmdq_finish_data(mmc, tag); + } + + if (status & CQIS_TCC) { + /* read CQTCN and complete the request */ + comp_status = cmdq_readl(cq_host, CQTCN); + if (!comp_status) + goto out; + /* + * The CQTCN must be cleared before notifying req completion + * to upper layers to avoid missing completion notification + * of new requests with the same tag. + */ + cmdq_writel(cq_host, comp_status, CQTCN); + /* + * A write memory barrier is necessary to guarantee that CQTCN + * gets cleared first before next doorbell for the same tag is + * set but that is already achieved by the barrier present + * before setting doorbell, hence one is not needed here. + */ + for_each_set_bit(tag, &comp_status, cq_host->num_slots) { + /* complete the corresponding mrq */ + pr_debug("%s: completing tag -> %lu\n", + mmc_hostname(mmc), tag); + cmdq_finish_data(mmc, tag); + } + } + + if (status & CQIS_HAC) { + if (cq_host->ops->post_cqe_halt) + cq_host->ops->post_cqe_halt(mmc); + /* halt is completed, wakeup waiting thread */ + complete(&cq_host->halt_comp); + } + +out: + return IRQ_HANDLED; +} +EXPORT_SYMBOL(cmdq_irq); + +/* cmdq_halt_poll - Halting CQE using polling method. + * @mmc: struct mmc_host + * @halt: bool halt + * This is used mainly from interrupt context to halt/unhalt + * CQE engine. + */ +static int cmdq_halt_poll(struct mmc_host *mmc, bool halt) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + int retries = 100; + + if (!halt) { + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) & ~HALT, + CQCTL); + return 0; + } + + cmdq_set_halt_irq(cq_host, false); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) | HALT, CQCTL); + while (retries) { + if (!(cmdq_readl(cq_host, CQCTL) & HALT)) { + udelay(5); + retries--; + continue; + } else { + if (cq_host->ops->post_cqe_halt) + cq_host->ops->post_cqe_halt(mmc); + /* halt done: re-enable legacy interrupts */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, + false); + mmc_host_set_halt(mmc); + break; + } + } + cmdq_set_halt_irq(cq_host, true); + return retries ? 0 : -ETIMEDOUT; +} + +/* May sleep */ +static int cmdq_halt(struct mmc_host *mmc, bool halt) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + u32 ret = 0; + int retries = 3; + + cmdq_runtime_pm_get(cq_host); + if (halt) { + while (retries) { + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) | HALT, + CQCTL); + ret = wait_for_completion_timeout(&cq_host->halt_comp, + msecs_to_jiffies(HALT_TIMEOUT_MS)); + if (!ret && !(cmdq_readl(cq_host, CQCTL) & HALT)) { + retries--; + continue; + } else { + /* halt done: re-enable legacy interrupts */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, + false); + break; + } + } + ret = retries ? 0 : -ETIMEDOUT; + } else { + if (cq_host->ops->set_transfer_params) + cq_host->ops->set_transfer_params(mmc); + if (cq_host->ops->set_block_size) + cq_host->ops->set_block_size(mmc); + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) & ~HALT, + CQCTL); + } + cmdq_runtime_pm_put(cq_host); + return ret; +} + +static void cmdq_post_req(struct mmc_host *mmc, int tag, int err) +{ + struct cmdq_host *cq_host = NULL; + struct mmc_request *mrq = NULL; + struct mmc_data *data = NULL; + + if (WARN_ON(!mmc)) + return; + + cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + mrq = get_req_by_tag(cq_host, tag); + data = mrq->data; + + if (data) { + data->error = err; + dma_unmap_sg(mmc_dev(mmc), data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (err) + data->bytes_xfered = 0; + else + data->bytes_xfered = blk_rq_bytes(mrq->req); + + } +} + +static void cmdq_dumpstate(struct mmc_host *mmc) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + cmdq_runtime_pm_get(cq_host); + cmdq_dumpregs(cq_host); + cmdq_runtime_pm_put(cq_host); +} + +static const struct mmc_cmdq_host_ops cmdq_host_ops = { + .enable = cmdq_enable, + .disable = cmdq_disable, + .request = cmdq_request, + .post_req = cmdq_post_req, + .halt = cmdq_halt, + .reset = cmdq_reset, + .dumpstate = cmdq_dumpstate, +}; + +struct cmdq_host *cmdq_pltfm_init(struct platform_device *pdev) +{ + struct cmdq_host *cq_host = NULL; + struct resource *cmdq_memres = NULL; + + /* check and setup CMDQ interface */ + cmdq_memres = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "cmdq_mem"); + if (!cmdq_memres) { + dev_dbg(&pdev->dev, "CMDQ not supported\n"); + return ERR_PTR(-EINVAL); + } + + cq_host = kzalloc(sizeof(*cq_host), GFP_KERNEL); + if (!cq_host) { + dev_err(&pdev->dev, "failed to allocate memory for CMDQ\n"); + return ERR_PTR(-ENOMEM); + } + cq_host->mmio = devm_ioremap(&pdev->dev, + cmdq_memres->start, + resource_size(cmdq_memres)); + if (!cq_host->mmio) { + dev_err(&pdev->dev, "failed to remap cmdq regs\n"); + kfree(cq_host); + return ERR_PTR(-EBUSY); + } + dev_dbg(&pdev->dev, "CMDQ ioremap: done\n"); + + return cq_host; +} +EXPORT_SYMBOL(cmdq_pltfm_init); + +int cmdq_init(struct cmdq_host *cq_host, struct mmc_host *mmc, + bool dma64) +{ + int err = 0; + + cq_host->dma64 = dma64; + cq_host->mmc = mmc; + cq_host->mmc->cmdq_private = cq_host; + + cq_host->num_slots = NUM_SLOTS; + cq_host->dcmd_slot = DCMD_SLOT; + + mmc->cmdq_ops = &cmdq_host_ops; + mmc->num_cq_slots = NUM_SLOTS; + mmc->dcmd_cq_slot = DCMD_SLOT; + + cq_host->mrq_slot = kzalloc(sizeof(cq_host->mrq_slot) * + cq_host->num_slots, GFP_KERNEL); + if (!cq_host->mrq_slot) + return -ENOMEM; + + init_completion(&cq_host->halt_comp); + return err; +} +EXPORT_SYMBOL(cmdq_init);