diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 1303dca74cf9..c43d2d268d84 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -54,6 +54,7 @@ #define NFC_REG_H6_RDATA_1 0x004C #define NFC_REG_A10_USER_DATA 0x0050 #define NFC_REG_H6_USER_DATA 0x0080 +#define NFC_H6_USER_DATA_REGS 32 #define NFC_REG_USER_DATA(nfc, x) (nfc->caps->reg_user_data + ((x) * 4)) #define NFC_REG_H6_USER_DATA_LEN 0x0070 /* A USER_DATA_LEN register can hold the length of 8 USER_DATA registers */ @@ -946,44 +947,40 @@ static u8 sunxi_nfc_user_data_sz(struct sunxi_nand_chip *sunxi_nand, return sunxi_nand->user_data_bytes[logical_step]; } -/* - * Variable-length user data is concatenated in four-byte registers. - * PIO uses hardware step zero; DMA slots follow the logical page steps. - */ -static unsigned int -sunxi_nfc_user_data_reg_index(struct sunxi_nand_chip *sunxi_nand, int hw_step) +/* Keep the on-flash layout, but fit each DMA batch in the user-data bank. */ +static int sunxi_nfc_dma_batch_steps(struct nand_chip *nand, + int first_step, int end_step) { - unsigned int byte_offset = 0; + struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); + unsigned int user_data_sz = 0; int i; - for (i = 0; i < hw_step; i++) - byte_offset += sunxi_nfc_user_data_sz(sunxi_nand, i); + if (!nfc->caps->reg_user_data_len) + return end_step - first_step; + + for (i = first_step; i < end_step; i++) { + user_data_sz += sunxi_nfc_user_data_sz(sunxi_nand, i); + if (user_data_sz > NFC_H6_USER_DATA_REGS * sizeof(u32)) + break; + } - return byte_offset / 4; + return i - first_step; } +/* PIO uses register zero; DMA concatenates user data within each batch. */ static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, - int hw_step, bool bbm, int page, + unsigned int reg_index, bool bbm, int page, unsigned int user_data_sz) { struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); u32 user_data; + unsigned int i; - if (!nfc->caps->reg_user_data_len) { - user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, hw_step)); - sunxi_nfc_user_data_to_buf(user_data, oob); - } else { - unsigned int reg_index = sunxi_nfc_user_data_reg_index(sunxi_nand, hw_step); - unsigned int reg_off; - u8 *ptr = oob; - unsigned int i; - - for (i = 0; i < user_data_sz / 4; i++, ptr += 4) { - reg_off = NFC_REG_USER_DATA(nfc, reg_index + i); - user_data = readl(nfc->regs + reg_off); - sunxi_nfc_user_data_to_buf(user_data, ptr); - } + for (i = 0; i < user_data_sz / 4; i++) { + user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i)); + sunxi_nfc_user_data_to_buf(user_data, oob + i * 4); } /* Undo hardware de-randomization for a plain on-flash BBM. */ @@ -1038,7 +1035,8 @@ static void sunxi_nfc_set_user_data_len_pio(struct sunxi_nfc *nfc, int len) nfc->regs + NFC_REG_USER_DATA_LEN(nfc, 0)); } -static void sunxi_nfc_set_user_data_len_dma(struct nand_chip *nand, int nchunks) +static void sunxi_nfc_set_user_data_len_dma(struct nand_chip *nand, + int first_step, int nchunks) { struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); @@ -1054,7 +1052,7 @@ static void sunxi_nfc_set_user_data_len_dma(struct nand_chip *nand, int nchunks) val = 0; for (step = first; step < nchunks && step < first + NFC_REG_USER_DATA_LEN_CAPACITY; step++) { - len = sunxi_nfc_user_data_sz(sunxi_nand, step); + len = sunxi_nfc_user_data_sz(sunxi_nand, first_step + step); code = sunxi_nfc_user_data_len_code(nfc, len); if (code >= 0) val |= field_prep(NFC_USER_DATA_LEN_MSK(step), code); @@ -1064,13 +1062,14 @@ static void sunxi_nfc_set_user_data_len_dma(struct nand_chip *nand, int nchunks) } static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, - const u8 *oob, int hw_step, + const u8 *oob, unsigned int reg_index, bool bbm, int page, unsigned int user_data_sz) { struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); u8 user_data[SUNXI_NFC_MAX_USER_DATA_SZ] = {}; + unsigned int i; /* Pre-randomize the BBM so the hardware writes it plain on flash. */ if (bbm && (nand->options & NAND_NEED_SCRAMBLING) && @@ -1080,18 +1079,9 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, oob = user_data; } - if (!nfc->caps->reg_user_data_len) { - writel(sunxi_nfc_buf_to_user_data(oob), - nfc->regs + NFC_REG_USER_DATA(nfc, hw_step)); - } else { - unsigned int reg_index = sunxi_nfc_user_data_reg_index(sunxi_nand, hw_step); - const u8 *ptr = oob; - unsigned int i; - - for (i = 0; i < user_data_sz / 4; i++, ptr += 4) { - writel(sunxi_nfc_buf_to_user_data(ptr), - nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i)); - } + for (i = 0; i < user_data_sz / 4; i++) { + writel(sunxi_nfc_buf_to_user_data(oob + i * 4), + nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i)); } } @@ -1374,7 +1364,7 @@ static int sunxi_get_ecc_offset(struct sunxi_nand_chip *sunxi_nand, struct nand_ecc_ctrl *ecc, int logical_step) { return sunxi_get_oob_offset(sunxi_nand, ecc, logical_step) + - sunxi_nfc_user_data_sz(sunxi_nand, logical_step); + sunxi_nfc_user_data_sz(sunxi_nand, logical_step); } static int sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, @@ -1568,36 +1558,47 @@ sunxi_nfc_hw_ecc_finish_randomized_read(struct nand_chip *nand, u8 *buf, return result->max_bitflips; } -static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, - int oob_required, int page, - int nchunks) +static int sunxi_nfc_hw_ecc_read_batch_dma(struct nand_chip *nand, u8 *buf, + int oob_required, int page, + int first_step, int nchunks, + struct sunxi_nfc_ecc_status *result, + bool *erased_chunk_found) { bool randomized = nand->options & NAND_NEED_SCRAMBLING; struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; - struct sunxi_nfc_ecc_status result = {}; struct sunxi_nfc_ecc_snapshot snapshot; - unsigned int corrected = mtd->ecc_stats.corrected; - unsigned int failed = mtd->ecc_stats.failed; + unsigned int reg_index = 0, user_data_sz; unsigned int max_bitflips = 0; - bool erased_chunk_found = false; int ret, i; struct scatterlist sg; u32 wait; + if (first_step) { + ret = nand_change_read_column_op(nand, first_step * ecc->size, + NULL, 0, false); + if (ret) + return ret; + } + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); if (ret) return ret; - ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, + ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size, + ecc->size, nchunks, DMA_FROM_DEVICE, &sg); if (ret) return ret; sunxi_nfc_hw_ecc_enable(nand); - sunxi_nfc_set_user_data_len_dma(nand, nchunks); + sunxi_nfc_set_user_data_len_dma(nand, first_step, nchunks); + /* exec_op() restores the page's spare base during column changes. */ + if (first_step) + writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step), + nfc->regs + NFC_REG_SPARE_AREA(nfc)); sunxi_nfc_randomizer_config(nand, page, false); sunxi_nfc_randomizer_enable(nand); @@ -1628,18 +1629,21 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf sunxi_nfc_hw_ecc_read_status(nand, &snapshot); - for (i = 0; i < nchunks; i++) { - int data_off = i * ecc->size; - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + for (i = 0; i < nchunks; i++, reg_index += user_data_sz / 4) { + int logical_step = first_step + i; + int data_off = logical_step * ecc->size; + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step); u8 *data = buf + data_off; u8 *oob = nand->oob_poi + oob_off; bool erased; int bitflips; + user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step); + if (sunxi_nand->randomized_oob) { - sunxi_nfc_hw_ecc_record_status(nand, &result, i, i, &snapshot); - sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, !i, + sunxi_nfc_hw_ecc_record_status(nand, result, logical_step, i, + &snapshot); + sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, reg_index, !logical_step, page, user_data_sz); continue; } @@ -1659,44 +1663,73 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf oob, ecc->bytes + user_data_sz, false); if (ret) - goto err_stats; + return ret; - sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, !i, + sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, reg_index, !logical_step, page, user_data_sz); } if (erased) - erased_chunk_found = true; + *erased_chunk_found = true; sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, bitflips); } if (sunxi_nand->randomized_oob) - return sunxi_nfc_hw_ecc_finish_randomized_read(nand, buf, &result, - oob_required, true, page); + return 0; if (snapshot.status & NFC_ECC_ERR_MSK(nfc)) { for (i = 0; i < nchunks; i++) { - int data_off = i * ecc->size; - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + int logical_step = first_step + i; + int data_off = logical_step * ecc->size; + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step); u8 *data = buf + data_off; u8 *oob = nand->oob_poi + oob_off; if (!(snapshot.status & NFC_ECC_ERR(i))) continue; + user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step); ret = sunxi_nfc_hw_ecc_read_error(nand, data, data_off, oob, mtd->writesize + oob_off, user_data_sz, &max_bitflips); if (ret < 0) - goto err_stats; + return ret; if (ret) - erased_chunk_found = true; + *erased_chunk_found = true; } } + return max_bitflips; +} + +static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, u8 *buf, + int oob_required, int page, int nchunks) +{ + struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct mtd_info *mtd = nand_to_mtd(nand); + struct sunxi_nfc_ecc_status result = {}; + unsigned int corrected = mtd->ecc_stats.corrected; + unsigned int failed = mtd->ecc_stats.failed; + unsigned int max_bitflips = 0; + bool erased_chunk_found = false; + int first_step, batch_steps, ret; + + for (first_step = 0; first_step < nchunks; first_step += batch_steps) { + batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, nchunks); + ret = sunxi_nfc_hw_ecc_read_batch_dma(nand, buf, oob_required, page, + first_step, batch_steps, &result, + &erased_chunk_found); + if (ret < 0) + goto err_stats; + max_bitflips = max_t(unsigned int, max_bitflips, ret); + } + + if (sunxi_nand->randomized_oob) + return sunxi_nfc_hw_ecc_finish_randomized_read(nand, buf, &result, + oob_required, true, page); + if (oob_required) { ret = sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, NULL, !erased_chunk_found, page); @@ -2029,68 +2062,87 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, { struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; struct scatterlist sg; u32 wait; - int ret, i; + int first_step, batch_steps, ret, i; sunxi_nfc_select_chip(nand, nand->cur_cs); - ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); - if (ret) - return ret; + for (first_step = 0; first_step < ecc->steps; first_step += batch_steps) { + unsigned int reg_index = 0; - ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, - DMA_TO_DEVICE, &sg); - if (ret) - goto pio_fallback; + batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, ecc->steps); + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); + if (ret) + return ret; - sunxi_nfc_set_user_data_len_dma(nand, ecc->steps); - for (i = 0; i < ecc->steps; i++) { - unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); - int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); - const u8 *oob = nand->oob_poi + oob_off; + ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size, + ecc->size, batch_steps, DMA_TO_DEVICE, &sg); + if (ret) { + /* Only retry before any part of the page has been transferred. */ + if (first_step) + return ret; + goto pio_fallback; + } - sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page, - user_data_sz); - } + sunxi_nfc_set_user_data_len_dma(nand, first_step, batch_steps); + for (i = first_step; i < first_step + batch_steps; i++) { + unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i); + int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i); + const u8 *oob = nand->oob_poi + oob_off; - ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0); - if (ret) { - sunxi_nfc_dma_op_abort(nfc); - sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); - return ret; - } + sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, reg_index, !i, + page, user_data_sz); + reg_index += user_data_sz / 4; + } - sunxi_nfc_hw_ecc_enable(nand); - sunxi_nfc_randomizer_config(nand, page, false); - sunxi_nfc_randomizer_enable(nand); + if (first_step) + ret = nand_change_write_column_op(nand, first_step * ecc->size, + NULL, 0, false); + else + ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0); + if (ret) { + sunxi_nfc_dma_op_abort(nfc); + sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + return ret; + } - writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, - nfc->regs + NFC_REG_WCMD_SET); + /* exec_op() restores the page's spare base during column changes. */ + if (first_step) + writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step), + nfc->regs + NFC_REG_SPARE_AREA(nfc)); + sunxi_nfc_hw_ecc_enable(nand); + sunxi_nfc_randomizer_config(nand, page, false); + sunxi_nfc_randomizer_enable(nand); - wait = NFC_CMD_INT_FLAG; + writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, + nfc->regs + NFC_REG_WCMD_SET); - if (nfc->use_mdma) - wait |= NFC_DMA_INT_FLAG; - else - dma_async_issue_pending(nfc->dmac); + wait = NFC_CMD_INT_FLAG; - writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | - NFC_DATA_TRANS | NFC_ACCESS_DIR, - nfc->regs + NFC_REG_CMD); + if (nfc->use_mdma) + wait |= NFC_DMA_INT_FLAG; + else + dma_async_issue_pending(nfc->dmac); - ret = sunxi_nfc_wait_events(nfc, wait, false, 0); - if (ret) - sunxi_nfc_dma_op_abort(nfc); + writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | + NFC_DATA_TRANS | NFC_ACCESS_DIR, + nfc->regs + NFC_REG_CMD); - sunxi_nfc_randomizer_disable(nand); - sunxi_nfc_hw_ecc_disable(nand); + ret = sunxi_nfc_wait_events(nfc, wait, false, 0); + if (ret) + sunxi_nfc_dma_op_abort(nfc); - sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + sunxi_nfc_randomizer_disable(nand); + sunxi_nfc_hw_ecc_disable(nand); - if (ret) - return ret; + sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); + + if (ret) + return ret; + } if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) { /* TODO: use DMA to transfer extra OOB bytes ? */