1// SPDX-License-Identifier: (GPL-2.0+ OR MIT) 2/* 3 * Amlogic Meson Nand Flash Controller Driver 4 * 5 * Copyright (c) 2018 Amlogic, inc. 6 * Author: Liang Yang <liang.yang@amlogic.com> 7 * 8 * Copyright (c) 2023 SaluteDevices, Inc. 9 * Author: Arseniy Krasnov <avkrasnov@salutedevices.com> 10 */ 11 12#include <nand.h> 13#include <asm/io.h> 14#include <dm.h> 15#include <dm/device_compat.h> 16#include <dm/ofnode.h> 17#include <dm/uclass.h> 18#include <linux/bug.h> 19#include <linux/clk-provider.h> 20#include <linux/delay.h> 21#include <linux/dma-mapping.h> 22#include <linux/iopoll.h> 23#include <linux/mtd/mtd.h> 24#include <linux/mtd/rawnand.h> 25#include <linux/sizes.h> 26 27#define NFC_CMD_IDLE (0xc << 14) 28#define NFC_CMD_CLE (0x5 << 14) 29#define NFC_CMD_ALE (0x6 << 14) 30#define NFC_CMD_DWR (0x4 << 14) 31#define NFC_CMD_DRD (0x8 << 14) 32#define NFC_CMD_ADL ((0 << 16) | (3 << 20)) 33#define NFC_CMD_ADH ((1 << 16) | (3 << 20)) 34#define NFC_CMD_AIL ((2 << 16) | (3 << 20)) 35#define NFC_CMD_AIH ((3 << 16) | (3 << 20)) 36#define NFC_CMD_SEED ((8 << 16) | (3 << 20)) 37#define NFC_CMD_M2N ((0 << 17) | (2 << 20)) 38#define NFC_CMD_N2M ((1 << 17) | (2 << 20)) 39#define NFC_CMD_RB BIT(20) 40#define NFC_CMD_SCRAMBLER_ENABLE BIT(19) 41#define NFC_CMD_SCRAMBLER_DISABLE 0 42#define NFC_CMD_SHORTMODE_DISABLE 0 43#define NFC_CMD_RB_INT BIT(14) 44#define NFC_CMD_RB_INT_NO_PIN ((0xb << 10) | BIT(18) | BIT(16)) 45 46#define NFC_CMD_GET_SIZE(x) (((x) >> 22) & GENMASK(4, 0)) 47 48#define NFC_REG_CMD 0x00 49#define NFC_REG_CFG 0x04 50#define NFC_REG_DADR 0x08 51#define NFC_REG_IADR 0x0c 52#define NFC_REG_BUF 0x10 53#define NFC_REG_INFO 0x14 54#define NFC_REG_DC 0x18 55#define NFC_REG_ADR 0x1c 56#define NFC_REG_DL 0x20 57#define NFC_REG_DH 0x24 58#define NFC_REG_CADR 0x28 59#define NFC_REG_SADR 0x2c 60#define NFC_REG_PINS 0x30 61#define NFC_REG_VER 0x38 62 63#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \ 64 ( \ 65 (cmd_dir) | \ 66 (ran) | \ 67 ((bch) << 14) | \ 68 ((short_mode) << 13) | \ 69 (((page_size) & 0x7f) << 6) | \ 70 ((pages) & 0x3f) \ 71 ) 72 73#define GENCMDDADDRL(adl, addr) ((adl) | ((addr) & 0xffff)) 74#define GENCMDDADDRH(adh, addr) ((adh) | (((addr) >> 16) & 0xffff)) 75#define GENCMDIADDRL(ail, addr) ((ail) | ((addr) & 0xffff)) 76#define GENCMDIADDRH(aih, addr) ((aih) | (((addr) >> 16) & 0xffff)) 77 78#define DMA_DIR(dir) ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N) 79 80#define ECC_CHECK_RETURN_FF -1 81 82#define NAND_CE0 (0xe << 10) 83#define NAND_CE1 (0xd << 10) 84 85#define DMA_BUSY_TIMEOUT_US 1000000 86#define CMD_DRAIN_TIMEOUT_US 1000 87#define ECC_POLL_TIMEOUT_US 15 88 89#define MAX_CE_NUM 2 90 91/* eMMC clock register, misc control */ 92#define CLK_SELECT_NAND BIT(31) 93#define CLK_ALWAYS_ON_NAND BIT(24) 94#define CLK_ENABLE_VALUE 0x245 95 96#define DIRREAD 1 97#define DIRWRITE 0 98 99#define ECC_PARITY_BCH8_512B 14 100#define ECC_COMPLETE BIT(31) 101#define ECC_ERR_CNT(x) (((x) >> 24) & GENMASK(5, 0)) 102#define ECC_ZERO_CNT(x) (((x) >> 16) & GENMASK(5, 0)) 103#define ECC_UNCORRECTABLE 0x3f 104 105#define PER_INFO_BYTE 8 106 107#define NFC_SEND_CMD(host, cmd) \ 108 (writel((cmd), (host)->reg_base + NFC_REG_CMD)) 109 110#define NFC_GET_CMD(host) \ 111 (readl((host)->reg_base + NFC_REG_CMD)) 112 113#define NFC_CMDFIFO_SIZE(host) ((NFC_GET_CMD((host)) >> 22) & GENMASK(4, 0)) 114 115#define NFC_CMD_MAKE_IDLE(ce, delay) ((ce) | NFC_CMD_IDLE | ((delay) & 0x3ff)) 116#define NFC_CMD_MAKE_DRD(ce, size) ((ce) | NFC_CMD_DRD | (size)) 117#define NFC_CMD_MAKE_DWR(ce, data) ((ce) | NFC_CMD_DWR | ((data) & 0xff)) 118#define NFC_CMD_MAKE_CLE(ce, cmd_val) ((ce) | NFC_CMD_CLE | ((cmd_val) & 0xff)) 119#define NFC_CMD_MAKE_ALE(ce, addr) ((ce) | NFC_CMD_ALE | ((addr) & 0xff)) 120 121#define NAND_TWB_TIME_CYCLE 10 122 123#define NFC_DEV_READY_TICK_MAX 5000 124 125/* Both values are recommended by vendor, as the most 126 * tested with almost all SLC NAND flash. Second value 127 * could be calculated dynamically from timing parameters, 128 * but we need both values for initial start of the NAND 129 * controller (e.g. before NAND subsystem processes timings), 130 * so use hardcoded constants. 131 */ 132#define NFC_DEFAULT_BUS_CYCLE 6 133#define NFC_DEFAULT_BUS_TIMING 7 134 135#define NFC_SEED_OFFSET 0xc2 136#define NFC_SEED_MASK 0x7fff 137 138#define DMA_ADDR_ALIGN 8 139 140struct meson_nfc_nand_chip { 141 struct list_head node; 142 struct nand_chip nand; 143 144 u32 bch_mode; 145 u8 *data_buf; 146 __le64 *info_buf; 147 u32 nsels; 148 u8 sels[]; 149}; 150 151struct meson_nfc_param { 152 u32 chip_select; 153 u32 rb_select; 154}; 155 156struct meson_nfc { 157 void __iomem *reg_base; 158 void __iomem *reg_clk; 159 struct list_head chips; 160 struct meson_nfc_param param; 161 struct udevice *dev; 162 dma_addr_t daddr; 163 dma_addr_t iaddr; 164 u32 data_bytes; 165 u32 info_bytes; 166 u64 assigned_cs; 167}; 168 169struct meson_nand_ecc { 170 u32 bch; 171 u32 strength; 172 u32 size; 173}; 174 175enum { 176 NFC_ECC_BCH8_512 = 1, 177 NFC_ECC_BCH8_1K, 178 NFC_ECC_BCH24_1K, 179 NFC_ECC_BCH30_1K, 180 NFC_ECC_BCH40_1K, 181 NFC_ECC_BCH50_1K, 182 NFC_ECC_BCH60_1K, 183}; 184 185#define MESON_ECC_DATA(b, s, sz) { .bch = (b), .strength = (s), .size = (sz) } 186 187static struct meson_nand_ecc meson_ecc[] = { 188 MESON_ECC_DATA(NFC_ECC_BCH8_512, 8, 512), 189 MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8, 1024), 190}; 191 192static int meson_nand_calc_ecc_bytes(int step_size, int strength) 193{ 194 int ecc_bytes; 195 196 if (step_size == 512 && strength == 8) 197 return ECC_PARITY_BCH8_512B; 198 199 ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8); 200 ecc_bytes = ALIGN(ecc_bytes, 2); 201 202 return ecc_bytes; 203} 204 205static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand) 206{ 207 return container_of(nand, struct meson_nfc_nand_chip, nand); 208} 209 210static void meson_nfc_nand_select_chip(struct mtd_info *mtd, int chip) 211{ 212 struct nand_chip *nand = mtd_to_nand(mtd); 213 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 214 struct meson_nfc *nfc = nand_get_controller_data(nand); 215 216 nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0; 217} 218 219static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time) 220{ 221 writel(NFC_CMD_MAKE_IDLE(nfc->param.chip_select, time), 222 nfc->reg_base + NFC_REG_CMD); 223} 224 225static void meson_nfc_cmd_seed(const struct meson_nfc *nfc, u32 seed) 226{ 227 writel(NFC_CMD_SEED | (NFC_SEED_OFFSET + (seed & NFC_SEED_MASK)), 228 nfc->reg_base + NFC_REG_CMD); 229} 230 231static void meson_nfc_cmd_access(struct nand_chip *nand, bool raw, bool dir, 232 int scrambler) 233{ 234 struct mtd_info *mtd = nand_to_mtd(nand); 235 const struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 236 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 237 u32 bch = meson_chip->bch_mode, cmd; 238 int len = mtd->writesize, pagesize, pages; 239 240 pagesize = nand->ecc.size; 241 242 if (raw) { 243 len = mtd->writesize + mtd->oobsize; 244 cmd = len | scrambler | DMA_DIR(dir); 245 writel(cmd, nfc->reg_base + NFC_REG_CMD); 246 return; 247 } 248 249 pages = len / nand->ecc.size; 250 251 cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch, 252 NFC_CMD_SHORTMODE_DISABLE, pagesize, pages); 253 254 writel(cmd, nfc->reg_base + NFC_REG_CMD); 255} 256 257static void meson_nfc_drain_cmd(struct meson_nfc *nfc) 258{ 259 /* 260 * Insert two commands to make sure all valid commands are finished. 261 * 262 * The Nand flash controller is designed as two stages pipleline - 263 * a) fetch and b) execute. 264 * There might be cases when the driver see command queue is empty, 265 * but the Nand flash controller still has two commands buffered, 266 * one is fetched into NFC request queue (ready to run), and another 267 * is actively executing. So pushing 2 "IDLE" commands guarantees that 268 * the pipeline is emptied. 269 */ 270 meson_nfc_cmd_idle(nfc, 0); 271 meson_nfc_cmd_idle(nfc, 0); 272} 273 274static int meson_nfc_wait_cmd_finish(const struct meson_nfc *nfc, 275 unsigned int timeout_us) 276{ 277 u32 cmd_size = 0; 278 279 /* wait cmd fifo is empty */ 280 return readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size, 281 !NFC_CMD_GET_SIZE(cmd_size), 282 timeout_us); 283} 284 285static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc) 286{ 287 meson_nfc_drain_cmd(nfc); 288 289 return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT_US); 290} 291 292static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i) 293{ 294 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 295 int len; 296 297 len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i; 298 299 return meson_chip->data_buf + len; 300} 301 302static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i) 303{ 304 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 305 int len, temp; 306 307 temp = nand->ecc.size + nand->ecc.bytes; 308 len = (temp + 2) * i; 309 310 return meson_chip->data_buf + len; 311} 312 313static void meson_nfc_get_data_oob(struct nand_chip *nand, 314 u8 *buf, u8 *oobbuf) 315{ 316 u8 *dsrc, *osrc; 317 int i, oob_len; 318 319 oob_len = nand->ecc.bytes + 2; 320 for (i = 0; i < nand->ecc.steps; i++) { 321 if (buf) { 322 dsrc = meson_nfc_data_ptr(nand, i); 323 memcpy(buf, dsrc, nand->ecc.size); 324 buf += nand->ecc.size; 325 } 326 327 if (oobbuf) { 328 osrc = meson_nfc_oob_ptr(nand, i); 329 memcpy(oobbuf, osrc, oob_len); 330 oobbuf += oob_len; 331 } 332 } 333} 334 335static void meson_nfc_set_data_oob(struct nand_chip *nand, 336 const u8 *buf, u8 *oobbuf) 337{ 338 int i, oob_len; 339 340 oob_len = nand->ecc.bytes + 2; 341 for (i = 0; i < nand->ecc.steps; i++) { 342 u8 *osrc; 343 344 if (buf) { 345 u8 *dsrc; 346 347 dsrc = meson_nfc_data_ptr(nand, i); 348 memcpy(dsrc, buf, nand->ecc.size); 349 buf += nand->ecc.size; 350 } 351 352 osrc = meson_nfc_oob_ptr(nand, i); 353 memcpy(osrc, oobbuf, oob_len); 354 oobbuf += oob_len; 355 } 356} 357 358static void meson_nfc_set_user_byte(struct nand_chip *nand, const u8 *oob_buf) 359{ 360 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 361 int i, count; 362 363 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) { 364 __le64 *info = &meson_chip->info_buf[i]; 365 366 *info |= oob_buf[count]; 367 *info |= oob_buf[count + 1] << 8; 368 } 369} 370 371static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf) 372{ 373 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 374 int i, count; 375 376 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) { 377 const __le64 *info = &meson_chip->info_buf[i]; 378 379 oob_buf[count] = *info; 380 oob_buf[count + 1] = *info >> 8; 381 } 382} 383 384static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips, 385 u64 *correct_bitmap) 386{ 387 struct mtd_info *mtd = nand_to_mtd(nand); 388 int ret = 0, i; 389 390 for (i = 0; i < nand->ecc.steps; i++) { 391 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 392 const __le64 *info = &meson_chip->info_buf[i]; 393 394 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) { 395 mtd->ecc_stats.corrected += ECC_ERR_CNT(*info); 396 *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info)); 397 *correct_bitmap |= BIT_ULL(i); 398 continue; 399 } 400 401 if ((nand->options & NAND_NEED_SCRAMBLING) && 402 ECC_ZERO_CNT(*info) < nand->ecc.strength) { 403 mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info); 404 *bitflips = max_t(u32, *bitflips, 405 ECC_ZERO_CNT(*info)); 406 ret = ECC_CHECK_RETURN_FF; 407 } else { 408 ret = -EBADMSG; 409 } 410 } 411 412 return ret; 413} 414 415static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf, 416 int datalen, void *infobuf, int infolen, 417 enum dma_data_direction dir) 418{ 419 struct meson_nfc *nfc = nand_get_controller_data(nand); 420 int ret; 421 u32 cmd; 422 423 nfc->daddr = dma_map_single(databuf, datalen, DMA_BIDIRECTIONAL); 424 ret = dma_mapping_error(nfc->dev, nfc->daddr); 425 if (ret) 426 return ret; 427 428 cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr); 429 writel(cmd, nfc->reg_base + NFC_REG_CMD); 430 431 cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr); 432 writel(cmd, nfc->reg_base + NFC_REG_CMD); 433 434 if (infobuf) { 435 nfc->iaddr = dma_map_single(infobuf, infolen, 436 DMA_BIDIRECTIONAL); 437 ret = dma_mapping_error(nfc->dev, nfc->iaddr); 438 if (ret) { 439 dma_unmap_single(nfc->daddr, datalen, dir); 440 return ret; 441 } 442 443 nfc->info_bytes = infolen; 444 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr); 445 writel(cmd, nfc->reg_base + NFC_REG_CMD); 446 447 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr); 448 writel(cmd, nfc->reg_base + NFC_REG_CMD); 449 } 450 451 return 0; 452} 453 454static void meson_nfc_dma_buffer_release(struct nand_chip *nand, 455 int datalen, int infolen, 456 enum dma_data_direction dir) 457{ 458 struct meson_nfc *nfc = nand_get_controller_data(nand); 459 460 dma_unmap_single(nfc->daddr, datalen, dir); 461 462 if (infolen) { 463 dma_unmap_single(nfc->iaddr, infolen, dir); 464 nfc->info_bytes = 0; 465 } 466} 467 468static void meson_nfc_read_buf(struct mtd_info *mtd, u8 *buf, int size) 469{ 470 struct nand_chip *nand = mtd_to_nand(mtd); 471 struct meson_nfc *nfc = nand_get_controller_data(nand); 472 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 473 u8 *dma_buf; 474 int ret; 475 u32 cmd; 476 477 if ((uintptr_t)buf % DMA_ADDR_ALIGN) { 478 unsigned long tmp_addr; 479 480 dma_buf = dma_alloc_coherent(size, &tmp_addr); 481 if (!dma_buf) 482 return; 483 } else { 484 dma_buf = buf; 485 } 486 487 ret = meson_nfc_dma_buffer_setup(nand, dma_buf, size, meson_chip->info_buf, 488 PER_INFO_BYTE, DMA_FROM_DEVICE); 489 if (ret) { 490 pr_err("Failed to setup DMA buffer %p/%p\n", dma_buf, 491 meson_chip->info_buf); 492 return; 493 } 494 495 cmd = NFC_CMD_N2M | size; 496 writel(cmd, nfc->reg_base + NFC_REG_CMD); 497 498 meson_nfc_drain_cmd(nfc); 499 meson_nfc_wait_cmd_finish(nfc, CMD_DRAIN_TIMEOUT_US); 500 meson_nfc_dma_buffer_release(nand, size, PER_INFO_BYTE, DMA_FROM_DEVICE); 501 502 if (buf != dma_buf) { 503 memcpy(buf, dma_buf, size); 504 dma_free_coherent(dma_buf); 505 } 506} 507 508static void meson_nfc_write_buf(struct mtd_info *mtd, const u8 *buf, int size) 509{ 510 struct nand_chip *nand = mtd_to_nand(mtd); 511 struct meson_nfc *nfc = nand_get_controller_data(nand); 512 u8 *dma_buf; 513 int ret; 514 u32 cmd; 515 516 if ((uintptr_t)buf % DMA_ADDR_ALIGN) { 517 unsigned long tmp_addr; 518 519 dma_buf = dma_alloc_coherent(size, &tmp_addr); 520 if (!dma_buf) 521 return; 522 523 memcpy(dma_buf, buf, size); 524 } else { 525 dma_buf = (u8 *)buf; 526 } 527 528 ret = meson_nfc_dma_buffer_setup(nand, (void *)dma_buf, size, NULL, 529 0, DMA_TO_DEVICE); 530 if (ret) { 531 pr_err("Failed to setup DMA buffer %p\n", dma_buf); 532 return; 533 } 534 535 cmd = NFC_CMD_M2N | size; 536 writel(cmd, nfc->reg_base + NFC_REG_CMD); 537 538 meson_nfc_drain_cmd(nfc); 539 meson_nfc_wait_cmd_finish(nfc, CMD_DRAIN_TIMEOUT_US); 540 meson_nfc_dma_buffer_release(nand, size, 0, DMA_TO_DEVICE); 541 542 if (buf != dma_buf) 543 dma_free_coherent(dma_buf); 544} 545 546static int meson_nfc_write_page_sub(struct nand_chip *nand, 547 int page, bool raw) 548{ 549 const struct mtd_info *mtd = nand_to_mtd(nand); 550 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 551 struct meson_nfc *nfc = nand_get_controller_data(nand); 552 int data_len, info_len; 553 int ret; 554 u32 cmd; 555 556 data_len = mtd->writesize + mtd->oobsize; 557 info_len = nand->ecc.steps * PER_INFO_BYTE; 558 559 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf, 560 data_len, meson_chip->info_buf, 561 info_len, DMA_TO_DEVICE); 562 if (ret) { 563 pr_err("Failed to setup DMA buffer %p/%p\n", 564 meson_chip->data_buf, meson_chip->info_buf); 565 return ret; 566 } 567 568 if (nand->options & NAND_NEED_SCRAMBLING) { 569 meson_nfc_cmd_seed(nfc, page); 570 meson_nfc_cmd_access(nand, raw, DIRWRITE, 571 NFC_CMD_SCRAMBLER_ENABLE); 572 } else { 573 meson_nfc_cmd_access(nand, raw, DIRWRITE, 574 NFC_CMD_SCRAMBLER_DISABLE); 575 } 576 577 cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG; 578 writel(cmd, nfc->reg_base + NFC_REG_CMD); 579 580 meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE); 581 582 return 0; 583} 584 585static int meson_nfc_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip, 586 const u8 *buf, int oob_required, int page) 587{ 588 meson_nfc_set_data_oob(chip, buf, oob_required ? chip->oob_poi : NULL); 589 590 return meson_nfc_write_page_sub(chip, page, true); 591} 592 593static int meson_nfc_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, 594 const u8 *buf, int oob_required, int page) 595{ 596 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(chip); 597 598 if (buf) 599 memcpy(meson_chip->data_buf, buf, mtd->writesize); 600 601 memset(meson_chip->info_buf, 0, chip->ecc.steps * PER_INFO_BYTE); 602 603 if (oob_required) 604 meson_nfc_set_user_byte(chip, chip->oob_poi); 605 606 return meson_nfc_write_page_sub(chip, page, false); 607} 608 609static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc, 610 struct nand_chip *nand, bool raw) 611{ 612 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 613 __le64 *info; 614 u32 neccpages; 615 int ret; 616 617 neccpages = raw ? 1 : nand->ecc.steps; 618 info = &meson_chip->info_buf[neccpages - 1]; 619 do { 620 udelay(ECC_POLL_TIMEOUT_US); 621 /* info is updated by nfc dma engine*/ 622 rmb(); 623 invalidate_dcache_range(nfc->iaddr, nfc->iaddr + nfc->info_bytes); 624 ret = *info & ECC_COMPLETE; 625 } while (!ret); 626} 627 628static int meson_nfc_read_page_sub(struct nand_chip *nand, 629 int page, bool raw) 630{ 631 struct mtd_info *mtd = nand_to_mtd(nand); 632 struct meson_nfc *nfc = nand_get_controller_data(nand); 633 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 634 u32 data_len, info_len; 635 int ret; 636 637 data_len = mtd->writesize + mtd->oobsize; 638 info_len = nand->ecc.steps * PER_INFO_BYTE; 639 640 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf, data_len, 641 meson_chip->info_buf, info_len, 642 DMA_FROM_DEVICE); 643 if (ret) 644 return ret; 645 646 if (nand->options & NAND_NEED_SCRAMBLING) { 647 meson_nfc_cmd_seed(nfc, page); 648 meson_nfc_cmd_access(nand, raw, DIRREAD, 649 NFC_CMD_SCRAMBLER_ENABLE); 650 } else { 651 meson_nfc_cmd_access(nand, raw, DIRREAD, 652 NFC_CMD_SCRAMBLER_DISABLE); 653 } 654 655 meson_nfc_wait_dma_finish(nfc); 656 meson_nfc_check_ecc_pages_valid(nfc, nand, raw); 657 658 meson_nfc_dma_buffer_release(nand, data_len, info_len, 659 DMA_FROM_DEVICE); 660 661 return 0; 662} 663 664static int meson_nfc_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip, 665 u8 *buf, int oob_required, int page) 666{ 667 int ret; 668 669 ret = meson_nfc_read_page_sub(chip, page, true); 670 if (ret) 671 return ret; 672 673 meson_nfc_get_data_oob(chip, buf, oob_required ? chip->oob_poi : NULL); 674 675 return 0; 676} 677 678static int meson_nfc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip, 679 u8 *buf, int oob_required, int page) 680{ 681 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(chip); 682 u64 correct_bitmap = 0; 683 u32 bitflips = 0; 684 int ret; 685 686 ret = meson_nfc_read_page_sub(chip, page, false); 687 if (ret) 688 return ret; 689 690 if (oob_required) 691 meson_nfc_get_user_byte(chip, chip->oob_poi); 692 693 ret = meson_nfc_ecc_correct(chip, &bitflips, &correct_bitmap); 694 695 if (ret == ECC_CHECK_RETURN_FF) { 696 if (buf) 697 memset(buf, 0xff, mtd->writesize); 698 699 if (oob_required) 700 memset(chip->oob_poi, 0xff, mtd->oobsize); 701 } else if (ret < 0) { 702 struct nand_ecc_ctrl *ecc; 703 int i; 704 705 if ((chip->options & NAND_NEED_SCRAMBLING) || !buf) { 706 mtd->ecc_stats.failed++; 707 return bitflips; 708 } 709 710 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page); 711 712 ret = meson_nfc_read_page_raw(mtd, chip, buf, 1, page); 713 if (ret) 714 return ret; 715 716 ecc = &chip->ecc; 717 718 for (i = 0; i < chip->ecc.steps ; i++) { 719 u8 *data = buf + i * ecc->size; 720 u8 *oob = chip->oob_poi + i * (ecc->bytes + 2); 721 722 if (correct_bitmap & BIT_ULL(i)) 723 continue; 724 725 ret = nand_check_erased_ecc_chunk(data, ecc->size, 726 oob, ecc->bytes + 2, 727 NULL, 0, 728 ecc->strength); 729 if (ret < 0) { 730 mtd->ecc_stats.failed++; 731 } else { 732 mtd->ecc_stats.corrected += ret; 733 bitflips = max_t(u32, bitflips, ret); 734 } 735 } 736 } else if (buf && buf != meson_chip->data_buf) { 737 memcpy(buf, meson_chip->data_buf, mtd->writesize); 738 } 739 740 return bitflips; 741} 742 743static int meson_nfc_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, 744 int page) 745{ 746 int ret; 747 748 ret = nand_read_page_op(chip, page, 0, NULL, 0); 749 if (ret) 750 return ret; 751 752 return meson_nfc_read_page_raw(mtd, chip, NULL, 1, page); 753} 754 755static int meson_nfc_read_oob(struct mtd_info *mtd, struct nand_chip *chip, 756 int page) 757{ 758 int ret; 759 760 ret = nand_read_page_op(chip, page, 0, NULL, 0); 761 if (ret) 762 return ret; 763 764 return meson_nfc_read_page_hwecc(mtd, chip, NULL, 1, page); 765} 766 767static int meson_nfc_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip, 768 int page) 769{ 770 int ret; 771 772 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0); 773 if (ret) 774 return ret; 775 776 ret = meson_nfc_write_page_raw(mtd, chip, NULL, 1, page); 777 if (ret) 778 return ret; 779 780 return nand_prog_page_end_op(chip); 781} 782 783static int meson_nfc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, 784 int page) 785{ 786 int ret; 787 788 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0); 789 if (ret) 790 return ret; 791 792 ret = meson_nfc_write_page_hwecc(mtd, chip, NULL, 1, page); 793 if (ret) 794 return ret; 795 796 return nand_prog_page_end_op(chip); 797} 798 799static void meson_nfc_nand_cmd_function(struct mtd_info *mtd, unsigned int command, 800 int column, int page_addr) 801{ 802 struct nand_chip *chip = mtd_to_nand(mtd); 803 804 chip->cmd_ctrl(mtd, command, NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); 805 806 if (column != -1 || page_addr != -1) { 807 int ctrl = NAND_CTRL_CHANGE | NAND_NCE | NAND_ALE; 808 809 /* Serially input address */ 810 if (column != -1) { 811 /* Adjust columns for 16 bit buswidth */ 812 if (chip->options & NAND_BUSWIDTH_16 && 813 !nand_opcode_8bits(command)) 814 column >>= 1; 815 816 chip->cmd_ctrl(mtd, column, ctrl); 817 ctrl &= ~NAND_CTRL_CHANGE; 818 /* Only output a single addr cycle for 8bits 819 * opcodes. 820 */ 821 if (!nand_opcode_8bits(command)) 822 chip->cmd_ctrl(mtd, column >> 8, ctrl); 823 } 824 825 if (page_addr != -1) { 826 chip->cmd_ctrl(mtd, page_addr, ctrl); 827 chip->cmd_ctrl(mtd, page_addr >> 8, NAND_NCE | 828 NAND_ALE); 829 /* One more address cycle for devices > 128MiB */ 830 if (chip->chipsize > SZ_128M) 831 chip->cmd_ctrl(mtd, page_addr >> 16, 832 NAND_NCE | NAND_ALE); 833 } 834 835 switch (command) { 836 case NAND_CMD_READ0: 837 chip->cmd_ctrl(mtd, NAND_CMD_READSTART, 838 NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); 839 fallthrough; 840 case NAND_CMD_PARAM: 841 nand_wait_ready(mtd); 842 nand_exit_status_op(chip); 843 } 844 } 845} 846 847static void meson_nfc_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl) 848{ 849 struct nand_chip *nand = mtd_to_nand(mtd); 850 struct meson_nfc *nfc = nand_get_controller_data(nand); 851 852 if (cmd == NAND_CMD_NONE) 853 return; 854 855 if (ctrl & NAND_CLE) 856 cmd = NFC_CMD_MAKE_CLE(nfc->param.chip_select, cmd); 857 else 858 cmd = NFC_CMD_MAKE_ALE(nfc->param.chip_select, cmd); 859 860 writel(cmd, nfc->reg_base + NFC_REG_CMD); 861} 862 863static void meson_nfc_wait_cmd_fifo(struct meson_nfc *nfc) 864{ 865 while ((NFC_GET_CMD(nfc) >> 22) & GENMASK(4, 0)) 866 ; 867} 868 869static u8 meson_nfc_nand_read_byte(struct mtd_info *mtd) 870{ 871 struct nand_chip *nand = mtd_to_nand(mtd); 872 struct meson_nfc *nfc = nand_get_controller_data(nand); 873 874 writel(NFC_CMD_MAKE_DRD(nfc->param.chip_select, 0), nfc->reg_base + NFC_REG_CMD); 875 876 meson_nfc_cmd_idle(nfc, NAND_TWB_TIME_CYCLE); 877 meson_nfc_cmd_idle(nfc, 0); 878 meson_nfc_cmd_idle(nfc, 0); 879 880 meson_nfc_wait_cmd_fifo(nfc); 881 882 return readl(nfc->reg_base + NFC_REG_BUF); 883} 884 885static void meson_nfc_nand_write_byte(struct mtd_info *mtd, u8 val) 886{ 887 struct nand_chip *nand = mtd_to_nand(mtd); 888 struct meson_nfc *nfc = nand_get_controller_data(nand); 889 890 meson_nfc_cmd_idle(nfc, NAND_TWB_TIME_CYCLE); 891 892 writel(NFC_CMD_MAKE_DWR(nfc->param.chip_select, val), nfc->reg_base + NFC_REG_CMD); 893 894 meson_nfc_cmd_idle(nfc, NAND_TWB_TIME_CYCLE); 895 meson_nfc_cmd_idle(nfc, 0); 896 meson_nfc_cmd_idle(nfc, 0); 897 898 meson_nfc_wait_cmd_fifo(nfc); 899} 900 901static int meson_nfc_dev_ready(struct mtd_info *mtd) 902{ 903 struct nand_chip *chip = mtd_to_nand(mtd); 904 unsigned int time_out_cnt = 0; 905 906 chip->select_chip(mtd, 0); 907 908 chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1); 909 910 do { 911 int status; 912 913 status = (int)chip->read_byte(mtd); 914 if (status & NAND_STATUS_READY) 915 break; 916 } while (time_out_cnt++ < NFC_DEV_READY_TICK_MAX); 917 918 return time_out_cnt != NFC_DEV_READY_TICK_MAX; 919} 920 921static int meson_chip_buffer_init(struct nand_chip *nand) 922{ 923 const struct mtd_info *mtd = nand_to_mtd(nand); 924 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 925 u32 page_bytes, info_bytes, nsectors; 926 unsigned long tmp_addr; 927 928 nsectors = mtd->writesize / nand->ecc.size; 929 930 page_bytes = mtd->writesize + mtd->oobsize; 931 info_bytes = nsectors * PER_INFO_BYTE; 932 933 meson_chip->data_buf = dma_alloc_coherent(page_bytes, &tmp_addr); 934 if (!meson_chip->data_buf) 935 return -ENOMEM; 936 937 meson_chip->info_buf = dma_alloc_coherent(info_bytes, &tmp_addr); 938 if (!meson_chip->info_buf) { 939 dma_free_coherent(meson_chip->data_buf); 940 return -ENOMEM; 941 } 942 943 return 0; 944} 945 946static const int axg_stepinfo_strengths[] = { 8 }; 947static const struct nand_ecc_step_info axg_stepinfo_1024 = { 948 .stepsize = 1024, 949 .strengths = axg_stepinfo_strengths, 950 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths) 951}; 952 953static const struct nand_ecc_step_info axg_stepinfo_512 = { 954 .stepsize = 512, 955 .strengths = axg_stepinfo_strengths, 956 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths) 957}; 958 959static const struct nand_ecc_step_info axg_stepinfo[] = { axg_stepinfo_1024, axg_stepinfo_512 }; 960 961static const struct nand_ecc_caps meson_axg_ecc_caps = { 962 .stepinfos = axg_stepinfo, 963 .nstepinfos = ARRAY_SIZE(axg_stepinfo), 964 .calc_ecc_bytes = meson_nand_calc_ecc_bytes, 965}; 966 967/* 968 * OOB layout: 969 * 970 * For ECC with 512 bytes step size: 971 * 0x00: AA AA BB BB BB BB BB BB BB BB BB BB BB BB BB BB 972 * 0x10: AA AA CC CC CC CC CC CC CC CC CC CC CC CC CC CC 973 * 0x20: 974 * 0x30: 975 * 976 * For ECC with 1024 bytes step size: 977 * 0x00: AA AA BB BB BB BB BB BB BB BB BB BB BB BB BB BB 978 * 0x10: AA AA CC CC CC CC CC CC CC CC CC CC CC CC CC CC 979 * 0x20: AA AA DD DD DD DD DD DD DD DD DD DD DD DD DD DD 980 * 0x30: AA AA EE EE EE EE EE EE EE EE EE EE EE EE EE EE 981 * 982 * AA - user bytes. 983 * BB, CC, DD, EE - ECC code bytes for each step. 984 */ 985static struct nand_ecclayout nand_oob; 986 987static void meson_nfc_init_nand_oob(struct nand_chip *nand) 988{ 989 int section_size = 2 + nand->ecc.bytes; 990 int i; 991 int k; 992 993 nand_oob.eccbytes = nand->ecc.steps * nand->ecc.bytes; 994 k = 0; 995 996 for (i = 0; i < nand->ecc.steps; i++) { 997 int j; 998 999 for (j = 0; j < nand->ecc.bytes; j++) 1000 nand_oob.eccpos[k++] = (i * section_size) + 2 + j; 1001 1002 nand_oob.oobfree[i].offset = (i * section_size); 1003 nand_oob.oobfree[i].length = 2; 1004 } 1005 1006 nand_oob.oobavail = 2 * nand->ecc.steps; 1007 nand->ecc.layout = &nand_oob; 1008} 1009 1010static int meson_nfc_init_ecc(struct nand_chip *nand, ofnode node) 1011{ 1012 const struct mtd_info *mtd = nand_to_mtd(nand); 1013 int ret; 1014 int i; 1015 1016 ret = nand_check_ecc_caps(nand, &meson_axg_ecc_caps, mtd->oobsize - 2); 1017 if (ret) 1018 return ret; 1019 1020 for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) { 1021 if (meson_ecc[i].strength == nand->ecc.strength && 1022 meson_ecc[i].size == nand->ecc.size) { 1023 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); 1024 1025 nand->ecc.steps = mtd->writesize / nand->ecc.size; 1026 meson_chip->bch_mode = meson_ecc[i].bch; 1027 1028 meson_nfc_init_nand_oob(nand); 1029 1030 return 0; 1031 } 1032 } 1033 1034 return -EINVAL; 1035} 1036 1037static int meson_nfc_nand_chip_init(struct udevice *dev, struct meson_nfc *nfc, 1038 ofnode node) 1039{ 1040 struct meson_nfc_nand_chip *meson_chip; 1041 struct nand_chip *nand; 1042 struct mtd_info *mtd; 1043 u32 cs[MAX_CE_NUM]; 1044 u32 nsels; 1045 int ret; 1046 int i; 1047 1048 if (!ofnode_get_property(node, "reg", &nsels)) { 1049 dev_err(dev, "\"reg\" property is not found\n"); 1050 return -ENODEV; 1051 } 1052 1053 nsels /= sizeof(u32); 1054 if (nsels >= MAX_CE_NUM) { 1055 dev_err(dev, "invalid size of CS array, max is %d\n", 1056 MAX_CE_NUM); 1057 return -EINVAL; 1058 } 1059 1060 ret = ofnode_read_u32_array(node, "reg", cs, nsels); 1061 if (ret < 0) { 1062 dev_err(dev, "failed to read \"reg\" property\n"); 1063 return ret; 1064 } 1065 1066 for (i = 0; i < nsels; i++) { 1067 if (test_and_set_bit(cs[i], &nfc->assigned_cs)) { 1068 dev_err(dev, "CS %d already assigned\n", cs[i]); 1069 return -EINVAL; 1070 } 1071 } 1072 1073 meson_chip = malloc(sizeof(*meson_chip) + nsels * sizeof(meson_chip->sels[0])); 1074 if (!meson_chip) { 1075 dev_err(dev, "failed to allocate memory for chip\n"); 1076 return -ENOMEM; 1077 } 1078 1079 meson_chip->nsels = nsels; 1080 nand = &meson_chip->nand; 1081 1082 nand->flash_node = node; 1083 nand_set_controller_data(nand, nfc); 1084 /* Set the driver entry points for MTD */ 1085 nand->cmdfunc = meson_nfc_nand_cmd_function; 1086 nand->cmd_ctrl = meson_nfc_cmd_ctrl; 1087 nand->select_chip = meson_nfc_nand_select_chip; 1088 nand->read_byte = meson_nfc_nand_read_byte; 1089 nand->write_byte = meson_nfc_nand_write_byte; 1090 nand->dev_ready = meson_nfc_dev_ready; 1091 1092 /* Buffer read/write routines */ 1093 nand->read_buf = meson_nfc_read_buf; 1094 nand->write_buf = meson_nfc_write_buf; 1095 nand->options |= NAND_NO_SUBPAGE_WRITE; 1096 1097 nand->ecc.mode = NAND_ECC_HW; 1098 nand->ecc.hwctl = NULL; 1099 nand->ecc.read_page = meson_nfc_read_page_hwecc; 1100 nand->ecc.write_page = meson_nfc_write_page_hwecc; 1101 nand->ecc.read_page_raw = meson_nfc_read_page_raw; 1102 nand->ecc.write_page_raw = meson_nfc_write_page_raw; 1103 1104 nand->ecc.read_oob = meson_nfc_read_oob; 1105 nand->ecc.write_oob = meson_nfc_write_oob; 1106 nand->ecc.read_oob_raw = meson_nfc_read_oob_raw; 1107 nand->ecc.write_oob_raw = meson_nfc_write_oob_raw; 1108 1109 nand->ecc.algo = NAND_ECC_BCH; 1110 1111 mtd = nand_to_mtd(nand); 1112 1113 ret = nand_scan_ident(mtd, 1, NULL); 1114 if (ret) { 1115 dev_err(dev, "'nand_scan_ident()' failed: %d\n", ret); 1116 goto err_chip_free; 1117 } 1118 1119 ret = meson_nfc_init_ecc(nand, node); 1120 if (ret) { 1121 dev_err(dev, "failed to init ECC settings: %d\n", ret); 1122 goto err_chip_free; 1123 } 1124 1125 ret = meson_chip_buffer_init(nand); 1126 if (ret) { 1127 dev_err(dev, "failed to init DMA buffers: %d\n", ret); 1128 goto err_chip_free; 1129 } 1130 1131 /* 'nand_scan_tail()' needs ECC parameters to be already 1132 * set and correct. 1133 */ 1134 ret = nand_scan_tail(mtd); 1135 if (ret) { 1136 dev_err(dev, "'nand_scan_tail()' failed: %d\n", ret); 1137 goto err_chip_buf_free; 1138 } 1139 1140 ret = nand_register(0, mtd); 1141 if (ret) { 1142 dev_err(dev, "'nand_register()' failed: %d\n", ret); 1143 goto err_chip_buf_free; 1144 } 1145 1146 list_add_tail(&meson_chip->node, &nfc->chips); 1147 1148 return 0; 1149 1150err_chip_buf_free: 1151 dma_free_coherent(meson_chip->info_buf); 1152 dma_free_coherent(meson_chip->data_buf); 1153 1154err_chip_free: 1155 free(meson_chip); 1156 1157 return ret; 1158} 1159 1160static int meson_nfc_nand_chips_init(struct udevice *dev, 1161 struct meson_nfc *nfc) 1162{ 1163 ofnode parent = dev_ofnode(dev); 1164 ofnode node; 1165 1166 ofnode_for_each_subnode(node, parent) { 1167 int ret = meson_nfc_nand_chip_init(dev, nfc, node); 1168 1169 if (ret) 1170 return ret; 1171 } 1172 1173 return 0; 1174} 1175 1176static void meson_nfc_clk_init(struct meson_nfc *nfc) 1177{ 1178 u32 bus_cycle = NFC_DEFAULT_BUS_CYCLE; 1179 u32 bus_timing = NFC_DEFAULT_BUS_TIMING; 1180 u32 bus_cfg_val; 1181 1182 writel(CLK_ALWAYS_ON_NAND | CLK_SELECT_NAND | CLK_ENABLE_VALUE, nfc->reg_clk); 1183 writel(0, nfc->reg_base + NFC_REG_CFG); 1184 1185 bus_cfg_val = (((bus_cycle - 1) & 31) | ((bus_timing & 31) << 5)); 1186 writel(bus_cfg_val, nfc->reg_base + NFC_REG_CFG); 1187 writel(BIT(31), nfc->reg_base + NFC_REG_CMD); 1188} 1189 1190static int meson_probe(struct udevice *dev) 1191{ 1192 struct meson_nfc *nfc = dev_get_priv(dev); 1193 void *addr; 1194 int ret; 1195 1196 addr = dev_read_addr_ptr(dev); 1197 if (!addr) { 1198 dev_err(dev, "base register address not found\n"); 1199 return -EINVAL; 1200 } 1201 1202 nfc->reg_base = addr; 1203 1204 addr = dev_read_addr_index_ptr(dev, 1); 1205 if (!addr) { 1206 dev_err(dev, "clk register address not found\n"); 1207 return -EINVAL; 1208 } 1209 1210 nfc->reg_clk = addr; 1211 nfc->dev = dev; 1212 1213 meson_nfc_clk_init(nfc); 1214 1215 ret = meson_nfc_nand_chips_init(dev, nfc); 1216 if (ret) { 1217 dev_err(nfc->dev, "failed to init chips\n"); 1218 return ret; 1219 } 1220 1221 return 0; 1222} 1223 1224static const struct udevice_id meson_nand_dt_ids[] = { 1225 {.compatible = "amlogic,meson-axg-nfc",}, 1226 { /* sentinel */ } 1227}; 1228 1229U_BOOT_DRIVER(meson_nand) = { 1230 .name = "meson_nand", 1231 .id = UCLASS_MTD, 1232 .of_match = meson_nand_dt_ids, 1233 .probe = meson_probe, 1234 .priv_auto = sizeof(struct meson_nfc), 1235}; 1236 1237void board_nand_init(void) 1238{ 1239 struct udevice *dev; 1240 int ret; 1241 1242 ret = uclass_get_device_by_driver(UCLASS_MTD, 1243 DM_DRIVER_GET(meson_nand), &dev); 1244 1245 if (ret && ret != -ENODEV) 1246 pr_err("Failed to initialize: %d\n", ret); 1247} 1248