1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2017 ATMEL 4 * Copyright 2017 Free Electrons 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 * 8 * Derived from the atmel_nand.c driver which contained the following 9 * copyrights: 10 * 11 * Copyright 2003 Rick Bronson 12 * 13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8) 14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) 15 * 16 * Derived from drivers/mtd/spia.c (removed in v3.8) 17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com) 18 * 19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 21 * 22 * Derived from Das U-Boot source code 23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) 24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas 25 * 26 * Add Programmable Multibit ECC support for various AT91 SoC 27 * Copyright 2012 ATMEL, Hong Xu 28 * 29 * Add Nand Flash Controller support for SAMA5 SoC 30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) 31 * 32 * The PMECC is an hardware assisted BCH engine, which means part of the 33 * ECC algorithm is left to the software. The hardware/software repartition 34 * is explained in the "PMECC Controller Functional Description" chapter in 35 * Atmel datasheets, and some of the functions in this file are directly 36 * implementing the algorithms described in the "Software Implementation" 37 * sub-section. 38 * 39 * TODO: it seems that the software BCH implementation in lib/bch.c is already 40 * providing some of the logic we are implementing here. It would be smart 41 * to expose the needed lib/bch.c helpers/functions and re-use them here. 42 */ 43 44#include <linux/genalloc.h> 45#include <linux/iopoll.h> 46#include <linux/module.h> 47#include <linux/mtd/rawnand.h> 48#include <linux/of_irq.h> 49#include <linux/of_platform.h> 50#include <linux/platform_device.h> 51#include <linux/slab.h> 52 53#include "pmecc.h" 54 55/* Galois field dimension */ 56#define PMECC_GF_DIMENSION_13 13 57#define PMECC_GF_DIMENSION_14 14 58 59/* Primitive Polynomial used by PMECC */ 60#define PMECC_GF_13_PRIMITIVE_POLY 0x201b 61#define PMECC_GF_14_PRIMITIVE_POLY 0x4443 62 63#define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 64#define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 65 66/* Time out value for reading PMECC status register */ 67#define PMECC_MAX_TIMEOUT_MS 100 68 69/* PMECC Register Definitions */ 70#define ATMEL_PMECC_CFG 0x0 71#define PMECC_CFG_BCH_STRENGTH(x) (x) 72#define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) 73#define PMECC_CFG_SECTOR512 (0 << 4) 74#define PMECC_CFG_SECTOR1024 (1 << 4) 75#define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) 76#define PMECC_CFG_READ_OP (0 << 12) 77#define PMECC_CFG_WRITE_OP (1 << 12) 78#define PMECC_CFG_SPARE_ENABLE BIT(16) 79#define PMECC_CFG_AUTO_ENABLE BIT(20) 80 81#define ATMEL_PMECC_SAREA 0x4 82#define ATMEL_PMECC_SADDR 0x8 83#define ATMEL_PMECC_EADDR 0xc 84 85#define ATMEL_PMECC_CLK 0x10 86#define PMECC_CLK_133MHZ (2 << 0) 87 88#define ATMEL_PMECC_CTRL 0x14 89#define PMECC_CTRL_RST BIT(0) 90#define PMECC_CTRL_DATA BIT(1) 91#define PMECC_CTRL_USER BIT(2) 92#define PMECC_CTRL_ENABLE BIT(4) 93#define PMECC_CTRL_DISABLE BIT(5) 94 95#define ATMEL_PMECC_SR 0x18 96#define PMECC_SR_BUSY BIT(0) 97#define PMECC_SR_ENABLE BIT(4) 98 99#define ATMEL_PMECC_IER 0x1c 100#define ATMEL_PMECC_IDR 0x20 101#define ATMEL_PMECC_IMR 0x24 102#define ATMEL_PMECC_ISR 0x28 103#define PMECC_ERROR_INT BIT(0) 104 105#define ATMEL_PMECC_ECC(sector, n) \ 106 ((((sector) + 1) * 0x40) + (n)) 107 108#define ATMEL_PMECC_REM(sector, n) \ 109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) 110 111/* PMERRLOC Register Definitions */ 112#define ATMEL_PMERRLOC_ELCFG 0x0 113#define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) 114#define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) 115#define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) 116 117#define ATMEL_PMERRLOC_ELPRIM 0x4 118#define ATMEL_PMERRLOC_ELEN 0x8 119#define ATMEL_PMERRLOC_ELDIS 0xc 120#define PMERRLOC_DISABLE BIT(0) 121 122#define ATMEL_PMERRLOC_ELSR 0x10 123#define PMERRLOC_ELSR_BUSY BIT(0) 124 125#define ATMEL_PMERRLOC_ELIER 0x14 126#define ATMEL_PMERRLOC_ELIDR 0x18 127#define ATMEL_PMERRLOC_ELIMR 0x1c 128#define ATMEL_PMERRLOC_ELISR 0x20 129#define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) 130#define PMERRLOC_CALC_DONE BIT(0) 131 132#define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) 133 134#define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) 135 136struct atmel_pmecc_gf_tables { 137 u16 *alpha_to; 138 u16 *index_of; 139}; 140 141struct atmel_pmecc_caps { 142 const int *strengths; 143 int nstrengths; 144 int el_offset; 145 bool correct_erased_chunks; 146}; 147 148struct atmel_pmecc { 149 struct device *dev; 150 const struct atmel_pmecc_caps *caps; 151 152 struct { 153 void __iomem *base; 154 void __iomem *errloc; 155 } regs; 156 157 struct mutex lock; 158}; 159 160struct atmel_pmecc_user_conf_cache { 161 u32 cfg; 162 u32 sarea; 163 u32 saddr; 164 u32 eaddr; 165}; 166 167struct atmel_pmecc_user { 168 struct atmel_pmecc_user_conf_cache cache; 169 struct atmel_pmecc *pmecc; 170 const struct atmel_pmecc_gf_tables *gf_tables; 171 int eccbytes; 172 s16 *partial_syn; 173 s16 *si; 174 s16 *lmu; 175 s16 *smu; 176 s32 *mu; 177 s32 *dmu; 178 s32 *delta; 179 u32 isr; 180}; 181 182static DEFINE_MUTEX(pmecc_gf_tables_lock); 183static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; 184static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; 185 186static inline int deg(unsigned int poly) 187{ 188 /* polynomial degree is the most-significant bit index */ 189 return fls(poly) - 1; 190} 191 192static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, 193 struct atmel_pmecc_gf_tables *gf_tables) 194{ 195 unsigned int i, x = 1; 196 const unsigned int k = BIT(deg(poly)); 197 unsigned int nn = BIT(mm) - 1; 198 199 /* primitive polynomial must be of degree m */ 200 if (k != (1u << mm)) 201 return -EINVAL; 202 203 for (i = 0; i < nn; i++) { 204 gf_tables->alpha_to[i] = x; 205 gf_tables->index_of[x] = i; 206 if (i && (x == 1)) 207 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ 208 return -EINVAL; 209 x <<= 1; 210 if (x & k) 211 x ^= poly; 212 } 213 gf_tables->alpha_to[nn] = 1; 214 gf_tables->index_of[0] = 0; 215 216 return 0; 217} 218 219static const struct atmel_pmecc_gf_tables * 220atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) 221{ 222 struct atmel_pmecc_gf_tables *gf_tables; 223 unsigned int poly, degree, table_size; 224 int ret; 225 226 if (req->ecc.sectorsize == 512) { 227 degree = PMECC_GF_DIMENSION_13; 228 poly = PMECC_GF_13_PRIMITIVE_POLY; 229 table_size = PMECC_LOOKUP_TABLE_SIZE_512; 230 } else { 231 degree = PMECC_GF_DIMENSION_14; 232 poly = PMECC_GF_14_PRIMITIVE_POLY; 233 table_size = PMECC_LOOKUP_TABLE_SIZE_1024; 234 } 235 236 gf_tables = kzalloc(sizeof(*gf_tables) + 237 (2 * table_size * sizeof(u16)), 238 GFP_KERNEL); 239 if (!gf_tables) 240 return ERR_PTR(-ENOMEM); 241 242 gf_tables->alpha_to = (void *)(gf_tables + 1); 243 gf_tables->index_of = gf_tables->alpha_to + table_size; 244 245 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); 246 if (ret) { 247 kfree(gf_tables); 248 return ERR_PTR(ret); 249 } 250 251 return gf_tables; 252} 253 254static const struct atmel_pmecc_gf_tables * 255atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) 256{ 257 const struct atmel_pmecc_gf_tables **gf_tables, *ret; 258 259 mutex_lock(&pmecc_gf_tables_lock); 260 if (req->ecc.sectorsize == 512) 261 gf_tables = &pmecc_gf_tables_512; 262 else 263 gf_tables = &pmecc_gf_tables_1024; 264 265 ret = *gf_tables; 266 267 if (!ret) { 268 ret = atmel_pmecc_create_gf_tables(req); 269 if (!IS_ERR(ret)) 270 *gf_tables = ret; 271 } 272 mutex_unlock(&pmecc_gf_tables_lock); 273 274 return ret; 275} 276 277static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, 278 struct atmel_pmecc_user_req *req) 279{ 280 int i, max_eccbytes, eccbytes = 0, eccstrength = 0; 281 282 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) 283 return -EINVAL; 284 285 if (req->ecc.ooboffset >= 0 && 286 req->ecc.ooboffset + req->ecc.bytes > req->oobsize) 287 return -EINVAL; 288 289 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { 290 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 291 return -EINVAL; 292 293 if (req->pagesize > 512) 294 req->ecc.sectorsize = 1024; 295 else 296 req->ecc.sectorsize = 512; 297 } 298 299 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) 300 return -EINVAL; 301 302 if (req->pagesize % req->ecc.sectorsize) 303 return -EINVAL; 304 305 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; 306 307 max_eccbytes = req->ecc.bytes; 308 309 for (i = 0; i < pmecc->caps->nstrengths; i++) { 310 int nbytes, strength = pmecc->caps->strengths[i]; 311 312 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && 313 strength < req->ecc.strength) 314 continue; 315 316 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), 317 8); 318 nbytes *= req->ecc.nsectors; 319 320 if (nbytes > max_eccbytes) 321 break; 322 323 eccstrength = strength; 324 eccbytes = nbytes; 325 326 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 327 break; 328 } 329 330 if (!eccstrength) 331 return -EINVAL; 332 333 req->ecc.bytes = eccbytes; 334 req->ecc.strength = eccstrength; 335 336 if (req->ecc.ooboffset < 0) 337 req->ecc.ooboffset = req->oobsize - eccbytes; 338 339 return 0; 340} 341 342struct atmel_pmecc_user * 343atmel_pmecc_create_user(struct atmel_pmecc *pmecc, 344 struct atmel_pmecc_user_req *req) 345{ 346 struct atmel_pmecc_user *user; 347 const struct atmel_pmecc_gf_tables *gf_tables; 348 int strength, size, ret; 349 350 ret = atmel_pmecc_prepare_user_req(pmecc, req); 351 if (ret) 352 return ERR_PTR(ret); 353 354 size = sizeof(*user); 355 size = ALIGN(size, sizeof(u16)); 356 /* Reserve space for partial_syn, si and smu */ 357 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * 358 (2 + req->ecc.strength + 2); 359 /* Reserve space for lmu. */ 360 size += (req->ecc.strength + 1) * sizeof(u16); 361 /* Reserve space for mu, dmu and delta. */ 362 size = ALIGN(size, sizeof(s32)); 363 size += (req->ecc.strength + 1) * sizeof(s32) * 3; 364 365 user = kzalloc(size, GFP_KERNEL); 366 if (!user) 367 return ERR_PTR(-ENOMEM); 368 369 user->pmecc = pmecc; 370 371 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); 372 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); 373 user->lmu = user->si + ((2 * req->ecc.strength) + 1); 374 user->smu = user->lmu + (req->ecc.strength + 1); 375 user->mu = (s32 *)PTR_ALIGN(user->smu + 376 (((2 * req->ecc.strength) + 1) * 377 (req->ecc.strength + 2)), 378 sizeof(s32)); 379 user->dmu = user->mu + req->ecc.strength + 1; 380 user->delta = user->dmu + req->ecc.strength + 1; 381 382 gf_tables = atmel_pmecc_get_gf_tables(req); 383 if (IS_ERR(gf_tables)) { 384 kfree(user); 385 return ERR_CAST(gf_tables); 386 } 387 388 user->gf_tables = gf_tables; 389 390 user->eccbytes = req->ecc.bytes / req->ecc.nsectors; 391 392 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { 393 if (pmecc->caps->strengths[strength] == req->ecc.strength) 394 break; 395 } 396 397 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | 398 PMECC_CFG_NSECTORS(req->ecc.nsectors); 399 400 if (req->ecc.sectorsize == 1024) 401 user->cache.cfg |= PMECC_CFG_SECTOR1024; 402 403 user->cache.sarea = req->oobsize - 1; 404 user->cache.saddr = req->ecc.ooboffset; 405 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; 406 407 return user; 408} 409EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); 410 411void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user) 412{ 413 kfree(user); 414} 415EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user); 416 417static int get_strength(struct atmel_pmecc_user *user) 418{ 419 const int *strengths = user->pmecc->caps->strengths; 420 421 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; 422} 423 424static int get_sectorsize(struct atmel_pmecc_user *user) 425{ 426 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512; 427} 428 429static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) 430{ 431 int strength = get_strength(user); 432 u32 value; 433 int i; 434 435 /* Fill odd syndromes */ 436 for (i = 0; i < strength; i++) { 437 value = readl_relaxed(user->pmecc->regs.base + 438 ATMEL_PMECC_REM(sector, i / 2)); 439 if (i & 1) 440 value >>= 16; 441 442 user->partial_syn[(2 * i) + 1] = value; 443 } 444} 445 446static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) 447{ 448 int degree = get_sectorsize(user) == 512 ? 13 : 14; 449 int cw_len = BIT(degree) - 1; 450 int strength = get_strength(user); 451 s16 *alpha_to = user->gf_tables->alpha_to; 452 s16 *index_of = user->gf_tables->index_of; 453 s16 *partial_syn = user->partial_syn; 454 s16 *si; 455 int i, j; 456 457 /* 458 * si[] is a table that holds the current syndrome value, 459 * an element of that table belongs to the field 460 */ 461 si = user->si; 462 463 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); 464 465 /* Computation 2t syndromes based on S(x) */ 466 /* Odd syndromes */ 467 for (i = 1; i < 2 * strength; i += 2) { 468 for (j = 0; j < degree; j++) { 469 if (partial_syn[i] & BIT(j)) 470 si[i] = alpha_to[i * j] ^ si[i]; 471 } 472 } 473 /* Even syndrome = (Odd syndrome) ** 2 */ 474 for (i = 2, j = 1; j <= strength; i = ++j << 1) { 475 if (si[j] == 0) { 476 si[i] = 0; 477 } else { 478 s16 tmp; 479 480 tmp = index_of[si[j]]; 481 tmp = (tmp * 2) % cw_len; 482 si[i] = alpha_to[tmp]; 483 } 484 } 485} 486 487static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) 488{ 489 s16 *lmu = user->lmu; 490 s16 *si = user->si; 491 s32 *mu = user->mu; 492 s32 *dmu = user->dmu; 493 s32 *delta = user->delta; 494 int degree = get_sectorsize(user) == 512 ? 13 : 14; 495 int cw_len = BIT(degree) - 1; 496 int strength = get_strength(user); 497 int num = 2 * strength + 1; 498 s16 *index_of = user->gf_tables->index_of; 499 s16 *alpha_to = user->gf_tables->alpha_to; 500 int i, j, k; 501 u32 dmu_0_count, tmp; 502 s16 *smu = user->smu; 503 504 /* index of largest delta */ 505 int ro; 506 int largest; 507 int diff; 508 509 dmu_0_count = 0; 510 511 /* First Row */ 512 513 /* Mu */ 514 mu[0] = -1; 515 516 memset(smu, 0, sizeof(s16) * num); 517 smu[0] = 1; 518 519 /* discrepancy set to 1 */ 520 dmu[0] = 1; 521 /* polynom order set to 0 */ 522 lmu[0] = 0; 523 delta[0] = (mu[0] * 2 - lmu[0]) >> 1; 524 525 /* Second Row */ 526 527 /* Mu */ 528 mu[1] = 0; 529 /* Sigma(x) set to 1 */ 530 memset(&smu[num], 0, sizeof(s16) * num); 531 smu[num] = 1; 532 533 /* discrepancy set to S1 */ 534 dmu[1] = si[1]; 535 536 /* polynom order set to 0 */ 537 lmu[1] = 0; 538 539 delta[1] = (mu[1] * 2 - lmu[1]) >> 1; 540 541 /* Init the Sigma(x) last row */ 542 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); 543 544 for (i = 1; i <= strength; i++) { 545 mu[i + 1] = i << 1; 546 /* Begin Computing Sigma (Mu+1) and L(mu) */ 547 /* check if discrepancy is set to 0 */ 548 if (dmu[i] == 0) { 549 dmu_0_count++; 550 551 tmp = ((strength - (lmu[i] >> 1) - 1) / 2); 552 if ((strength - (lmu[i] >> 1) - 1) & 0x1) 553 tmp += 2; 554 else 555 tmp += 1; 556 557 if (dmu_0_count == tmp) { 558 for (j = 0; j <= (lmu[i] >> 1) + 1; j++) 559 smu[(strength + 1) * num + j] = 560 smu[i * num + j]; 561 562 lmu[strength + 1] = lmu[i]; 563 return; 564 } 565 566 /* copy polynom */ 567 for (j = 0; j <= lmu[i] >> 1; j++) 568 smu[(i + 1) * num + j] = smu[i * num + j]; 569 570 /* copy previous polynom order to the next */ 571 lmu[i + 1] = lmu[i]; 572 } else { 573 ro = 0; 574 largest = -1; 575 /* find largest delta with dmu != 0 */ 576 for (j = 0; j < i; j++) { 577 if ((dmu[j]) && (delta[j] > largest)) { 578 largest = delta[j]; 579 ro = j; 580 } 581 } 582 583 /* compute difference */ 584 diff = (mu[i] - mu[ro]); 585 586 /* Compute degree of the new smu polynomial */ 587 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) 588 lmu[i + 1] = lmu[i]; 589 else 590 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; 591 592 /* Init smu[i+1] with 0 */ 593 for (k = 0; k < num; k++) 594 smu[(i + 1) * num + k] = 0; 595 596 /* Compute smu[i+1] */ 597 for (k = 0; k <= lmu[ro] >> 1; k++) { 598 s16 a, b, c; 599 600 if (!(smu[ro * num + k] && dmu[i])) 601 continue; 602 603 a = index_of[dmu[i]]; 604 b = index_of[dmu[ro]]; 605 c = index_of[smu[ro * num + k]]; 606 tmp = a + (cw_len - b) + c; 607 a = alpha_to[tmp % cw_len]; 608 smu[(i + 1) * num + (k + diff)] = a; 609 } 610 611 for (k = 0; k <= lmu[i] >> 1; k++) 612 smu[(i + 1) * num + k] ^= smu[i * num + k]; 613 } 614 615 /* End Computing Sigma (Mu+1) and L(mu) */ 616 /* In either case compute delta */ 617 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; 618 619 /* Do not compute discrepancy for the last iteration */ 620 if (i >= strength) 621 continue; 622 623 for (k = 0; k <= (lmu[i + 1] >> 1); k++) { 624 tmp = 2 * (i - 1); 625 if (k == 0) { 626 dmu[i + 1] = si[tmp + 3]; 627 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { 628 s16 a, b, c; 629 630 a = index_of[smu[(i + 1) * num + k]]; 631 b = si[2 * (i - 1) + 3 - k]; 632 c = index_of[b]; 633 tmp = a + c; 634 tmp %= cw_len; 635 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; 636 } 637 } 638 } 639} 640 641static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) 642{ 643 int sector_size = get_sectorsize(user); 644 int degree = sector_size == 512 ? 13 : 14; 645 struct atmel_pmecc *pmecc = user->pmecc; 646 int strength = get_strength(user); 647 int ret, roots_nbr, i, err_nbr = 0; 648 int num = (2 * strength) + 1; 649 s16 *smu = user->smu; 650 u32 val; 651 652 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); 653 654 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { 655 writel_relaxed(smu[(strength + 1) * num + i], 656 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); 657 err_nbr++; 658 } 659 660 val = (err_nbr - 1) << 16; 661 if (sector_size == 1024) 662 val |= 1; 663 664 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); 665 writel((sector_size * 8) + (degree * strength), 666 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); 667 668 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + 669 ATMEL_PMERRLOC_ELISR, 670 val, val & PMERRLOC_CALC_DONE, 0, 671 PMECC_MAX_TIMEOUT_MS * 1000); 672 if (ret) { 673 dev_err(pmecc->dev, 674 "PMECC: Timeout to calculate error location.\n"); 675 return ret; 676 } 677 678 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; 679 /* Number of roots == degree of smu hence <= cap */ 680 if (roots_nbr == user->lmu[strength + 1] >> 1) 681 return err_nbr - 1; 682 683 /* 684 * Number of roots does not match the degree of smu 685 * unable to correct error. 686 */ 687 return -EBADMSG; 688} 689 690int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, 691 void *data, void *ecc) 692{ 693 struct atmel_pmecc *pmecc = user->pmecc; 694 int sectorsize = get_sectorsize(user); 695 int eccbytes = user->eccbytes; 696 int i, nerrors; 697 698 if (!(user->isr & BIT(sector))) 699 return 0; 700 701 atmel_pmecc_gen_syndrome(user, sector); 702 atmel_pmecc_substitute(user); 703 atmel_pmecc_get_sigma(user); 704 705 nerrors = atmel_pmecc_err_location(user); 706 if (nerrors < 0) 707 return nerrors; 708 709 for (i = 0; i < nerrors; i++) { 710 const char *area; 711 int byte, bit; 712 u32 errpos; 713 u8 *ptr; 714 715 errpos = readl_relaxed(pmecc->regs.errloc + 716 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); 717 errpos--; 718 719 byte = errpos / 8; 720 bit = errpos % 8; 721 722 if (byte < sectorsize) { 723 ptr = data + byte; 724 area = "data"; 725 } else if (byte < sectorsize + eccbytes) { 726 ptr = ecc + byte - sectorsize; 727 area = "ECC"; 728 } else { 729 dev_dbg(pmecc->dev, 730 "Invalid errpos value (%d, max is %d)\n", 731 errpos, (sectorsize + eccbytes) * 8); 732 return -EINVAL; 733 } 734 735 dev_dbg(pmecc->dev, 736 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", 737 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); 738 739 *ptr ^= BIT(bit); 740 } 741 742 return nerrors; 743} 744EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); 745 746bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) 747{ 748 return user->pmecc->caps->correct_erased_chunks; 749} 750EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); 751 752void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, 753 int sector, void *ecc) 754{ 755 struct atmel_pmecc *pmecc = user->pmecc; 756 u8 *ptr = ecc; 757 int i; 758 759 for (i = 0; i < user->eccbytes; i++) 760 ptr[i] = readb_relaxed(pmecc->regs.base + 761 ATMEL_PMECC_ECC(sector, i)); 762} 763EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); 764 765void atmel_pmecc_reset(struct atmel_pmecc *pmecc) 766{ 767 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); 768 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 769} 770EXPORT_SYMBOL_GPL(atmel_pmecc_reset); 771 772int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) 773{ 774 struct atmel_pmecc *pmecc = user->pmecc; 775 u32 cfg; 776 777 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { 778 dev_err(pmecc->dev, "Bad ECC operation!"); 779 return -EINVAL; 780 } 781 782 mutex_lock(&user->pmecc->lock); 783 784 cfg = user->cache.cfg; 785 if (op == NAND_ECC_WRITE) 786 cfg |= PMECC_CFG_WRITE_OP; 787 else 788 cfg |= PMECC_CFG_AUTO_ENABLE; 789 790 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); 791 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); 792 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); 793 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); 794 795 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 796 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); 797 798 return 0; 799} 800EXPORT_SYMBOL_GPL(atmel_pmecc_enable); 801 802void atmel_pmecc_disable(struct atmel_pmecc_user *user) 803{ 804 atmel_pmecc_reset(user->pmecc); 805 mutex_unlock(&user->pmecc->lock); 806} 807EXPORT_SYMBOL_GPL(atmel_pmecc_disable); 808 809int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) 810{ 811 struct atmel_pmecc *pmecc = user->pmecc; 812 u32 status; 813 int ret; 814 815 ret = readl_relaxed_poll_timeout(pmecc->regs.base + 816 ATMEL_PMECC_SR, 817 status, !(status & PMECC_SR_BUSY), 0, 818 PMECC_MAX_TIMEOUT_MS * 1000); 819 if (ret) { 820 dev_err(pmecc->dev, 821 "Timeout while waiting for PMECC ready.\n"); 822 return ret; 823 } 824 825 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); 826 827 return 0; 828} 829EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); 830 831static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, 832 const struct atmel_pmecc_caps *caps, 833 int pmecc_res_idx, int errloc_res_idx) 834{ 835 struct device *dev = &pdev->dev; 836 struct atmel_pmecc *pmecc; 837 838 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); 839 if (!pmecc) 840 return ERR_PTR(-ENOMEM); 841 842 pmecc->caps = caps; 843 pmecc->dev = dev; 844 mutex_init(&pmecc->lock); 845 846 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx); 847 if (IS_ERR(pmecc->regs.base)) 848 return ERR_CAST(pmecc->regs.base); 849 850 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx); 851 if (IS_ERR(pmecc->regs.errloc)) 852 return ERR_CAST(pmecc->regs.errloc); 853 854 /* Disable all interrupts before registering the PMECC handler. */ 855 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); 856 atmel_pmecc_reset(pmecc); 857 858 return pmecc; 859} 860 861static void devm_atmel_pmecc_put(struct device *dev, void *res) 862{ 863 struct atmel_pmecc **pmecc = res; 864 865 put_device((*pmecc)->dev); 866} 867 868static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, 869 struct device_node *np) 870{ 871 struct platform_device *pdev; 872 struct atmel_pmecc *pmecc, **ptr; 873 int ret; 874 875 pdev = of_find_device_by_node(np); 876 if (!pdev) 877 return ERR_PTR(-EPROBE_DEFER); 878 pmecc = platform_get_drvdata(pdev); 879 if (!pmecc) { 880 ret = -EPROBE_DEFER; 881 goto err_put_device; 882 } 883 884 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); 885 if (!ptr) { 886 ret = -ENOMEM; 887 goto err_put_device; 888 } 889 890 *ptr = pmecc; 891 892 devres_add(userdev, ptr); 893 894 return pmecc; 895 896err_put_device: 897 put_device(&pdev->dev); 898 return ERR_PTR(ret); 899} 900 901static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; 902 903static struct atmel_pmecc_caps at91sam9g45_caps = { 904 .strengths = atmel_pmecc_strengths, 905 .nstrengths = 5, 906 .el_offset = 0x8c, 907}; 908 909static struct atmel_pmecc_caps sama5d4_caps = { 910 .strengths = atmel_pmecc_strengths, 911 .nstrengths = 5, 912 .el_offset = 0x8c, 913 .correct_erased_chunks = true, 914}; 915 916static struct atmel_pmecc_caps sama5d2_caps = { 917 .strengths = atmel_pmecc_strengths, 918 .nstrengths = 6, 919 .el_offset = 0xac, 920 .correct_erased_chunks = true, 921}; 922 923static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = { 924 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, 925 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, 926 { /* sentinel */ } 927}; 928 929struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) 930{ 931 struct atmel_pmecc *pmecc; 932 struct device_node *np; 933 934 if (!userdev) 935 return ERR_PTR(-EINVAL); 936 937 if (!userdev->of_node) 938 return NULL; 939 940 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); 941 if (np) { 942 pmecc = atmel_pmecc_get_by_node(userdev, np); 943 of_node_put(np); 944 } else { 945 /* 946 * Support old DT bindings: in this case the PMECC iomem 947 * resources are directly defined in the user pdev at position 948 * 1 and 2. Extract all relevant information from there. 949 */ 950 struct platform_device *pdev = to_platform_device(userdev); 951 const struct atmel_pmecc_caps *caps; 952 const struct of_device_id *match; 953 954 /* No PMECC engine available. */ 955 if (!of_property_read_bool(userdev->of_node, 956 "atmel,has-pmecc")) 957 return NULL; 958 959 caps = &at91sam9g45_caps; 960 961 /* Find the caps associated to the NAND dev node. */ 962 match = of_match_node(atmel_pmecc_legacy_match, 963 userdev->of_node); 964 if (match && match->data) 965 caps = match->data; 966 967 pmecc = atmel_pmecc_create(pdev, caps, 1, 2); 968 } 969 970 return pmecc; 971} 972EXPORT_SYMBOL(devm_atmel_pmecc_get); 973 974static const struct of_device_id atmel_pmecc_match[] = { 975 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, 976 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, 977 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, 978 { /* sentinel */ } 979}; 980MODULE_DEVICE_TABLE(of, atmel_pmecc_match); 981 982static int atmel_pmecc_probe(struct platform_device *pdev) 983{ 984 struct device *dev = &pdev->dev; 985 const struct atmel_pmecc_caps *caps; 986 struct atmel_pmecc *pmecc; 987 988 caps = of_device_get_match_data(&pdev->dev); 989 if (!caps) { 990 dev_err(dev, "Invalid caps\n"); 991 return -EINVAL; 992 } 993 994 pmecc = atmel_pmecc_create(pdev, caps, 0, 1); 995 if (IS_ERR(pmecc)) 996 return PTR_ERR(pmecc); 997 998 platform_set_drvdata(pdev, pmecc); 999 1000 return 0; 1001} 1002 1003static struct platform_driver atmel_pmecc_driver = { 1004 .driver = { 1005 .name = "atmel-pmecc", 1006 .of_match_table = atmel_pmecc_match, 1007 }, 1008 .probe = atmel_pmecc_probe, 1009}; 1010module_platform_driver(atmel_pmecc_driver); 1011 1012MODULE_LICENSE("GPL"); 1013MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 1014MODULE_DESCRIPTION("PMECC engine driver"); 1015MODULE_ALIAS("platform:atmel_pmecc"); 1016