1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * This file provides ECC correction for more than 1 bit per block of data,
4 * using binary BCH codes. It relies on the generic BCH library lib/bch.c.
5 *
6 * Copyright �� 2011 Ivan Djelic <ivan.djelic@parrot.com>
7 */
8
9#include <linux/types.h>
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/slab.h>
13#include <linux/bitops.h>
14#include <linux/mtd/nand.h>
15#include <linux/mtd/nand-ecc-sw-bch.h>
16
17/**
18 * nand_ecc_sw_bch_calculate - Calculate the ECC corresponding to a data block
19 * @nand: NAND device
20 * @buf: Input buffer with raw data
21 * @code: Output buffer with ECC
22 */
23int nand_ecc_sw_bch_calculate(struct nand_device *nand,
24			      const unsigned char *buf, unsigned char *code)
25{
26	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
27	unsigned int i;
28
29	memset(code, 0, engine_conf->code_size);
30	bch_encode(engine_conf->bch, buf, nand->ecc.ctx.conf.step_size, code);
31
32	/* apply mask so that an erased page is a valid codeword */
33	for (i = 0; i < engine_conf->code_size; i++)
34		code[i] ^= engine_conf->eccmask[i];
35
36	return 0;
37}
38EXPORT_SYMBOL(nand_ecc_sw_bch_calculate);
39
40/**
41 * nand_ecc_sw_bch_correct - Detect, correct and report bit error(s)
42 * @nand: NAND device
43 * @buf: Raw data read from the chip
44 * @read_ecc: ECC bytes from the chip
45 * @calc_ecc: ECC calculated from the raw data
46 *
47 * Detect and correct bit errors for a data block.
48 */
49int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf,
50			    unsigned char *read_ecc, unsigned char *calc_ecc)
51{
52	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
53	unsigned int step_size = nand->ecc.ctx.conf.step_size;
54	unsigned int *errloc = engine_conf->errloc;
55	int i, count;
56
57	count = bch_decode(engine_conf->bch, NULL, step_size, read_ecc,
58			   calc_ecc, NULL, errloc);
59	if (count > 0) {
60		for (i = 0; i < count; i++) {
61			if (errloc[i] < (step_size * 8))
62				/* The error is in the data area: correct it */
63				buf[errloc[i] >> 3] ^= (1 << (errloc[i] & 7));
64
65			/* Otherwise the error is in the ECC area: nothing to do */
66			pr_debug("%s: corrected bitflip %u\n", __func__,
67				 errloc[i]);
68		}
69	} else if (count < 0) {
70		pr_err("ECC unrecoverable error\n");
71		count = -EBADMSG;
72	}
73
74	return count;
75}
76EXPORT_SYMBOL(nand_ecc_sw_bch_correct);
77
78/**
79 * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources
80 * @nand: NAND device
81 */
82static void nand_ecc_sw_bch_cleanup(struct nand_device *nand)
83{
84	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
85
86	bch_free(engine_conf->bch);
87	kfree(engine_conf->errloc);
88	kfree(engine_conf->eccmask);
89}
90
91/**
92 * nand_ecc_sw_bch_init - Initialize software BCH ECC engine
93 * @nand: NAND device
94 *
95 * Returns: a pointer to a new NAND BCH control structure, or NULL upon failure
96 *
97 * Initialize NAND BCH error correction. @nand.ecc parameters 'step_size' and
98 * 'bytes' are used to compute the following BCH parameters:
99 *     m, the Galois field order
100 *     t, the error correction capability
101 * 'bytes' should be equal to the number of bytes required to store m * t
102 * bits, where m is such that 2^m - 1 > step_size * 8.
103 *
104 * Example: to configure 4 bit correction per 512 bytes, you should pass
105 * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8)
106 * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits)
107 */
108static int nand_ecc_sw_bch_init(struct nand_device *nand)
109{
110	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
111	unsigned int eccsize = nand->ecc.ctx.conf.step_size;
112	unsigned int eccbytes = engine_conf->code_size;
113	unsigned int m, t, i;
114	unsigned char *erased_page;
115	int ret;
116
117	m = fls(1 + (8 * eccsize));
118	t = (eccbytes * 8) / m;
119
120	engine_conf->bch = bch_init(m, t, 0, false);
121	if (!engine_conf->bch)
122		return -EINVAL;
123
124	engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL);
125	engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc),
126					    GFP_KERNEL);
127	if (!engine_conf->eccmask || !engine_conf->errloc) {
128		ret = -ENOMEM;
129		goto cleanup;
130	}
131
132	/* Compute and store the inverted ECC of an erased step */
133	erased_page = kmalloc(eccsize, GFP_KERNEL);
134	if (!erased_page) {
135		ret = -ENOMEM;
136		goto cleanup;
137	}
138
139	memset(erased_page, 0xff, eccsize);
140	bch_encode(engine_conf->bch, erased_page, eccsize,
141		   engine_conf->eccmask);
142	kfree(erased_page);
143
144	for (i = 0; i < eccbytes; i++)
145		engine_conf->eccmask[i] ^= 0xff;
146
147	/* Verify that the number of code bytes has the expected value */
148	if (engine_conf->bch->ecc_bytes != eccbytes) {
149		pr_err("Invalid number of ECC bytes: %u, expected: %u\n",
150		       eccbytes, engine_conf->bch->ecc_bytes);
151		ret = -EINVAL;
152		goto cleanup;
153	}
154
155	/* Sanity checks */
156	if (8 * (eccsize + eccbytes) >= (1 << m)) {
157		pr_err("ECC step size is too large (%u)\n", eccsize);
158		ret = -EINVAL;
159		goto cleanup;
160	}
161
162	return 0;
163
164cleanup:
165	nand_ecc_sw_bch_cleanup(nand);
166
167	return ret;
168}
169
170int nand_ecc_sw_bch_init_ctx(struct nand_device *nand)
171{
172	struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
173	struct mtd_info *mtd = nanddev_to_mtd(nand);
174	struct nand_ecc_sw_bch_conf *engine_conf;
175	unsigned int code_size = 0, nsteps;
176	int ret;
177
178	/* Only large page NAND chips may use BCH */
179	if (mtd->oobsize < 64) {
180		pr_err("BCH cannot be used with small page NAND chips\n");
181		return -EINVAL;
182	}
183
184	if (!mtd->ooblayout)
185		mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
186
187	conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
188	conf->algo = NAND_ECC_ALGO_BCH;
189	conf->step_size = nand->ecc.user_conf.step_size;
190	conf->strength = nand->ecc.user_conf.strength;
191
192	/*
193	 * Board driver should supply ECC size and ECC strength
194	 * values to select how many bits are correctable.
195	 * Otherwise, default to 512 bytes for large page devices and 256 for
196	 * small page devices.
197	 */
198	if (!conf->step_size) {
199		if (mtd->oobsize >= 64)
200			conf->step_size = 512;
201		else
202			conf->step_size = 256;
203
204		conf->strength = 4;
205	}
206
207	nsteps = mtd->writesize / conf->step_size;
208
209	/* Maximize */
210	if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
211		conf->step_size = 1024;
212		nsteps = mtd->writesize / conf->step_size;
213		/* Reserve 2 bytes for the BBM */
214		code_size = (mtd->oobsize - 2) / nsteps;
215		conf->strength = code_size * 8 / fls(8 * conf->step_size);
216	}
217
218	if (!code_size)
219		code_size = DIV_ROUND_UP(conf->strength *
220					 fls(8 * conf->step_size), 8);
221
222	if (!conf->strength)
223		conf->strength = (code_size * 8) / fls(8 * conf->step_size);
224
225	if (!code_size && !conf->strength) {
226		pr_err("Missing ECC parameters\n");
227		return -EINVAL;
228	}
229
230	engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);
231	if (!engine_conf)
232		return -ENOMEM;
233
234	ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand);
235	if (ret)
236		goto free_engine_conf;
237
238	engine_conf->code_size = code_size;
239	engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
240	engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
241	if (!engine_conf->calc_buf || !engine_conf->code_buf) {
242		ret = -ENOMEM;
243		goto free_bufs;
244	}
245
246	nand->ecc.ctx.priv = engine_conf;
247	nand->ecc.ctx.nsteps = nsteps;
248	nand->ecc.ctx.total = nsteps * code_size;
249
250	ret = nand_ecc_sw_bch_init(nand);
251	if (ret)
252		goto free_bufs;
253
254	/* Verify the layout validity */
255	if (mtd_ooblayout_count_eccbytes(mtd) !=
256	    nand->ecc.ctx.nsteps * engine_conf->code_size) {
257		pr_err("Invalid ECC layout\n");
258		ret = -EINVAL;
259		goto cleanup_bch_ctx;
260	}
261
262	return 0;
263
264cleanup_bch_ctx:
265	nand_ecc_sw_bch_cleanup(nand);
266free_bufs:
267	nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
268	kfree(engine_conf->calc_buf);
269	kfree(engine_conf->code_buf);
270free_engine_conf:
271	kfree(engine_conf);
272
273	return ret;
274}
275EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx);
276
277void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand)
278{
279	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
280
281	if (engine_conf) {
282		nand_ecc_sw_bch_cleanup(nand);
283		nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
284		kfree(engine_conf->calc_buf);
285		kfree(engine_conf->code_buf);
286		kfree(engine_conf);
287	}
288}
289EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx);
290
291static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand,
292					  struct nand_page_io_req *req)
293{
294	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
295	struct mtd_info *mtd = nanddev_to_mtd(nand);
296	int eccsize = nand->ecc.ctx.conf.step_size;
297	int eccbytes = engine_conf->code_size;
298	int eccsteps = nand->ecc.ctx.nsteps;
299	int total = nand->ecc.ctx.total;
300	u8 *ecccalc = engine_conf->calc_buf;
301	const u8 *data;
302	int i;
303
304	/* Nothing to do for a raw operation */
305	if (req->mode == MTD_OPS_RAW)
306		return 0;
307
308	/* This engine does not provide BBM/free OOB bytes protection */
309	if (!req->datalen)
310		return 0;
311
312	nand_ecc_tweak_req(&engine_conf->req_ctx, req);
313
314	/* No more preparation for page read */
315	if (req->type == NAND_PAGE_READ)
316		return 0;
317
318	/* Preparation for page write: derive the ECC bytes and place them */
319	for (i = 0, data = req->databuf.out;
320	     eccsteps;
321	     eccsteps--, i += eccbytes, data += eccsize)
322		nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
323
324	return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out,
325					  0, total);
326}
327
328static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand,
329					 struct nand_page_io_req *req)
330{
331	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
332	struct mtd_info *mtd = nanddev_to_mtd(nand);
333	int eccsize = nand->ecc.ctx.conf.step_size;
334	int total = nand->ecc.ctx.total;
335	int eccbytes = engine_conf->code_size;
336	int eccsteps = nand->ecc.ctx.nsteps;
337	u8 *ecccalc = engine_conf->calc_buf;
338	u8 *ecccode = engine_conf->code_buf;
339	unsigned int max_bitflips = 0;
340	u8 *data = req->databuf.in;
341	int i, ret;
342
343	/* Nothing to do for a raw operation */
344	if (req->mode == MTD_OPS_RAW)
345		return 0;
346
347	/* This engine does not provide BBM/free OOB bytes protection */
348	if (!req->datalen)
349		return 0;
350
351	/* No more preparation for page write */
352	if (req->type == NAND_PAGE_WRITE) {
353		nand_ecc_restore_req(&engine_conf->req_ctx, req);
354		return 0;
355	}
356
357	/* Finish a page read: retrieve the (raw) ECC bytes*/
358	ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0,
359					 total);
360	if (ret)
361		return ret;
362
363	/* Calculate the ECC bytes */
364	for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize)
365		nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
366
367	/* Finish a page read: compare and correct */
368	for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in;
369	     eccsteps;
370	     eccsteps--, i += eccbytes, data += eccsize) {
371		int stat =  nand_ecc_sw_bch_correct(nand, data,
372						    &ecccode[i],
373						    &ecccalc[i]);
374		if (stat < 0) {
375			mtd->ecc_stats.failed++;
376		} else {
377			mtd->ecc_stats.corrected += stat;
378			max_bitflips = max_t(unsigned int, max_bitflips, stat);
379		}
380	}
381
382	nand_ecc_restore_req(&engine_conf->req_ctx, req);
383
384	return max_bitflips;
385}
386
387static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = {
388	.init_ctx = nand_ecc_sw_bch_init_ctx,
389	.cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx,
390	.prepare_io_req = nand_ecc_sw_bch_prepare_io_req,
391	.finish_io_req = nand_ecc_sw_bch_finish_io_req,
392};
393
394static struct nand_ecc_engine nand_ecc_sw_bch_engine = {
395	.ops = &nand_ecc_sw_bch_engine_ops,
396};
397
398struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void)
399{
400	return &nand_ecc_sw_bch_engine;
401}
402EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine);
403
404MODULE_LICENSE("GPL");
405MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>");
406MODULE_DESCRIPTION("NAND software BCH ECC support");
407