1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2008 Oracle.  All rights reserved.
4 *
5 * Based on jffs2 zlib code:
6 * Copyright �� 2001-2007 Red Hat, Inc.
7 * Created by David Woodhouse <dwmw2@infradead.org>
8 */
9
10#include <linux/kernel.h>
11#include <linux/slab.h>
12#include <linux/zlib.h>
13#include <linux/zutil.h>
14#include <linux/mm.h>
15#include <linux/init.h>
16#include <linux/err.h>
17#include <linux/sched.h>
18#include <linux/pagemap.h>
19#include <linux/bio.h>
20#include <linux/refcount.h>
21#include "compression.h"
22
23/* workspace buffer size for s390 zlib hardware support */
24#define ZLIB_DFLTCC_BUF_SIZE    (4 * PAGE_SIZE)
25
26struct workspace {
27	z_stream strm;
28	char *buf;
29	unsigned int buf_size;
30	struct list_head list;
31	int level;
32};
33
34static struct workspace_manager wsm;
35
36struct list_head *zlib_get_workspace(unsigned int level)
37{
38	struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);
39	struct workspace *workspace = list_entry(ws, struct workspace, list);
40
41	workspace->level = level;
42
43	return ws;
44}
45
46void zlib_free_workspace(struct list_head *ws)
47{
48	struct workspace *workspace = list_entry(ws, struct workspace, list);
49
50	kvfree(workspace->strm.workspace);
51	kfree(workspace->buf);
52	kfree(workspace);
53}
54
55struct list_head *zlib_alloc_workspace(unsigned int level)
56{
57	struct workspace *workspace;
58	int workspacesize;
59
60	workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
61	if (!workspace)
62		return ERR_PTR(-ENOMEM);
63
64	workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
65			zlib_inflate_workspacesize());
66	workspace->strm.workspace = kvzalloc(workspacesize, GFP_KERNEL | __GFP_NOWARN);
67	workspace->level = level;
68	workspace->buf = NULL;
69	/*
70	 * In case of s390 zlib hardware support, allocate lager workspace
71	 * buffer. If allocator fails, fall back to a single page buffer.
72	 */
73	if (zlib_deflate_dfltcc_enabled()) {
74		workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,
75					 __GFP_NOMEMALLOC | __GFP_NORETRY |
76					 __GFP_NOWARN | GFP_NOIO);
77		workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;
78	}
79	if (!workspace->buf) {
80		workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
81		workspace->buf_size = PAGE_SIZE;
82	}
83	if (!workspace->strm.workspace || !workspace->buf)
84		goto fail;
85
86	INIT_LIST_HEAD(&workspace->list);
87
88	return &workspace->list;
89fail:
90	zlib_free_workspace(&workspace->list);
91	return ERR_PTR(-ENOMEM);
92}
93
94int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
95		u64 start, struct page **pages, unsigned long *out_pages,
96		unsigned long *total_in, unsigned long *total_out)
97{
98	struct workspace *workspace = list_entry(ws, struct workspace, list);
99	int ret;
100	char *data_in = NULL;
101	char *cpage_out;
102	int nr_pages = 0;
103	struct page *in_page = NULL;
104	struct page *out_page = NULL;
105	unsigned long bytes_left;
106	unsigned int in_buf_pages;
107	unsigned long len = *total_out;
108	unsigned long nr_dest_pages = *out_pages;
109	const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
110
111	*out_pages = 0;
112	*total_out = 0;
113	*total_in = 0;
114
115	if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) {
116		pr_warn("BTRFS: deflateInit failed\n");
117		ret = -EIO;
118		goto out;
119	}
120
121	workspace->strm.total_in = 0;
122	workspace->strm.total_out = 0;
123
124	out_page = btrfs_alloc_compr_page();
125	if (out_page == NULL) {
126		ret = -ENOMEM;
127		goto out;
128	}
129	cpage_out = page_address(out_page);
130	pages[0] = out_page;
131	nr_pages = 1;
132
133	workspace->strm.next_in = workspace->buf;
134	workspace->strm.avail_in = 0;
135	workspace->strm.next_out = cpage_out;
136	workspace->strm.avail_out = PAGE_SIZE;
137
138	while (workspace->strm.total_in < len) {
139		/*
140		 * Get next input pages and copy the contents to
141		 * the workspace buffer if required.
142		 */
143		if (workspace->strm.avail_in == 0) {
144			bytes_left = len - workspace->strm.total_in;
145			in_buf_pages = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE),
146					   workspace->buf_size / PAGE_SIZE);
147			if (in_buf_pages > 1) {
148				int i;
149
150				for (i = 0; i < in_buf_pages; i++) {
151					if (data_in) {
152						kunmap_local(data_in);
153						put_page(in_page);
154					}
155					in_page = find_get_page(mapping,
156								start >> PAGE_SHIFT);
157					data_in = kmap_local_page(in_page);
158					copy_page(workspace->buf + i * PAGE_SIZE,
159						  data_in);
160					start += PAGE_SIZE;
161				}
162				workspace->strm.next_in = workspace->buf;
163			} else {
164				if (data_in) {
165					kunmap_local(data_in);
166					put_page(in_page);
167				}
168				in_page = find_get_page(mapping,
169							start >> PAGE_SHIFT);
170				data_in = kmap_local_page(in_page);
171				start += PAGE_SIZE;
172				workspace->strm.next_in = data_in;
173			}
174			workspace->strm.avail_in = min(bytes_left,
175						       (unsigned long) workspace->buf_size);
176		}
177
178		ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);
179		if (ret != Z_OK) {
180			pr_debug("BTRFS: deflate in loop returned %d\n",
181			       ret);
182			zlib_deflateEnd(&workspace->strm);
183			ret = -EIO;
184			goto out;
185		}
186
187		/* we're making it bigger, give up */
188		if (workspace->strm.total_in > 8192 &&
189		    workspace->strm.total_in <
190		    workspace->strm.total_out) {
191			ret = -E2BIG;
192			goto out;
193		}
194		/* we need another page for writing out.  Test this
195		 * before the total_in so we will pull in a new page for
196		 * the stream end if required
197		 */
198		if (workspace->strm.avail_out == 0) {
199			if (nr_pages == nr_dest_pages) {
200				ret = -E2BIG;
201				goto out;
202			}
203			out_page = btrfs_alloc_compr_page();
204			if (out_page == NULL) {
205				ret = -ENOMEM;
206				goto out;
207			}
208			cpage_out = page_address(out_page);
209			pages[nr_pages] = out_page;
210			nr_pages++;
211			workspace->strm.avail_out = PAGE_SIZE;
212			workspace->strm.next_out = cpage_out;
213		}
214		/* we're all done */
215		if (workspace->strm.total_in >= len)
216			break;
217		if (workspace->strm.total_out > max_out)
218			break;
219	}
220	workspace->strm.avail_in = 0;
221	/*
222	 * Call deflate with Z_FINISH flush parameter providing more output
223	 * space but no more input data, until it returns with Z_STREAM_END.
224	 */
225	while (ret != Z_STREAM_END) {
226		ret = zlib_deflate(&workspace->strm, Z_FINISH);
227		if (ret == Z_STREAM_END)
228			break;
229		if (ret != Z_OK && ret != Z_BUF_ERROR) {
230			zlib_deflateEnd(&workspace->strm);
231			ret = -EIO;
232			goto out;
233		} else if (workspace->strm.avail_out == 0) {
234			/* get another page for the stream end */
235			if (nr_pages == nr_dest_pages) {
236				ret = -E2BIG;
237				goto out;
238			}
239			out_page = btrfs_alloc_compr_page();
240			if (out_page == NULL) {
241				ret = -ENOMEM;
242				goto out;
243			}
244			cpage_out = page_address(out_page);
245			pages[nr_pages] = out_page;
246			nr_pages++;
247			workspace->strm.avail_out = PAGE_SIZE;
248			workspace->strm.next_out = cpage_out;
249		}
250	}
251	zlib_deflateEnd(&workspace->strm);
252
253	if (workspace->strm.total_out >= workspace->strm.total_in) {
254		ret = -E2BIG;
255		goto out;
256	}
257
258	ret = 0;
259	*total_out = workspace->strm.total_out;
260	*total_in = workspace->strm.total_in;
261out:
262	*out_pages = nr_pages;
263	if (data_in) {
264		kunmap_local(data_in);
265		put_page(in_page);
266	}
267
268	return ret;
269}
270
271int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
272{
273	struct workspace *workspace = list_entry(ws, struct workspace, list);
274	int ret = 0, ret2;
275	int wbits = MAX_WBITS;
276	char *data_in;
277	size_t total_out = 0;
278	unsigned long page_in_index = 0;
279	size_t srclen = cb->compressed_len;
280	unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
281	unsigned long buf_start;
282	struct page **pages_in = cb->compressed_pages;
283
284	data_in = kmap_local_page(pages_in[page_in_index]);
285	workspace->strm.next_in = data_in;
286	workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
287	workspace->strm.total_in = 0;
288
289	workspace->strm.total_out = 0;
290	workspace->strm.next_out = workspace->buf;
291	workspace->strm.avail_out = workspace->buf_size;
292
293	/* If it's deflate, and it's got no preset dictionary, then
294	   we can tell zlib to skip the adler32 check. */
295	if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
296	    ((data_in[0] & 0x0f) == Z_DEFLATED) &&
297	    !(((data_in[0]<<8) + data_in[1]) % 31)) {
298
299		wbits = -((data_in[0] >> 4) + 8);
300		workspace->strm.next_in += 2;
301		workspace->strm.avail_in -= 2;
302	}
303
304	if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
305		pr_warn("BTRFS: inflateInit failed\n");
306		kunmap_local(data_in);
307		return -EIO;
308	}
309	while (workspace->strm.total_in < srclen) {
310		ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
311		if (ret != Z_OK && ret != Z_STREAM_END)
312			break;
313
314		buf_start = total_out;
315		total_out = workspace->strm.total_out;
316
317		/* we didn't make progress in this inflate call, we're done */
318		if (buf_start == total_out)
319			break;
320
321		ret2 = btrfs_decompress_buf2page(workspace->buf,
322				total_out - buf_start, cb, buf_start);
323		if (ret2 == 0) {
324			ret = 0;
325			goto done;
326		}
327
328		workspace->strm.next_out = workspace->buf;
329		workspace->strm.avail_out = workspace->buf_size;
330
331		if (workspace->strm.avail_in == 0) {
332			unsigned long tmp;
333			kunmap_local(data_in);
334			page_in_index++;
335			if (page_in_index >= total_pages_in) {
336				data_in = NULL;
337				break;
338			}
339			data_in = kmap_local_page(pages_in[page_in_index]);
340			workspace->strm.next_in = data_in;
341			tmp = srclen - workspace->strm.total_in;
342			workspace->strm.avail_in = min(tmp, PAGE_SIZE);
343		}
344	}
345	if (ret != Z_STREAM_END)
346		ret = -EIO;
347	else
348		ret = 0;
349done:
350	zlib_inflateEnd(&workspace->strm);
351	if (data_in)
352		kunmap_local(data_in);
353	return ret;
354}
355
356int zlib_decompress(struct list_head *ws, const u8 *data_in,
357		struct page *dest_page, unsigned long dest_pgoff, size_t srclen,
358		size_t destlen)
359{
360	struct workspace *workspace = list_entry(ws, struct workspace, list);
361	int ret = 0;
362	int wbits = MAX_WBITS;
363	unsigned long to_copy;
364
365	workspace->strm.next_in = data_in;
366	workspace->strm.avail_in = srclen;
367	workspace->strm.total_in = 0;
368
369	workspace->strm.next_out = workspace->buf;
370	workspace->strm.avail_out = workspace->buf_size;
371	workspace->strm.total_out = 0;
372	/* If it's deflate, and it's got no preset dictionary, then
373	   we can tell zlib to skip the adler32 check. */
374	if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
375	    ((data_in[0] & 0x0f) == Z_DEFLATED) &&
376	    !(((data_in[0]<<8) + data_in[1]) % 31)) {
377
378		wbits = -((data_in[0] >> 4) + 8);
379		workspace->strm.next_in += 2;
380		workspace->strm.avail_in -= 2;
381	}
382
383	if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
384		pr_warn("BTRFS: inflateInit failed\n");
385		return -EIO;
386	}
387
388	/*
389	 * Everything (in/out buf) should be at most one sector, there should
390	 * be no need to switch any input/output buffer.
391	 */
392	ret = zlib_inflate(&workspace->strm, Z_FINISH);
393	to_copy = min(workspace->strm.total_out, destlen);
394	if (ret != Z_STREAM_END)
395		goto out;
396
397	memcpy_to_page(dest_page, dest_pgoff, workspace->buf, to_copy);
398
399out:
400	if (unlikely(to_copy != destlen)) {
401		pr_warn_ratelimited("BTRFS: inflate failed, decompressed=%lu expected=%zu\n",
402					to_copy, destlen);
403		ret = -EIO;
404	} else {
405		ret = 0;
406	}
407
408	zlib_inflateEnd(&workspace->strm);
409
410	if (unlikely(to_copy < destlen))
411		memzero_page(dest_page, dest_pgoff + to_copy, destlen - to_copy);
412	return ret;
413}
414
415const struct btrfs_compress_op btrfs_zlib_compress = {
416	.workspace_manager	= &wsm,
417	.max_level		= 9,
418	.default_level		= BTRFS_ZLIB_DEFAULT_LEVEL,
419};
420