1139749Simp// SPDX-License-Identifier: GPL-2.0-or-later 233323Sphk#include <linux/zlib.h> 333323Sphk#include "compress.h" 433323Sphk 533323Sphkstruct z_erofs_deflate { 633323Sphk struct z_erofs_deflate *next; 733323Sphk struct z_stream_s z; 833323Sphk u8 bounce[PAGE_SIZE]; 933323Sphk}; 1036938Sphk 1136938Sphkstatic DEFINE_SPINLOCK(z_erofs_deflate_lock); 12185003Sjhbstatic unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; 1336938Sphkstatic struct z_erofs_deflate *z_erofs_deflate_head; 1436938Sphkstatic DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); 1533323Sphk 1633323Sphkmodule_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); 17119418Sobrien 18119418Sobrienvoid z_erofs_deflate_exit(void) 19119418Sobrien{ 2033323Sphk /* there should be no running fs instance */ 21187576Sjhb while (z_erofs_deflate_avail_strms) { 2233323Sphk struct z_erofs_deflate *strm; 2333323Sphk 2455939Snsouch spin_lock(&z_erofs_deflate_lock); 25187576Sjhb strm = z_erofs_deflate_head; 2655939Snsouch if (!strm) { 2733323Sphk spin_unlock(&z_erofs_deflate_lock); 2836739Sphk continue; 2955939Snsouch } 3055939Snsouch z_erofs_deflate_head = NULL; 3155939Snsouch spin_unlock(&z_erofs_deflate_lock); 3233323Sphk 3333323Sphk while (strm) { 3455939Snsouch struct z_erofs_deflate *n = strm->next; 3555939Snsouch 3633323Sphk vfree(strm->z.workspace); 3749550Sphk kfree(strm); 3833323Sphk --z_erofs_deflate_avail_strms; 3997228Speter strm = n; 4088220Simp } 4149550Sphk } 42143390Simp} 4383818Sphk 44130585Sphkint __init z_erofs_deflate_init(void) 4583818Sphk{ 4683818Sphk /* by default, use # of possible CPUs instead */ 4783818Sphk if (!z_erofs_deflate_nstrms) 48187576Sjhb z_erofs_deflate_nstrms = num_possible_cpus(); 4983818Sphk 5055939Snsouch for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; 51187576Sjhb ++z_erofs_deflate_avail_strms) { 5255939Snsouch struct z_erofs_deflate *strm; 5355939Snsouch 5449550Sphk strm = kzalloc(sizeof(*strm), GFP_KERNEL); 5533323Sphk if (!strm) 56187576Sjhb goto out_failed; 5783818Sphk 5833323Sphk /* XXX: in-kernel zlib cannot shrink windowbits currently */ 5955939Snsouch strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); 6055939Snsouch if (!strm->z.workspace) { 6133323Sphk kfree(strm); 6255939Snsouch goto out_failed; 6333323Sphk } 6433396Sphk 6533396Sphk spin_lock(&z_erofs_deflate_lock); 6636739Sphk strm->next = z_erofs_deflate_head; 6733323Sphk z_erofs_deflate_head = strm; 6847625Sphk spin_unlock(&z_erofs_deflate_lock); 69126080Sphk } 70111815Sphk return 0; 71111815Sphk 72111815Sphkout_failed: 73111815Sphk erofs_err(NULL, "failed to allocate zlib workspace"); 7447625Sphk z_erofs_deflate_exit(); 7533323Sphk return -ENOMEM; 7656455Speter} 7756455Speter 7856455Speterint z_erofs_load_deflate_config(struct super_block *sb, 7956455Speter struct erofs_super_block *dsb, void *data, int size) 80127189Sguido{ 81127189Sguido struct z_erofs_deflate_cfgs *dfl = data; 82184130Sjhb 83127189Sguido if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { 84127189Sguido erofs_err(sb, "invalid deflate cfgs, size=%u", size); 8556455Speter return -EINVAL; 8656455Speter } 8755939Snsouch 8883818Sphk if (dfl->windowbits > MAX_WBITS) { 8983818Sphk erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); 9083818Sphk return -EOPNOTSUPP; 9183818Sphk } 9283818Sphk 9383818Sphk erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!"); 9488220Simp return 0; 9583818Sphk} 9683818Sphk 9783818Sphkint z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, 9883818Sphk struct page **pgpl) 9955939Snsouch{ 10033323Sphk const unsigned int nrpages_out = 10188220Simp PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 10288220Simp const unsigned int nrpages_in = 10388220Simp PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; 10488220Simp struct super_block *sb = rq->sb; 10588220Simp unsigned int insz, outsz, pofs; 10688220Simp struct z_erofs_deflate *strm; 10788220Simp u8 *kin, *kout = NULL; 10888220Simp bool bounced = false; 10988220Simp int no = -1, ni = 0, j = 0, zerr, err; 11088220Simp 111130585Sphk /* 1. get the exact DEFLATE compressed size */ 112187576Sjhb kin = kmap_local_page(*rq->in); 11333323Sphk err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, 114183053Sjhb min_t(unsigned int, rq->inputsize, 115183053Sjhb sb->s_blocksize - rq->pageofs_in)); 116183053Sjhb if (err) { 117183053Sjhb kunmap_local(kin); 11888220Simp return err; 119187576Sjhb } 120187576Sjhb 12188220Simp /* 2. get an available DEFLATE context */ 122187576Sjhbagain: 12388220Simp spin_lock(&z_erofs_deflate_lock); 124187576Sjhb strm = z_erofs_deflate_head; 125187576Sjhb if (!strm) { 126187576Sjhb spin_unlock(&z_erofs_deflate_lock); 127187576Sjhb wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); 128187576Sjhb goto again; 129187576Sjhb } 130187576Sjhb z_erofs_deflate_head = strm->next; 131187576Sjhb spin_unlock(&z_erofs_deflate_lock); 132187576Sjhb 133187576Sjhb /* 3. multi-call decompress */ 134187576Sjhb insz = rq->inputsize; 13588220Simp outsz = rq->outputsize; 13688220Simp zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); 13783818Sphk if (zerr != Z_OK) { 13888220Simp err = -EIO; 139108322Srwatson goto failed_zinit; 14088220Simp } 14183818Sphk 14288220Simp pofs = rq->pageofs_out; 14388220Simp strm->z.avail_in = min_t(u32, insz, PAGE_SIZE - rq->pageofs_in); 14483818Sphk insz -= strm->z.avail_in; 14533323Sphk strm->z.next_in = kin + rq->pageofs_in; 146187576Sjhb strm->z.avail_out = 0; 147187576Sjhb 148187576Sjhb while (1) { 14983818Sphk if (!strm->z.avail_out) { 150187576Sjhb if (++no >= nrpages_out || !outsz) { 15133323Sphk erofs_err(sb, "insufficient space for decompressed data"); 15283818Sphk err = -EFSCORRUPTED; 15383818Sphk break; 15488220Simp } 15583818Sphk 15683818Sphk if (kout) 15783818Sphk kunmap_local(kout); 15883818Sphk strm->z.avail_out = min_t(u32, outsz, PAGE_SIZE - pofs); 15983818Sphk outsz -= strm->z.avail_out; 16083818Sphk if (!rq->out[no]) { 16183818Sphk rq->out[no] = erofs_allocpage(pgpl, rq->gfp); 16283818Sphk if (!rq->out[no]) { 16383818Sphk kout = NULL; 16483818Sphk err = -ENOMEM; 16583818Sphk break; 16683818Sphk } 16783818Sphk set_page_private(rq->out[no], 16883818Sphk Z_EROFS_SHORTLIVED_PAGE); 16983818Sphk } 17088220Simp kout = kmap_local_page(rq->out[no]); 17183818Sphk strm->z.next_out = kout + pofs; 17283818Sphk pofs = 0; 17383818Sphk } 17483818Sphk 17583818Sphk if (!strm->z.avail_in && insz) { 17683818Sphk if (++ni >= nrpages_in) { 17783818Sphk erofs_err(sb, "invalid compressed data"); 17883818Sphk err = -EFSCORRUPTED; 17983818Sphk break; 18083818Sphk } 18183818Sphk 18288220Simp if (kout) { /* unlike kmap(), take care of the orders */ 18383818Sphk j = strm->z.next_out - kout; 18483818Sphk kunmap_local(kout); 18583818Sphk } 18683818Sphk kunmap_local(kin); 187187576Sjhb strm->z.avail_in = min_t(u32, insz, PAGE_SIZE); 18883818Sphk insz -= strm->z.avail_in; 18983818Sphk kin = kmap_local_page(rq->in[ni]); 19088220Simp strm->z.next_in = kin; 191108322Srwatson bounced = false; 19288220Simp if (kout) { 19383818Sphk kout = kmap_local_page(rq->out[no]); 19488220Simp strm->z.next_out = kout + j; 195106563Sjhb } 19683818Sphk } 19783818Sphk 198187576Sjhb /* 19983818Sphk * Handle overlapping: Use bounced buffer if the compressed 20083818Sphk * data is under processing; Or use short-lived pages from the 20188220Simp * on-stack pagepool where pages share among the same request 202187576Sjhb * and not _all_ inplace I/O pages are needed to be doubled. 20333323Sphk */ 20455939Snsouch if (!bounced && rq->out[no] == rq->in[ni]) { 20533323Sphk memcpy(strm->bounce, strm->z.next_in, strm->z.avail_in); 20633323Sphk strm->z.next_in = strm->bounce; 20733323Sphk bounced = true; 208130585Sphk } 20933323Sphk 21083818Sphk for (j = ni + 1; j < nrpages_in; ++j) { 211187576Sjhb struct page *tmppage; 212106563Sjhb 213187576Sjhb if (rq->out[no] != rq->in[j]) 21433323Sphk continue; 215187576Sjhb tmppage = erofs_allocpage(pgpl, rq->gfp); 216187576Sjhb if (!tmppage) { 217187576Sjhb err = -ENOMEM; 218187576Sjhb goto failed; 219187576Sjhb } 220187576Sjhb set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 221187576Sjhb copy_highpage(tmppage, rq->in[j]); 22283818Sphk rq->in[j] = tmppage; 22383818Sphk } 22483818Sphk 225187576Sjhb zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); 226187576Sjhb if (zerr != Z_OK || !(outsz + strm->z.avail_out)) { 227187576Sjhb if (zerr == Z_OK && rq->partial_decoding) 22846053Sphk break; 22955939Snsouch if (zerr == Z_STREAM_END && !outsz) 23055939Snsouch break; 23183818Sphk erofs_err(sb, "failed to decompress %d in[%u] out[%u]", 23288220Simp zerr, rq->inputsize, rq->outputsize); 23383818Sphk err = -EFSCORRUPTED; 23483818Sphk break; 23583818Sphk } 236143390Simp } 23783818Sphkfailed: 238187576Sjhb if (zlib_inflateEnd(&strm->z) != Z_OK && !err) 239187576Sjhb err = -EIO; 24083818Sphk if (kout) 24146053Sphk kunmap_local(kout); 24283818Sphkfailed_zinit: 243187576Sjhb kunmap_local(kin); 244187576Sjhb /* 4. push back DEFLATE stream context to the global list */ 24533323Sphk spin_lock(&z_erofs_deflate_lock); 24633323Sphk strm->next = z_erofs_deflate_head; 24733323Sphk z_erofs_deflate_head = strm; 24833323Sphk spin_unlock(&z_erofs_deflate_lock); 249130585Sphk wake_up(&z_erofs_deflate_wq); 25033323Sphk return err; 25183818Sphk} 252106563Sjhb