1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) Gao Xiang <xiang@kernel.org>
4 *
5 * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
6 * per-CPU virtual memory (in pages) in advance to store such inplace I/O
7 * data if inplace decompression is failed (due to unmet inplace margin for
8 * example).
9 */
10#include "internal.h"
11
12struct erofs_pcpubuf {
13	raw_spinlock_t lock;
14	void *ptr;
15	struct page **pages;
16	unsigned int nrpages;
17};
18
19static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
20
21void *erofs_get_pcpubuf(unsigned int requiredpages)
22	__acquires(pcb->lock)
23{
24	struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
25
26	raw_spin_lock(&pcb->lock);
27	/* check if the per-CPU buffer is too small */
28	if (requiredpages > pcb->nrpages) {
29		raw_spin_unlock(&pcb->lock);
30		put_cpu_var(erofs_pcb);
31		/* (for sparse checker) pretend pcb->lock is still taken */
32		__acquire(pcb->lock);
33		return NULL;
34	}
35	return pcb->ptr;
36}
37
38void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
39{
40	struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
41
42	DBG_BUGON(pcb->ptr != ptr);
43	raw_spin_unlock(&pcb->lock);
44	put_cpu_var(erofs_pcb);
45}
46
47/* the next step: support per-CPU page buffers hotplug */
48int erofs_pcpubuf_growsize(unsigned int nrpages)
49{
50	static DEFINE_MUTEX(pcb_resize_mutex);
51	static unsigned int pcb_nrpages;
52	struct page *pagepool = NULL;
53	int delta, cpu, ret, i;
54
55	mutex_lock(&pcb_resize_mutex);
56	delta = nrpages - pcb_nrpages;
57	ret = 0;
58	/* avoid shrinking pcpubuf, since no idea how many fses rely on */
59	if (delta <= 0)
60		goto out;
61
62	for_each_possible_cpu(cpu) {
63		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
64		struct page **pages, **oldpages;
65		void *ptr, *old_ptr;
66
67		pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
68		if (!pages) {
69			ret = -ENOMEM;
70			break;
71		}
72
73		for (i = 0; i < nrpages; ++i) {
74			pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
75			if (!pages[i]) {
76				ret = -ENOMEM;
77				oldpages = pages;
78				goto free_pagearray;
79			}
80		}
81		ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
82		if (!ptr) {
83			ret = -ENOMEM;
84			oldpages = pages;
85			goto free_pagearray;
86		}
87		raw_spin_lock(&pcb->lock);
88		old_ptr = pcb->ptr;
89		pcb->ptr = ptr;
90		oldpages = pcb->pages;
91		pcb->pages = pages;
92		i = pcb->nrpages;
93		pcb->nrpages = nrpages;
94		raw_spin_unlock(&pcb->lock);
95
96		if (!oldpages) {
97			DBG_BUGON(old_ptr);
98			continue;
99		}
100
101		if (old_ptr)
102			vunmap(old_ptr);
103free_pagearray:
104		while (i)
105			erofs_pagepool_add(&pagepool, oldpages[--i]);
106		kfree(oldpages);
107		if (ret)
108			break;
109	}
110	pcb_nrpages = nrpages;
111	erofs_release_pages(&pagepool);
112out:
113	mutex_unlock(&pcb_resize_mutex);
114	return ret;
115}
116
117void __init erofs_pcpubuf_init(void)
118{
119	int cpu;
120
121	for_each_possible_cpu(cpu) {
122		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
123
124		raw_spin_lock_init(&pcb->lock);
125	}
126}
127
128void erofs_pcpubuf_exit(void)
129{
130	int cpu, i;
131
132	for_each_possible_cpu(cpu) {
133		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
134
135		if (pcb->ptr) {
136			vunmap(pcb->ptr);
137			pcb->ptr = NULL;
138		}
139		if (!pcb->pages)
140			continue;
141
142		for (i = 0; i < pcb->nrpages; ++i)
143			if (pcb->pages[i])
144				put_page(pcb->pages[i]);
145		kfree(pcb->pages);
146		pcb->pages = NULL;
147	}
148}
149