subr_percpu.c revision 1.16 
1/*	$NetBSD: subr_percpu.c,v 1.16 2012/01/27 19:48:40 para Exp $	*/
2
3/*-
4 * Copyright (c)2007,2008 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29/*
30 * per-cpu storage.
31 */
32
33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: subr_percpu.c,v 1.16 2012/01/27 19:48:40 para Exp $");
35
36#include <sys/param.h>
37#include <sys/cpu.h>
38#include <sys/kmem.h>
39#include <sys/kernel.h>
40#include <sys/mutex.h>
41#include <sys/percpu.h>
42#include <sys/rwlock.h>
43#include <sys/vmem.h>
44#include <sys/xcall.h>
45
46#define	PERCPU_QUANTUM_SIZE	(ALIGNBYTES + 1)
47#define	PERCPU_QCACHE_MAX	0
48#define	PERCPU_IMPORT_SIZE	2048
49
50#if defined(DIAGNOSTIC)
51#define	MAGIC	0x50435055	/* "PCPU" */
52#define	percpu_encrypt(pc)	((pc) ^ MAGIC)
53#define	percpu_decrypt(pc)	((pc) ^ MAGIC)
54#else /* defined(DIAGNOSTIC) */
55#define	percpu_encrypt(pc)	(pc)
56#define	percpu_decrypt(pc)	(pc)
57#endif /* defined(DIAGNOSTIC) */
58
59static krwlock_t	percpu_swap_lock	__cacheline_aligned;
60static kmutex_t		percpu_allocation_lock	__cacheline_aligned;
61static vmem_t *		percpu_offset_arena	__cacheline_aligned;
62static unsigned int	percpu_nextoff		__cacheline_aligned;
63
64static percpu_cpu_t *
65cpu_percpu(struct cpu_info *ci)
66{
67
68	return &ci->ci_data.cpu_percpu;
69}
70
71static unsigned int
72percpu_offset(percpu_t *pc)
73{
74	const unsigned int off = percpu_decrypt((uintptr_t)pc);
75
76	KASSERT(off < percpu_nextoff);
77	return off;
78}
79
80/*
81 * percpu_cpu_swap: crosscall handler for percpu_cpu_enlarge
82 */
83
84static void
85percpu_cpu_swap(void *p1, void *p2)
86{
87	struct cpu_info * const ci = p1;
88	percpu_cpu_t * const newpcc = p2;
89	percpu_cpu_t * const pcc = cpu_percpu(ci);
90
91	KASSERT(ci == curcpu() || !mp_online);
92
93	/*
94	 * swap *pcc and *newpcc unless anyone has beaten us.
95	 */
96	rw_enter(&percpu_swap_lock, RW_WRITER);
97	if (newpcc->pcc_size > pcc->pcc_size) {
98		percpu_cpu_t tmp;
99		int s;
100
101		tmp = *pcc;
102
103		/*
104		 * block interrupts so that we don't lose their modifications.
105		 */
106
107		s = splhigh();
108
109		/*
110		 * copy data to new storage.
111		 */
112
113		memcpy(newpcc->pcc_data, pcc->pcc_data, pcc->pcc_size);
114
115		/*
116		 * this assignment needs to be atomic for percpu_getptr_remote.
117		 */
118
119		pcc->pcc_data = newpcc->pcc_data;
120
121		splx(s);
122
123		pcc->pcc_size = newpcc->pcc_size;
124		*newpcc = tmp;
125	}
126	rw_exit(&percpu_swap_lock);
127}
128
129/*
130 * percpu_cpu_enlarge: ensure that percpu_cpu_t of each cpus have enough space
131 */
132
133static void
134percpu_cpu_enlarge(size_t size)
135{
136	CPU_INFO_ITERATOR cii;
137	struct cpu_info *ci;
138
139	for (CPU_INFO_FOREACH(cii, ci)) {
140		percpu_cpu_t pcc;
141
142		pcc.pcc_data = kmem_alloc(size, KM_SLEEP); /* XXX cacheline */
143		pcc.pcc_size = size;
144		if (!mp_online) {
145			percpu_cpu_swap(ci, &pcc);
146		} else {
147			uint64_t where;
148
149			where = xc_unicast(0, percpu_cpu_swap, ci, &pcc, ci);
150			xc_wait(where);
151		}
152		KASSERT(pcc.pcc_size < size);
153		if (pcc.pcc_data != NULL) {
154			kmem_free(pcc.pcc_data, pcc.pcc_size);
155		}
156	}
157}
158
159/*
160 * percpu_backend_alloc: vmem import callback for percpu_offset_arena
161 */
162
163static int
164percpu_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize,
165    vm_flag_t vmflags, vmem_addr_t *addrp)
166{
167	unsigned int offset;
168	unsigned int nextoff;
169
170	ASSERT_SLEEPABLE();
171	KASSERT(dummy == NULL);
172
173	if ((vmflags & VM_NOSLEEP) != 0)
174		return ENOMEM;
175
176	size = roundup(size, PERCPU_IMPORT_SIZE);
177	mutex_enter(&percpu_allocation_lock);
178	offset = percpu_nextoff;
179	percpu_nextoff = nextoff = percpu_nextoff + size;
180	mutex_exit(&percpu_allocation_lock);
181
182	percpu_cpu_enlarge(nextoff);
183
184	*resultsize = size;
185	*addrp = (vmem_addr_t)offset;
186	return 0;
187}
188
189static void
190percpu_zero_cb(void *vp, void *vp2, struct cpu_info *ci)
191{
192	size_t sz = (uintptr_t)vp2;
193
194	memset(vp, 0, sz);
195}
196
197/*
198 * percpu_zero: initialize percpu storage with zero.
199 */
200
201static void
202percpu_zero(percpu_t *pc, size_t sz)
203{
204
205	percpu_foreach(pc, percpu_zero_cb, (void *)(uintptr_t)sz);
206}
207
208/*
209 * percpu_init: subsystem initialization
210 */
211
212void
213percpu_init(void)
214{
215
216	ASSERT_SLEEPABLE();
217	rw_init(&percpu_swap_lock);
218	mutex_init(&percpu_allocation_lock, MUTEX_DEFAULT, IPL_NONE);
219	percpu_nextoff = PERCPU_QUANTUM_SIZE;
220
221	percpu_offset_arena = vmem_xcreate("percpu", 0, 0, PERCPU_QUANTUM_SIZE,
222	    percpu_backend_alloc, NULL, NULL, PERCPU_QCACHE_MAX, VM_SLEEP,
223	    IPL_NONE);
224}
225
226/*
227 * percpu_init_cpu: cpu initialization
228 *
229 * => should be called before the cpu appears on the list for CPU_INFO_FOREACH.
230 */
231
232void
233percpu_init_cpu(struct cpu_info *ci)
234{
235	percpu_cpu_t * const pcc = cpu_percpu(ci);
236	size_t size = percpu_nextoff; /* XXX racy */
237
238	ASSERT_SLEEPABLE();
239	pcc->pcc_size = size;
240	if (size) {
241		pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP);
242	}
243}
244
245/*
246 * percpu_alloc: allocate percpu storage
247 *
248 * => called in thread context.
249 * => considered as an expensive and rare operation.
250 * => allocated storage is initialized with zeros.
251 */
252
253percpu_t *
254percpu_alloc(size_t size)
255{
256	vmem_addr_t offset;
257	percpu_t *pc;
258
259	ASSERT_SLEEPABLE();
260	if (vmem_alloc(percpu_offset_arena, size, VM_SLEEP | VM_BESTFIT,
261	    &offset) != 0)
262		return NULL;
263	pc = (percpu_t *)percpu_encrypt((uintptr_t)offset);
264	percpu_zero(pc, size);
265	return pc;
266}
267
268/*
269 * percpu_free: free percpu storage
270 *
271 * => called in thread context.
272 * => considered as an expensive and rare operation.
273 */
274
275void
276percpu_free(percpu_t *pc, size_t size)
277{
278
279	ASSERT_SLEEPABLE();
280	vmem_free(percpu_offset_arena, (vmem_addr_t)percpu_offset(pc), size);
281}
282
283/*
284 * percpu_getref:
285 *
286 * => safe to be used in either thread or interrupt context
287 * => disables preemption; must be bracketed with a percpu_putref()
288 */
289
290void *
291percpu_getref(percpu_t *pc)
292{
293
294	KPREEMPT_DISABLE(curlwp);
295	return percpu_getptr_remote(pc, curcpu());
296}
297
298/*
299 * percpu_putref:
300 *
301 * => drops the preemption-disabled count after caller is done with per-cpu
302 *    data
303 */
304
305void
306percpu_putref(percpu_t *pc)
307{
308
309	KPREEMPT_ENABLE(curlwp);
310}
311
312/*
313 * percpu_traverse_enter, percpu_traverse_exit, percpu_getptr_remote:
314 * helpers to access remote cpu's percpu data.
315 *
316 * => called in thread context.
317 * => percpu_traverse_enter can block low-priority xcalls.
318 * => typical usage would be:
319 *
320 *	sum = 0;
321 *	percpu_traverse_enter();
322 *	for (CPU_INFO_FOREACH(cii, ci)) {
323 *		unsigned int *p = percpu_getptr_remote(pc, ci);
324 *		sum += *p;
325 *	}
326 *	percpu_traverse_exit();
327 */
328
329void
330percpu_traverse_enter(void)
331{
332
333	ASSERT_SLEEPABLE();
334	rw_enter(&percpu_swap_lock, RW_READER);
335}
336
337void
338percpu_traverse_exit(void)
339{
340
341	rw_exit(&percpu_swap_lock);
342}
343
344void *
345percpu_getptr_remote(percpu_t *pc, struct cpu_info *ci)
346{
347
348	return &((char *)cpu_percpu(ci)->pcc_data)[percpu_offset(pc)];
349}
350
351/*
352 * percpu_foreach: call the specified callback function for each cpus.
353 *
354 * => called in thread context.
355 * => caller should not rely on the cpu iteration order.
356 * => the callback function should be minimum because it is executed with
357 *    holding a global lock, which can block low-priority xcalls.
358 *    eg. it's illegal for a callback function to sleep for memory allocation.
359 */
360void
361percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg)
362{
363	CPU_INFO_ITERATOR cii;
364	struct cpu_info *ci;
365
366	percpu_traverse_enter();
367	for (CPU_INFO_FOREACH(cii, ci)) {
368		(*cb)(percpu_getptr_remote(pc, ci), arg, ci);
369	}
370	percpu_traverse_exit();
371}
372