1/*	$OpenBSD: subr_percpu.c,v 1.11 2023/09/16 09:33:27 mpi Exp $ */
2
3/*
4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/pool.h>
22#include <sys/malloc.h>
23
24#include <sys/percpu.h>
25
26#ifdef MULTIPROCESSOR
27struct pool cpumem_pl;
28
29void
30percpu_init(void)
31{
32	pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpusfound, 0,
33	    IPL_NONE, PR_WAITOK, "percpumem", &pool_allocator_single);
34}
35
36struct cpumem *
37cpumem_get(struct pool *pp)
38{
39	struct cpumem *cm;
40	unsigned int cpu;
41
42	cm = pool_get(&cpumem_pl, PR_WAITOK);
43
44	for (cpu = 0; cpu < ncpusfound; cpu++)
45		cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO);
46
47	return (cm);
48}
49
50void
51cpumem_put(struct pool *pp, struct cpumem *cm)
52{
53	unsigned int cpu;
54
55	for (cpu = 0; cpu < ncpusfound; cpu++)
56		pool_put(pp, cm[cpu].mem);
57
58	pool_put(&cpumem_pl, cm);
59}
60
61struct cpumem *
62cpumem_malloc(size_t sz, int type)
63{
64	struct cpumem *cm;
65	unsigned int cpu;
66
67	sz = roundup(sz, CACHELINESIZE);
68
69	cm = pool_get(&cpumem_pl, PR_WAITOK);
70
71	for (cpu = 0; cpu < ncpusfound; cpu++)
72		cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
73
74	return (cm);
75}
76
77struct cpumem *
78cpumem_malloc_ncpus(struct cpumem *bootcm, size_t sz, int type)
79{
80	struct cpumem *cm;
81	unsigned int cpu;
82
83	sz = roundup(sz, CACHELINESIZE);
84
85	cm = pool_get(&cpumem_pl, PR_WAITOK);
86
87	cm[0].mem = bootcm[0].mem;
88	for (cpu = 1; cpu < ncpusfound; cpu++)
89		cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
90
91	return (cm);
92}
93
94void
95cpumem_free(struct cpumem *cm, int type, size_t sz)
96{
97	unsigned int cpu;
98
99	sz = roundup(sz, CACHELINESIZE);
100
101	for (cpu = 0; cpu < ncpusfound; cpu++)
102		free(cm[cpu].mem, type, sz);
103
104	pool_put(&cpumem_pl, cm);
105}
106
107void *
108cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
109{
110	i->cpu = 0;
111
112	return (cm[0].mem);
113}
114
115void *
116cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
117{
118	unsigned int cpu = ++i->cpu;
119
120	if (cpu >= ncpusfound)
121		return (NULL);
122
123	return (cm[cpu].mem);
124}
125
126struct cpumem *
127counters_alloc(unsigned int n)
128{
129	struct cpumem *cm;
130	struct cpumem_iter cmi;
131	uint64_t *counters;
132	unsigned int i;
133
134	KASSERT(n > 0);
135
136	n++; /* add space for a generation number */
137	cm = cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS);
138
139	CPUMEM_FOREACH(counters, &cmi, cm) {
140		for (i = 0; i < n; i++)
141			counters[i] = 0;
142	}
143
144	return (cm);
145}
146
147struct cpumem *
148counters_alloc_ncpus(struct cpumem *cm, unsigned int n)
149{
150	n++; /* the generation number */
151	return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS));
152}
153
154void
155counters_free(struct cpumem *cm, unsigned int n)
156{
157	n++; /* generation number */
158	cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t));
159}
160
161void
162counters_read(struct cpumem *cm, uint64_t *output, unsigned int n,
163    uint64_t *scratch)
164{
165	struct cpumem_iter cmi;
166	uint64_t *gen, *counters, *temp = scratch;
167	uint64_t enter, leave;
168	unsigned int i;
169
170	for (i = 0; i < n; i++)
171		output[i] = 0;
172
173	if (scratch == NULL)
174		temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK);
175
176	gen = cpumem_first(&cmi, cm);
177	do {
178		counters = gen + 1;
179
180		enter = *gen;
181		for (;;) {
182			/* the generation number is odd during an update */
183			while (enter & 1) {
184				yield();
185				enter = *gen;
186			}
187
188			membar_consumer();
189			for (i = 0; i < n; i++)
190				temp[i] = counters[i];
191
192			membar_consumer();
193			leave = *gen;
194
195			if (enter == leave)
196				break;
197
198			enter = leave;
199		}
200
201		for (i = 0; i < n; i++)
202			output[i] += temp[i];
203
204		gen = cpumem_next(&cmi, cm);
205	} while (gen != NULL);
206
207	if (scratch == NULL)
208		free(temp, M_TEMP, n * sizeof(uint64_t));
209}
210
211void
212counters_zero(struct cpumem *cm, unsigned int n)
213{
214	struct cpumem_iter cmi;
215	uint64_t *counters;
216	unsigned int i;
217
218	counters = cpumem_first(&cmi, cm);
219	membar_producer();
220	do {
221		for (i = 0; i < n; i++)
222			counters[i] = 0;
223		/* zero the generation numbers too */
224		membar_producer();
225		counters[i] = 0;
226
227		counters = cpumem_next(&cmi, cm);
228	} while (counters != NULL);
229}
230
231#else /* MULTIPROCESSOR */
232
233/*
234 * Uniprocessor implementation of per-CPU data structures.
235 *
236 * UP percpu memory is a single memory allocation cast to/from the
237 * cpumem struct. It is not scaled up to the size of cacheline because
238 * there's no other cache to contend with.
239 */
240
241void
242percpu_init(void)
243{
244	/* nop */
245}
246
247struct cpumem *
248cpumem_get(struct pool *pp)
249{
250	return (pool_get(pp, PR_WAITOK | PR_ZERO));
251}
252
253void
254cpumem_put(struct pool *pp, struct cpumem *cm)
255{
256	pool_put(pp, cm);
257}
258
259struct cpumem *
260cpumem_malloc(size_t sz, int type)
261{
262	return (malloc(sz, type, M_WAITOK | M_ZERO));
263}
264
265struct cpumem *
266cpumem_malloc_ncpus(struct cpumem *cm, size_t sz, int type)
267{
268	return (cm);
269}
270
271void
272cpumem_free(struct cpumem *cm, int type, size_t sz)
273{
274	free(cm, type, sz);
275}
276
277void *
278cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
279{
280	return (cm);
281}
282
283void *
284cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
285{
286	return (NULL);
287}
288
289struct cpumem *
290counters_alloc(unsigned int n)
291{
292	KASSERT(n > 0);
293
294	return (cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS));
295}
296
297struct cpumem *
298counters_alloc_ncpus(struct cpumem *cm, unsigned int n)
299{
300	/* this is unnecessary, but symmetrical */
301	return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS));
302}
303
304void
305counters_free(struct cpumem *cm, unsigned int n)
306{
307	cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t));
308}
309
310void
311counters_read(struct cpumem *cm, uint64_t *output, unsigned int n,
312    uint64_t *scratch)
313{
314	uint64_t *counters;
315	unsigned int i;
316	int s;
317
318	counters = (uint64_t *)cm;
319
320	s = splhigh();
321	for (i = 0; i < n; i++)
322		output[i] = counters[i];
323	splx(s);
324}
325
326void
327counters_zero(struct cpumem *cm, unsigned int n)
328{
329	uint64_t *counters;
330	unsigned int i;
331	int s;
332
333	counters = (uint64_t *)cm;
334
335	s = splhigh();
336	for (i = 0; i < n; i++)
337		counters[i] = 0;
338	splx(s);
339}
340
341#endif /* MULTIPROCESSOR */
342