1/* $OpenBSD: subr_percpu.c,v 1.11 2023/09/16 09:33:27 mpi Exp $ */ 2 3/* 4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include <sys/param.h> 20#include <sys/systm.h> 21#include <sys/pool.h> 22#include <sys/malloc.h> 23 24#include <sys/percpu.h> 25 26#ifdef MULTIPROCESSOR 27struct pool cpumem_pl; 28 29void 30percpu_init(void) 31{ 32 pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpusfound, 0, 33 IPL_NONE, PR_WAITOK, "percpumem", &pool_allocator_single); 34} 35 36struct cpumem * 37cpumem_get(struct pool *pp) 38{ 39 struct cpumem *cm; 40 unsigned int cpu; 41 42 cm = pool_get(&cpumem_pl, PR_WAITOK); 43 44 for (cpu = 0; cpu < ncpusfound; cpu++) 45 cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO); 46 47 return (cm); 48} 49 50void 51cpumem_put(struct pool *pp, struct cpumem *cm) 52{ 53 unsigned int cpu; 54 55 for (cpu = 0; cpu < ncpusfound; cpu++) 56 pool_put(pp, cm[cpu].mem); 57 58 pool_put(&cpumem_pl, cm); 59} 60 61struct cpumem * 62cpumem_malloc(size_t sz, int type) 63{ 64 struct cpumem *cm; 65 unsigned int cpu; 66 67 sz = roundup(sz, CACHELINESIZE); 68 69 cm = pool_get(&cpumem_pl, PR_WAITOK); 70 71 for (cpu = 0; cpu < ncpusfound; cpu++) 72 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 73 74 return (cm); 75} 76 77struct cpumem * 78cpumem_malloc_ncpus(struct cpumem *bootcm, size_t sz, int type) 79{ 80 struct cpumem *cm; 81 unsigned int cpu; 82 83 sz = roundup(sz, CACHELINESIZE); 84 85 cm = pool_get(&cpumem_pl, PR_WAITOK); 86 87 cm[0].mem = bootcm[0].mem; 88 for (cpu = 1; cpu < ncpusfound; cpu++) 89 cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO); 90 91 return (cm); 92} 93 94void 95cpumem_free(struct cpumem *cm, int type, size_t sz) 96{ 97 unsigned int cpu; 98 99 sz = roundup(sz, CACHELINESIZE); 100 101 for (cpu = 0; cpu < ncpusfound; cpu++) 102 free(cm[cpu].mem, type, sz); 103 104 pool_put(&cpumem_pl, cm); 105} 106 107void * 108cpumem_first(struct cpumem_iter *i, struct cpumem *cm) 109{ 110 i->cpu = 0; 111 112 return (cm[0].mem); 113} 114 115void * 116cpumem_next(struct cpumem_iter *i, struct cpumem *cm) 117{ 118 unsigned int cpu = ++i->cpu; 119 120 if (cpu >= ncpusfound) 121 return (NULL); 122 123 return (cm[cpu].mem); 124} 125 126struct cpumem * 127counters_alloc(unsigned int n) 128{ 129 struct cpumem *cm; 130 struct cpumem_iter cmi; 131 uint64_t *counters; 132 unsigned int i; 133 134 KASSERT(n > 0); 135 136 n++; /* add space for a generation number */ 137 cm = cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS); 138 139 CPUMEM_FOREACH(counters, &cmi, cm) { 140 for (i = 0; i < n; i++) 141 counters[i] = 0; 142 } 143 144 return (cm); 145} 146 147struct cpumem * 148counters_alloc_ncpus(struct cpumem *cm, unsigned int n) 149{ 150 n++; /* the generation number */ 151 return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS)); 152} 153 154void 155counters_free(struct cpumem *cm, unsigned int n) 156{ 157 n++; /* generation number */ 158 cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t)); 159} 160 161void 162counters_read(struct cpumem *cm, uint64_t *output, unsigned int n, 163 uint64_t *scratch) 164{ 165 struct cpumem_iter cmi; 166 uint64_t *gen, *counters, *temp = scratch; 167 uint64_t enter, leave; 168 unsigned int i; 169 170 for (i = 0; i < n; i++) 171 output[i] = 0; 172 173 if (scratch == NULL) 174 temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK); 175 176 gen = cpumem_first(&cmi, cm); 177 do { 178 counters = gen + 1; 179 180 enter = *gen; 181 for (;;) { 182 /* the generation number is odd during an update */ 183 while (enter & 1) { 184 yield(); 185 enter = *gen; 186 } 187 188 membar_consumer(); 189 for (i = 0; i < n; i++) 190 temp[i] = counters[i]; 191 192 membar_consumer(); 193 leave = *gen; 194 195 if (enter == leave) 196 break; 197 198 enter = leave; 199 } 200 201 for (i = 0; i < n; i++) 202 output[i] += temp[i]; 203 204 gen = cpumem_next(&cmi, cm); 205 } while (gen != NULL); 206 207 if (scratch == NULL) 208 free(temp, M_TEMP, n * sizeof(uint64_t)); 209} 210 211void 212counters_zero(struct cpumem *cm, unsigned int n) 213{ 214 struct cpumem_iter cmi; 215 uint64_t *counters; 216 unsigned int i; 217 218 counters = cpumem_first(&cmi, cm); 219 membar_producer(); 220 do { 221 for (i = 0; i < n; i++) 222 counters[i] = 0; 223 /* zero the generation numbers too */ 224 membar_producer(); 225 counters[i] = 0; 226 227 counters = cpumem_next(&cmi, cm); 228 } while (counters != NULL); 229} 230 231#else /* MULTIPROCESSOR */ 232 233/* 234 * Uniprocessor implementation of per-CPU data structures. 235 * 236 * UP percpu memory is a single memory allocation cast to/from the 237 * cpumem struct. It is not scaled up to the size of cacheline because 238 * there's no other cache to contend with. 239 */ 240 241void 242percpu_init(void) 243{ 244 /* nop */ 245} 246 247struct cpumem * 248cpumem_get(struct pool *pp) 249{ 250 return (pool_get(pp, PR_WAITOK | PR_ZERO)); 251} 252 253void 254cpumem_put(struct pool *pp, struct cpumem *cm) 255{ 256 pool_put(pp, cm); 257} 258 259struct cpumem * 260cpumem_malloc(size_t sz, int type) 261{ 262 return (malloc(sz, type, M_WAITOK | M_ZERO)); 263} 264 265struct cpumem * 266cpumem_malloc_ncpus(struct cpumem *cm, size_t sz, int type) 267{ 268 return (cm); 269} 270 271void 272cpumem_free(struct cpumem *cm, int type, size_t sz) 273{ 274 free(cm, type, sz); 275} 276 277void * 278cpumem_first(struct cpumem_iter *i, struct cpumem *cm) 279{ 280 return (cm); 281} 282 283void * 284cpumem_next(struct cpumem_iter *i, struct cpumem *cm) 285{ 286 return (NULL); 287} 288 289struct cpumem * 290counters_alloc(unsigned int n) 291{ 292 KASSERT(n > 0); 293 294 return (cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS)); 295} 296 297struct cpumem * 298counters_alloc_ncpus(struct cpumem *cm, unsigned int n) 299{ 300 /* this is unnecessary, but symmetrical */ 301 return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS)); 302} 303 304void 305counters_free(struct cpumem *cm, unsigned int n) 306{ 307 cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t)); 308} 309 310void 311counters_read(struct cpumem *cm, uint64_t *output, unsigned int n, 312 uint64_t *scratch) 313{ 314 uint64_t *counters; 315 unsigned int i; 316 int s; 317 318 counters = (uint64_t *)cm; 319 320 s = splhigh(); 321 for (i = 0; i < n; i++) 322 output[i] = counters[i]; 323 splx(s); 324} 325 326void 327counters_zero(struct cpumem *cm, unsigned int n) 328{ 329 uint64_t *counters; 330 unsigned int i; 331 int s; 332 333 counters = (uint64_t *)cm; 334 335 s = splhigh(); 336 for (i = 0; i < n; i++) 337 counters[i] = 0; 338 splx(s); 339} 340 341#endif /* MULTIPROCESSOR */ 342