subr_pcpu.c revision 194784
1169689Skan/*- 2169689Skan * Copyright (c) 2001 Wind River Systems, Inc. 3169689Skan * All rights reserved. 4169689Skan * Written by: John Baldwin <jhb@FreeBSD.org> 5169689Skan * 6169689Skan * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 7169689Skan * All rights reserved. 8169689Skan * 9169689Skan * Redistribution and use in source and binary forms, with or without 10169689Skan * modification, are permitted provided that the following conditions 11169689Skan * are met: 12169689Skan * 1. Redistributions of source code must retain the above copyright 13169689Skan * notice, this list of conditions and the following disclaimer. 14169689Skan * 2. Redistributions in binary form must reproduce the above copyright 15169689Skan * notice, this list of conditions and the following disclaimer in the 16169689Skan * documentation and/or other materials provided with the distribution. 17169689Skan * 4. Neither the name of the author nor the names of any co-contributors 18169689Skan * may be used to endorse or promote products derived from this software 19169689Skan * without specific prior written permission. 20169689Skan * 21169689Skan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22169689Skan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23169689Skan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24169689Skan * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34/* 35 * This module provides MI support for per-cpu data. 36 * 37 * Each architecture determines the mapping of logical CPU IDs to physical 38 * CPUs. The requirements of this mapping are as follows: 39 * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. 40 * - The mapping is not required to be dense. That is, there may be 41 * gaps in the mappings. 42 * - The platform sets the value of MAXCPU in <machine/param.h>. 43 * - It is suggested, but not required, that in the non-SMP case, the 44 * platform define MAXCPU to be 1 and define the logical ID of the 45 * sole CPU as 0. 46 */ 47 48#include <sys/cdefs.h> 49__FBSDID("$FreeBSD: head/sys/kern/subr_pcpu.c 194784 2009-06-23 22:42:39Z jeff $"); 50 51#include "opt_ddb.h" 52 53#include <sys/param.h> 54#include <sys/systm.h> 55#include <sys/sysctl.h> 56#include <sys/linker_set.h> 57#include <sys/lock.h> 58#include <sys/malloc.h> 59#include <sys/pcpu.h> 60#include <sys/proc.h> 61#include <sys/smp.h> 62#include <sys/sx.h> 63#include <ddb/ddb.h> 64 65MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); 66 67struct dpcpu_free { 68 uintptr_t df_start; 69 int df_len; 70 TAILQ_ENTRY(dpcpu_free) df_link; 71}; 72 73static DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); 74static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); 75static struct sx dpcpu_lock; 76uintptr_t dpcpu_off[MAXCPU]; 77struct pcpu *cpuid_to_pcpu[MAXCPU]; 78struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead); 79 80/* 81 * Initialize the MI portions of a struct pcpu. 82 */ 83void 84pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 85{ 86 87 bzero(pcpu, size); 88 KASSERT(cpuid >= 0 && cpuid < MAXCPU, 89 ("pcpu_init: invalid cpuid %d", cpuid)); 90 pcpu->pc_cpuid = cpuid; 91 pcpu->pc_cpumask = 1 << cpuid; 92 cpuid_to_pcpu[cpuid] = pcpu; 93 SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); 94 cpu_pcpu_init(pcpu, cpuid, size); 95 pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; 96 pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; 97#ifdef KTR 98 snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid); 99#endif 100} 101 102void 103dpcpu_init(void *dpcpu, int cpuid) 104{ 105 struct pcpu *pcpu; 106 107 pcpu = pcpu_find(cpuid); 108 pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; 109 110 /* 111 * Initialize defaults from our linker section. 112 */ 113 memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); 114 115 /* 116 * Place it in the global pcpu offset array. 117 */ 118 dpcpu_off[cpuid] = pcpu->pc_dynamic; 119} 120 121static void 122dpcpu_startup(void *dummy __unused) 123{ 124 struct dpcpu_free *df; 125 126 df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 127 df->df_start = (uintptr_t)&DPCPU_NAME(modspace); 128 df->df_len = DPCPU_MODSIZE; 129 TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); 130 sx_init(&dpcpu_lock, "dpcpu alloc lock"); 131} 132SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); 133 134/* 135 * First-fit extent based allocator for allocating space in the per-cpu 136 * region reserved for modules. This is only intended for use by the 137 * kernel linkers to place module linker sets. 138 */ 139void * 140dpcpu_alloc(int size) 141{ 142 struct dpcpu_free *df; 143 void *s; 144 145 s = NULL; 146 size = roundup2(size, sizeof(void *)); 147 sx_xlock(&dpcpu_lock); 148 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 149 if (df->df_len < size) 150 continue; 151 if (df->df_len == size) { 152 s = (void *)df->df_start; 153 TAILQ_REMOVE(&dpcpu_head, df, df_link); 154 free(df, M_PCPU); 155 break; 156 } 157 s = (void *)df->df_start; 158 df->df_len -= size; 159 df->df_start = df->df_start + size; 160 break; 161 } 162 sx_xunlock(&dpcpu_lock); 163 164 return (s); 165} 166 167/* 168 * Free dynamic per-cpu space at module unload time. 169 */ 170void 171dpcpu_free(void *s, int size) 172{ 173 struct dpcpu_free *df; 174 struct dpcpu_free *dn; 175 uintptr_t start; 176 uintptr_t end; 177 178 size = roundup2(size, sizeof(void *)); 179 start = (uintptr_t)s; 180 end = start + size; 181 /* 182 * Free a region of space and merge it with as many neighbors as 183 * possible. Keeping the list sorted simplifies this operation. 184 */ 185 sx_xlock(&dpcpu_lock); 186 TAILQ_FOREACH(df, &dpcpu_head, df_link) { 187 if (df->df_start > end) 188 break; 189 /* 190 * If we expand at the end of an entry we may have to 191 * merge it with the one following it as well. 192 */ 193 if (df->df_start + df->df_len == start) { 194 df->df_len += size; 195 dn = TAILQ_NEXT(df, df_link); 196 if (df->df_start + df->df_len == dn->df_start) { 197 df->df_len += dn->df_len; 198 TAILQ_REMOVE(&dpcpu_head, dn, df_link); 199 free(dn, M_PCPU); 200 } 201 sx_xunlock(&dpcpu_lock); 202 return; 203 } 204 if (df->df_start == end) { 205 df->df_start = start; 206 df->df_len += size; 207 sx_xunlock(&dpcpu_lock); 208 return; 209 } 210 } 211 dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 212 dn->df_start = start; 213 dn->df_len = size; 214 if (df) 215 TAILQ_INSERT_BEFORE(df, dn, df_link); 216 else 217 TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); 218 sx_xunlock(&dpcpu_lock); 219} 220 221/* 222 * Initialize the per-cpu storage from an updated linker-set region. 223 */ 224void 225dpcpu_copy(void *s, int size) 226{ 227#ifdef SMP 228 uintptr_t dpcpu; 229 int i; 230 231 for (i = 0; i < mp_ncpus; ++i) { 232 dpcpu = dpcpu_off[i]; 233 if (dpcpu == 0) 234 continue; 235 memcpy((void *)(dpcpu + (uintptr_t)s), s, size); 236 } 237#else 238 memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); 239#endif 240} 241 242/* 243 * Destroy a struct pcpu. 244 */ 245void 246pcpu_destroy(struct pcpu *pcpu) 247{ 248 249 SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); 250 cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; 251 dpcpu_off[pcpu->pc_cpuid] = 0; 252} 253 254/* 255 * Locate a struct pcpu by cpu id. 256 */ 257struct pcpu * 258pcpu_find(u_int cpuid) 259{ 260 261 return (cpuid_to_pcpu[cpuid]); 262} 263 264int 265sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) 266{ 267 int64_t count; 268#ifdef SMP 269 uintptr_t dpcpu; 270 int i; 271 272 count = 0; 273 for (i = 0; i < mp_ncpus; ++i) { 274 dpcpu = dpcpu_off[i]; 275 if (dpcpu == 0) 276 continue; 277 count += *(int64_t *)(dpcpu + (uintptr_t)arg1); 278 } 279#else 280 count = *(int64_t *)(dpcpu_off[0] + (uintptr_t)arg1); 281#endif 282 return (SYSCTL_OUT(req, &count, sizeof(count))); 283} 284 285int 286sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) 287{ 288 int count; 289#ifdef SMP 290 uintptr_t dpcpu; 291 int i; 292 293 count = 0; 294 for (i = 0; i < mp_ncpus; ++i) { 295 dpcpu = dpcpu_off[i]; 296 if (dpcpu == 0) 297 continue; 298 count += *(int *)(dpcpu + (uintptr_t)arg1); 299 } 300#else 301 count = *(int *)(dpcpu_off[0] + (uintptr_t)arg1); 302#endif 303 return (SYSCTL_OUT(req, &count, sizeof(count))); 304} 305 306#ifdef DDB 307 308static void 309show_pcpu(struct pcpu *pc) 310{ 311 struct thread *td; 312 313 db_printf("cpuid = %d\n", pc->pc_cpuid); 314 db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); 315 db_printf("curthread = "); 316 td = pc->pc_curthread; 317 if (td != NULL) 318 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 319 td->td_name); 320 else 321 db_printf("none\n"); 322 db_printf("curpcb = %p\n", pc->pc_curpcb); 323 db_printf("fpcurthread = "); 324 td = pc->pc_fpcurthread; 325 if (td != NULL) 326 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 327 td->td_name); 328 else 329 db_printf("none\n"); 330 db_printf("idlethread = "); 331 td = pc->pc_idlethread; 332 if (td != NULL) 333 db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 334 td->td_name); 335 else 336 db_printf("none\n"); 337 db_show_mdpcpu(pc); 338 339#ifdef VIMAGE 340 db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); 341#endif 342 343#ifdef WITNESS 344 db_printf("spin locks held:\n"); 345 witness_list_locks(&pc->pc_spinlocks); 346#endif 347} 348 349DB_SHOW_COMMAND(pcpu, db_show_pcpu) 350{ 351 struct pcpu *pc; 352 int id; 353 354 if (have_addr) 355 id = ((addr >> 4) % 16) * 10 + (addr % 16); 356 else 357 id = PCPU_GET(cpuid); 358 pc = pcpu_find(id); 359 if (pc == NULL) { 360 db_printf("CPU %d not found\n", id); 361 return; 362 } 363 show_pcpu(pc); 364} 365 366DB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) 367{ 368 struct pcpu *pc; 369 int id; 370 371 db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); 372 for (id = 0; id <= mp_maxid; id++) { 373 pc = pcpu_find(id); 374 if (pc != NULL) { 375 show_pcpu(pc); 376 db_printf("\n"); 377 } 378 } 379} 380DB_SHOW_ALIAS(allpcpu, db_show_cpu_all); 381#endif 382