subr_pcpu.c revision 207929
1139804Simp/*- 276440Sjhb * Copyright (c) 2001 Wind River Systems, Inc. 376440Sjhb * All rights reserved. 476440Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 576440Sjhb * 6194784Sjeff * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 7194784Sjeff * All rights reserved. 8194784Sjeff * 976440Sjhb * Redistribution and use in source and binary forms, with or without 1076440Sjhb * modification, are permitted provided that the following conditions 1176440Sjhb * are met: 1276440Sjhb * 1. Redistributions of source code must retain the above copyright 1376440Sjhb * notice, this list of conditions and the following disclaimer. 1476440Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1576440Sjhb * notice, this list of conditions and the following disclaimer in the 1676440Sjhb * documentation and/or other materials provided with the distribution. 1776440Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1876440Sjhb * may be used to endorse or promote products derived from this software 1976440Sjhb * without specific prior written permission. 2076440Sjhb * 2176440Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2276440Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2376440Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2476440Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2576440Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2676440Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2776440Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2876440Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2976440Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3076440Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3176440Sjhb * SUCH DAMAGE. 3276440Sjhb */ 3376440Sjhb 3476440Sjhb/* 3576440Sjhb * This module provides MI support for per-cpu data. 3685444Sjhb * 3785444Sjhb * Each architecture determines the mapping of logical CPU IDs to physical 3885444Sjhb * CPUs. The requirements of this mapping are as follows: 3985444Sjhb * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. 4085444Sjhb * - The mapping is not required to be dense. That is, there may be 4185444Sjhb * gaps in the mappings. 4285444Sjhb * - The platform sets the value of MAXCPU in <machine/param.h>. 4385444Sjhb * - It is suggested, but not required, that in the non-SMP case, the 4485444Sjhb * platform define MAXCPU to be 1 and define the logical ID of the 4585444Sjhb * sole CPU as 0. 4676440Sjhb */ 4776440Sjhb 48116182Sobrien#include <sys/cdefs.h> 49116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/subr_pcpu.c 207929 2010-05-11 18:24:22Z attilio $"); 50116182Sobrien 5187702Sjhb#include "opt_ddb.h" 5287702Sjhb 5376440Sjhb#include <sys/param.h> 5476440Sjhb#include <sys/systm.h> 55194784Sjeff#include <sys/sysctl.h> 5687702Sjhb#include <sys/linker_set.h> 5787702Sjhb#include <sys/lock.h> 58194784Sjeff#include <sys/malloc.h> 5976440Sjhb#include <sys/pcpu.h> 6087702Sjhb#include <sys/proc.h> 61150576Srwatson#include <sys/smp.h> 62194784Sjeff#include <sys/sx.h> 6387702Sjhb#include <ddb/ddb.h> 6476440Sjhb 65194784SjeffMALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); 66194784Sjeff 67194784Sjeffstruct dpcpu_free { 68194784Sjeff uintptr_t df_start; 69194784Sjeff int df_len; 70194784Sjeff TAILQ_ENTRY(dpcpu_free) df_link; 71194784Sjeff}; 72194784Sjeff 73194784Sjeffstatic DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); 74194784Sjeffstatic TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); 75194784Sjeffstatic struct sx dpcpu_lock; 76194784Sjeffuintptr_t dpcpu_off[MAXCPU]; 77173444Supsstruct pcpu *cpuid_to_pcpu[MAXCPU]; 7876440Sjhbstruct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead); 7976440Sjhb 8076440Sjhb/* 8187702Sjhb * Initialize the MI portions of a struct pcpu. 8276440Sjhb */ 8376440Sjhbvoid 8487702Sjhbpcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 8576440Sjhb{ 8676440Sjhb 8787702Sjhb bzero(pcpu, size); 8887702Sjhb KASSERT(cpuid >= 0 && cpuid < MAXCPU, 8987702Sjhb ("pcpu_init: invalid cpuid %d", cpuid)); 9087702Sjhb pcpu->pc_cpuid = cpuid; 9188901Speter pcpu->pc_cpumask = 1 << cpuid; 9287702Sjhb cpuid_to_pcpu[cpuid] = pcpu; 9387702Sjhb SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); 9487702Sjhb cpu_pcpu_init(pcpu, cpuid, size); 95173444Sups pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; 96173444Sups pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; 97187357Sjeff#ifdef KTR 98187357Sjeff snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid); 99187357Sjeff#endif 100194784Sjeff} 101173444Sups 102194784Sjeffvoid 103194784Sjeffdpcpu_init(void *dpcpu, int cpuid) 104194784Sjeff{ 105194784Sjeff struct pcpu *pcpu; 106194784Sjeff 107194784Sjeff pcpu = pcpu_find(cpuid); 108194784Sjeff pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; 109194784Sjeff 110194784Sjeff /* 111194784Sjeff * Initialize defaults from our linker section. 112194784Sjeff */ 113194784Sjeff memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); 114194784Sjeff 115194784Sjeff /* 116194784Sjeff * Place it in the global pcpu offset array. 117194784Sjeff */ 118194784Sjeff dpcpu_off[cpuid] = pcpu->pc_dynamic; 11976440Sjhb} 12076440Sjhb 121194784Sjeffstatic void 122194784Sjeffdpcpu_startup(void *dummy __unused) 123194784Sjeff{ 124194784Sjeff struct dpcpu_free *df; 125194784Sjeff 126194784Sjeff df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 127194784Sjeff df->df_start = (uintptr_t)&DPCPU_NAME(modspace); 128194784Sjeff df->df_len = DPCPU_MODSIZE; 129194784Sjeff TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); 130194784Sjeff sx_init(&dpcpu_lock, "dpcpu alloc lock"); 131194784Sjeff} 132194784SjeffSYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); 133194784Sjeff 13476440Sjhb/* 135194784Sjeff * First-fit extent based allocator for allocating space in the per-cpu 136194784Sjeff * region reserved for modules. This is only intended for use by the 137194784Sjeff * kernel linkers to place module linker sets. 138194784Sjeff */ 139194784Sjeffvoid * 140194784Sjeffdpcpu_alloc(int size) 141194784Sjeff{ 142194784Sjeff struct dpcpu_free *df; 143194784Sjeff void *s; 144194784Sjeff 145194784Sjeff s = NULL; 146194784Sjeff size = roundup2(size, sizeof(void *)); 147194784Sjeff sx_xlock(&dpcpu_lock); 148194784Sjeff TAILQ_FOREACH(df, &dpcpu_head, df_link) { 149194784Sjeff if (df->df_len < size) 150194784Sjeff continue; 151194784Sjeff if (df->df_len == size) { 152194784Sjeff s = (void *)df->df_start; 153194784Sjeff TAILQ_REMOVE(&dpcpu_head, df, df_link); 154194784Sjeff free(df, M_PCPU); 155194784Sjeff break; 156194784Sjeff } 157194784Sjeff s = (void *)df->df_start; 158194784Sjeff df->df_len -= size; 159194784Sjeff df->df_start = df->df_start + size; 160194784Sjeff break; 161194784Sjeff } 162194784Sjeff sx_xunlock(&dpcpu_lock); 163194784Sjeff 164194784Sjeff return (s); 165194784Sjeff} 166194784Sjeff 167194784Sjeff/* 168194784Sjeff * Free dynamic per-cpu space at module unload time. 169194784Sjeff */ 170194784Sjeffvoid 171194784Sjeffdpcpu_free(void *s, int size) 172194784Sjeff{ 173194784Sjeff struct dpcpu_free *df; 174194784Sjeff struct dpcpu_free *dn; 175194784Sjeff uintptr_t start; 176194784Sjeff uintptr_t end; 177194784Sjeff 178194784Sjeff size = roundup2(size, sizeof(void *)); 179194784Sjeff start = (uintptr_t)s; 180194784Sjeff end = start + size; 181194784Sjeff /* 182194784Sjeff * Free a region of space and merge it with as many neighbors as 183194784Sjeff * possible. Keeping the list sorted simplifies this operation. 184194784Sjeff */ 185194784Sjeff sx_xlock(&dpcpu_lock); 186194784Sjeff TAILQ_FOREACH(df, &dpcpu_head, df_link) { 187194784Sjeff if (df->df_start > end) 188194784Sjeff break; 189194784Sjeff /* 190194784Sjeff * If we expand at the end of an entry we may have to 191194784Sjeff * merge it with the one following it as well. 192194784Sjeff */ 193194784Sjeff if (df->df_start + df->df_len == start) { 194194784Sjeff df->df_len += size; 195194784Sjeff dn = TAILQ_NEXT(df, df_link); 196194784Sjeff if (df->df_start + df->df_len == dn->df_start) { 197194784Sjeff df->df_len += dn->df_len; 198194784Sjeff TAILQ_REMOVE(&dpcpu_head, dn, df_link); 199194784Sjeff free(dn, M_PCPU); 200194784Sjeff } 201194784Sjeff sx_xunlock(&dpcpu_lock); 202194784Sjeff return; 203194784Sjeff } 204194784Sjeff if (df->df_start == end) { 205194784Sjeff df->df_start = start; 206194784Sjeff df->df_len += size; 207194784Sjeff sx_xunlock(&dpcpu_lock); 208194784Sjeff return; 209194784Sjeff } 210194784Sjeff } 211194784Sjeff dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 212194784Sjeff dn->df_start = start; 213194784Sjeff dn->df_len = size; 214194784Sjeff if (df) 215194784Sjeff TAILQ_INSERT_BEFORE(df, dn, df_link); 216194784Sjeff else 217194784Sjeff TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); 218194784Sjeff sx_xunlock(&dpcpu_lock); 219194784Sjeff} 220194784Sjeff 221194784Sjeff/* 222194784Sjeff * Initialize the per-cpu storage from an updated linker-set region. 223194784Sjeff */ 224194784Sjeffvoid 225194784Sjeffdpcpu_copy(void *s, int size) 226194784Sjeff{ 227194784Sjeff#ifdef SMP 228194784Sjeff uintptr_t dpcpu; 229194784Sjeff int i; 230194784Sjeff 231194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 232194784Sjeff dpcpu = dpcpu_off[i]; 233194784Sjeff if (dpcpu == 0) 234194784Sjeff continue; 235194784Sjeff memcpy((void *)(dpcpu + (uintptr_t)s), s, size); 236194784Sjeff } 237194784Sjeff#else 238194784Sjeff memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); 239194784Sjeff#endif 240194784Sjeff} 241194784Sjeff 242194784Sjeff/* 24387702Sjhb * Destroy a struct pcpu. 24476440Sjhb */ 24587702Sjhbvoid 24687702Sjhbpcpu_destroy(struct pcpu *pcpu) 24776440Sjhb{ 24876440Sjhb 24987702Sjhb SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); 25087702Sjhb cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; 251194784Sjeff dpcpu_off[pcpu->pc_cpuid] = 0; 25276440Sjhb} 25387702Sjhb 25487702Sjhb/* 25587702Sjhb * Locate a struct pcpu by cpu id. 25687702Sjhb */ 25787702Sjhbstruct pcpu * 25887702Sjhbpcpu_find(u_int cpuid) 25987702Sjhb{ 26087702Sjhb 26187702Sjhb return (cpuid_to_pcpu[cpuid]); 26287702Sjhb} 26387702Sjhb 264194784Sjeffint 265194784Sjeffsysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) 266194784Sjeff{ 267194935Sjeff uintptr_t dpcpu; 268194784Sjeff int64_t count; 269194784Sjeff int i; 270194784Sjeff 271194784Sjeff count = 0; 272194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 273194784Sjeff dpcpu = dpcpu_off[i]; 274194784Sjeff if (dpcpu == 0) 275194784Sjeff continue; 276194784Sjeff count += *(int64_t *)(dpcpu + (uintptr_t)arg1); 277194784Sjeff } 278194784Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 279194784Sjeff} 280194784Sjeff 281194784Sjeffint 282194935Sjeffsysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) 283194935Sjeff{ 284194935Sjeff uintptr_t dpcpu; 285194935Sjeff long count; 286194935Sjeff int i; 287194935Sjeff 288194935Sjeff count = 0; 289194935Sjeff for (i = 0; i < mp_ncpus; ++i) { 290194935Sjeff dpcpu = dpcpu_off[i]; 291194935Sjeff if (dpcpu == 0) 292194935Sjeff continue; 293194935Sjeff count += *(long *)(dpcpu + (uintptr_t)arg1); 294194935Sjeff } 295194935Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 296194935Sjeff} 297194935Sjeff 298194935Sjeffint 299194784Sjeffsysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) 300194784Sjeff{ 301194935Sjeff uintptr_t dpcpu; 302194784Sjeff int count; 303194784Sjeff int i; 304194784Sjeff 305194784Sjeff count = 0; 306194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 307194784Sjeff dpcpu = dpcpu_off[i]; 308194784Sjeff if (dpcpu == 0) 309194784Sjeff continue; 310194784Sjeff count += *(int *)(dpcpu + (uintptr_t)arg1); 311194784Sjeff } 312194784Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 313194784Sjeff} 314194784Sjeff 31587702Sjhb#ifdef DDB 316196132SbzDB_SHOW_COMMAND(dpcpu_off, db_show_dpcpu_off) 317196132Sbz{ 318196132Sbz int id; 319150576Srwatson 320196132Sbz for (id = 0; id <= mp_maxid; id++) { 321196132Sbz if (CPU_ABSENT(id)) 322196132Sbz continue; 323196132Sbz db_printf("dpcpu_off[%2d] = 0x%jx (+ DPCPU_START = %p)\n", 324196132Sbz id, (uintmax_t)dpcpu_off[id], 325196132Sbz (void *)(uintptr_t)(dpcpu_off[id] + DPCPU_START)); 326196132Sbz } 327196132Sbz} 328196132Sbz 329150576Srwatsonstatic void 330150576Srwatsonshow_pcpu(struct pcpu *pc) 33187702Sjhb{ 33287702Sjhb struct thread *td; 33387702Sjhb 33487702Sjhb db_printf("cpuid = %d\n", pc->pc_cpuid); 335194784Sjeff db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); 33687702Sjhb db_printf("curthread = "); 33787702Sjhb td = pc->pc_curthread; 33887702Sjhb if (td != NULL) 33987702Sjhb db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 340173600Sjulian td->td_name); 34187702Sjhb else 34287702Sjhb db_printf("none\n"); 34387702Sjhb db_printf("curpcb = %p\n", pc->pc_curpcb); 34487702Sjhb db_printf("fpcurthread = "); 34587702Sjhb td = pc->pc_fpcurthread; 34687702Sjhb if (td != NULL) 34787702Sjhb db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 348173600Sjulian td->td_name); 34987702Sjhb else 35087702Sjhb db_printf("none\n"); 35187702Sjhb db_printf("idlethread = "); 35287702Sjhb td = pc->pc_idlethread; 35387702Sjhb if (td != NULL) 35487702Sjhb db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 355173600Sjulian td->td_name); 35687702Sjhb else 35787702Sjhb db_printf("none\n"); 35887702Sjhb db_show_mdpcpu(pc); 35987702Sjhb 360191816Szec#ifdef VIMAGE 361191816Szec db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); 362191816Szec#endif 363191816Szec 36487702Sjhb#ifdef WITNESS 36587702Sjhb db_printf("spin locks held:\n"); 366207929Sattilio witness_list_locks(&pc->pc_spinlocks, db_printf); 36787702Sjhb#endif 36887702Sjhb} 369150576Srwatson 370150576SrwatsonDB_SHOW_COMMAND(pcpu, db_show_pcpu) 371150576Srwatson{ 372150576Srwatson struct pcpu *pc; 373150576Srwatson int id; 374150576Srwatson 375150576Srwatson if (have_addr) 376150576Srwatson id = ((addr >> 4) % 16) * 10 + (addr % 16); 377150576Srwatson else 378150576Srwatson id = PCPU_GET(cpuid); 379150576Srwatson pc = pcpu_find(id); 380150576Srwatson if (pc == NULL) { 381150576Srwatson db_printf("CPU %d not found\n", id); 382150576Srwatson return; 383150576Srwatson } 384150576Srwatson show_pcpu(pc); 385150576Srwatson} 386150576Srwatson 387183054SsamDB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) 388150576Srwatson{ 389150576Srwatson struct pcpu *pc; 390150576Srwatson int id; 391150576Srwatson 392150576Srwatson db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); 393152021Sjhb for (id = 0; id <= mp_maxid; id++) { 394150576Srwatson pc = pcpu_find(id); 395150576Srwatson if (pc != NULL) { 396150576Srwatson show_pcpu(pc); 397150576Srwatson db_printf("\n"); 398150576Srwatson } 399150576Srwatson } 400150576Srwatson} 401183054SsamDB_SHOW_ALIAS(allpcpu, db_show_cpu_all); 40287702Sjhb#endif 403