subr_pcpu.c revision 227293
1139804Simp/*- 276440Sjhb * Copyright (c) 2001 Wind River Systems, Inc. 376440Sjhb * All rights reserved. 476440Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 576440Sjhb * 6194784Sjeff * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 7194784Sjeff * All rights reserved. 8194784Sjeff * 976440Sjhb * Redistribution and use in source and binary forms, with or without 1076440Sjhb * modification, are permitted provided that the following conditions 1176440Sjhb * are met: 1276440Sjhb * 1. Redistributions of source code must retain the above copyright 1376440Sjhb * notice, this list of conditions and the following disclaimer. 1476440Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1576440Sjhb * notice, this list of conditions and the following disclaimer in the 1676440Sjhb * documentation and/or other materials provided with the distribution. 1776440Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1876440Sjhb * may be used to endorse or promote products derived from this software 1976440Sjhb * without specific prior written permission. 2076440Sjhb * 2176440Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2276440Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2376440Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2476440Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2576440Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2676440Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2776440Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2876440Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2976440Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3076440Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3176440Sjhb * SUCH DAMAGE. 3276440Sjhb */ 3376440Sjhb 3476440Sjhb/* 3576440Sjhb * This module provides MI support for per-cpu data. 3685444Sjhb * 3785444Sjhb * Each architecture determines the mapping of logical CPU IDs to physical 3885444Sjhb * CPUs. The requirements of this mapping are as follows: 3985444Sjhb * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. 4085444Sjhb * - The mapping is not required to be dense. That is, there may be 4185444Sjhb * gaps in the mappings. 4285444Sjhb * - The platform sets the value of MAXCPU in <machine/param.h>. 4385444Sjhb * - It is suggested, but not required, that in the non-SMP case, the 4485444Sjhb * platform define MAXCPU to be 1 and define the logical ID of the 4585444Sjhb * sole CPU as 0. 4676440Sjhb */ 4776440Sjhb 48116182Sobrien#include <sys/cdefs.h> 49116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/subr_pcpu.c 227293 2011-11-07 06:44:47Z ed $"); 50116182Sobrien 5187702Sjhb#include "opt_ddb.h" 5287702Sjhb 5376440Sjhb#include <sys/param.h> 5476440Sjhb#include <sys/systm.h> 55194784Sjeff#include <sys/sysctl.h> 5687702Sjhb#include <sys/lock.h> 57194784Sjeff#include <sys/malloc.h> 5876440Sjhb#include <sys/pcpu.h> 5987702Sjhb#include <sys/proc.h> 60150576Srwatson#include <sys/smp.h> 61194784Sjeff#include <sys/sx.h> 6287702Sjhb#include <ddb/ddb.h> 6376440Sjhb 64227293Sedstatic MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); 65194784Sjeff 66194784Sjeffstruct dpcpu_free { 67194784Sjeff uintptr_t df_start; 68194784Sjeff int df_len; 69194784Sjeff TAILQ_ENTRY(dpcpu_free) df_link; 70194784Sjeff}; 71194784Sjeff 72215701Sdimstatic DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); 73194784Sjeffstatic TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); 74194784Sjeffstatic struct sx dpcpu_lock; 75194784Sjeffuintptr_t dpcpu_off[MAXCPU]; 76173444Supsstruct pcpu *cpuid_to_pcpu[MAXCPU]; 77222531Snwhitehornstruct cpuhead cpuhead = STAILQ_HEAD_INITIALIZER(cpuhead); 7876440Sjhb 7976440Sjhb/* 8087702Sjhb * Initialize the MI portions of a struct pcpu. 8176440Sjhb */ 8276440Sjhbvoid 8387702Sjhbpcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 8476440Sjhb{ 8576440Sjhb 8687702Sjhb bzero(pcpu, size); 8787702Sjhb KASSERT(cpuid >= 0 && cpuid < MAXCPU, 8887702Sjhb ("pcpu_init: invalid cpuid %d", cpuid)); 8987702Sjhb pcpu->pc_cpuid = cpuid; 9087702Sjhb cpuid_to_pcpu[cpuid] = pcpu; 91222531Snwhitehorn STAILQ_INSERT_TAIL(&cpuhead, pcpu, pc_allcpu); 9287702Sjhb cpu_pcpu_init(pcpu, cpuid, size); 93173444Sups pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; 94173444Sups pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; 95194784Sjeff} 96173444Sups 97194784Sjeffvoid 98194784Sjeffdpcpu_init(void *dpcpu, int cpuid) 99194784Sjeff{ 100194784Sjeff struct pcpu *pcpu; 101194784Sjeff 102194784Sjeff pcpu = pcpu_find(cpuid); 103194784Sjeff pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; 104194784Sjeff 105194784Sjeff /* 106194784Sjeff * Initialize defaults from our linker section. 107194784Sjeff */ 108194784Sjeff memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); 109194784Sjeff 110194784Sjeff /* 111194784Sjeff * Place it in the global pcpu offset array. 112194784Sjeff */ 113194784Sjeff dpcpu_off[cpuid] = pcpu->pc_dynamic; 11476440Sjhb} 11576440Sjhb 116194784Sjeffstatic void 117194784Sjeffdpcpu_startup(void *dummy __unused) 118194784Sjeff{ 119194784Sjeff struct dpcpu_free *df; 120194784Sjeff 121194784Sjeff df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 122194784Sjeff df->df_start = (uintptr_t)&DPCPU_NAME(modspace); 123208100Sbz df->df_len = DPCPU_MODMIN; 124194784Sjeff TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); 125194784Sjeff sx_init(&dpcpu_lock, "dpcpu alloc lock"); 126194784Sjeff} 127194784SjeffSYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); 128194784Sjeff 12976440Sjhb/* 130194784Sjeff * First-fit extent based allocator for allocating space in the per-cpu 131194784Sjeff * region reserved for modules. This is only intended for use by the 132194784Sjeff * kernel linkers to place module linker sets. 133194784Sjeff */ 134194784Sjeffvoid * 135194784Sjeffdpcpu_alloc(int size) 136194784Sjeff{ 137194784Sjeff struct dpcpu_free *df; 138194784Sjeff void *s; 139194784Sjeff 140194784Sjeff s = NULL; 141194784Sjeff size = roundup2(size, sizeof(void *)); 142194784Sjeff sx_xlock(&dpcpu_lock); 143194784Sjeff TAILQ_FOREACH(df, &dpcpu_head, df_link) { 144194784Sjeff if (df->df_len < size) 145194784Sjeff continue; 146194784Sjeff if (df->df_len == size) { 147194784Sjeff s = (void *)df->df_start; 148194784Sjeff TAILQ_REMOVE(&dpcpu_head, df, df_link); 149194784Sjeff free(df, M_PCPU); 150194784Sjeff break; 151194784Sjeff } 152194784Sjeff s = (void *)df->df_start; 153194784Sjeff df->df_len -= size; 154194784Sjeff df->df_start = df->df_start + size; 155194784Sjeff break; 156194784Sjeff } 157194784Sjeff sx_xunlock(&dpcpu_lock); 158194784Sjeff 159194784Sjeff return (s); 160194784Sjeff} 161194784Sjeff 162194784Sjeff/* 163194784Sjeff * Free dynamic per-cpu space at module unload time. 164194784Sjeff */ 165194784Sjeffvoid 166194784Sjeffdpcpu_free(void *s, int size) 167194784Sjeff{ 168194784Sjeff struct dpcpu_free *df; 169194784Sjeff struct dpcpu_free *dn; 170194784Sjeff uintptr_t start; 171194784Sjeff uintptr_t end; 172194784Sjeff 173194784Sjeff size = roundup2(size, sizeof(void *)); 174194784Sjeff start = (uintptr_t)s; 175194784Sjeff end = start + size; 176194784Sjeff /* 177194784Sjeff * Free a region of space and merge it with as many neighbors as 178194784Sjeff * possible. Keeping the list sorted simplifies this operation. 179194784Sjeff */ 180194784Sjeff sx_xlock(&dpcpu_lock); 181194784Sjeff TAILQ_FOREACH(df, &dpcpu_head, df_link) { 182194784Sjeff if (df->df_start > end) 183194784Sjeff break; 184194784Sjeff /* 185194784Sjeff * If we expand at the end of an entry we may have to 186194784Sjeff * merge it with the one following it as well. 187194784Sjeff */ 188194784Sjeff if (df->df_start + df->df_len == start) { 189194784Sjeff df->df_len += size; 190194784Sjeff dn = TAILQ_NEXT(df, df_link); 191194784Sjeff if (df->df_start + df->df_len == dn->df_start) { 192194784Sjeff df->df_len += dn->df_len; 193194784Sjeff TAILQ_REMOVE(&dpcpu_head, dn, df_link); 194194784Sjeff free(dn, M_PCPU); 195194784Sjeff } 196194784Sjeff sx_xunlock(&dpcpu_lock); 197194784Sjeff return; 198194784Sjeff } 199194784Sjeff if (df->df_start == end) { 200194784Sjeff df->df_start = start; 201194784Sjeff df->df_len += size; 202194784Sjeff sx_xunlock(&dpcpu_lock); 203194784Sjeff return; 204194784Sjeff } 205194784Sjeff } 206194784Sjeff dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); 207194784Sjeff dn->df_start = start; 208194784Sjeff dn->df_len = size; 209194784Sjeff if (df) 210194784Sjeff TAILQ_INSERT_BEFORE(df, dn, df_link); 211194784Sjeff else 212194784Sjeff TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); 213194784Sjeff sx_xunlock(&dpcpu_lock); 214194784Sjeff} 215194784Sjeff 216194784Sjeff/* 217194784Sjeff * Initialize the per-cpu storage from an updated linker-set region. 218194784Sjeff */ 219194784Sjeffvoid 220194784Sjeffdpcpu_copy(void *s, int size) 221194784Sjeff{ 222194784Sjeff#ifdef SMP 223194784Sjeff uintptr_t dpcpu; 224194784Sjeff int i; 225194784Sjeff 226194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 227194784Sjeff dpcpu = dpcpu_off[i]; 228194784Sjeff if (dpcpu == 0) 229194784Sjeff continue; 230194784Sjeff memcpy((void *)(dpcpu + (uintptr_t)s), s, size); 231194784Sjeff } 232194784Sjeff#else 233194784Sjeff memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); 234194784Sjeff#endif 235194784Sjeff} 236194784Sjeff 237194784Sjeff/* 23887702Sjhb * Destroy a struct pcpu. 23976440Sjhb */ 24087702Sjhbvoid 24187702Sjhbpcpu_destroy(struct pcpu *pcpu) 24276440Sjhb{ 24376440Sjhb 244222531Snwhitehorn STAILQ_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); 24587702Sjhb cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; 246194784Sjeff dpcpu_off[pcpu->pc_cpuid] = 0; 24776440Sjhb} 24887702Sjhb 24987702Sjhb/* 25087702Sjhb * Locate a struct pcpu by cpu id. 25187702Sjhb */ 25287702Sjhbstruct pcpu * 25387702Sjhbpcpu_find(u_int cpuid) 25487702Sjhb{ 25587702Sjhb 25687702Sjhb return (cpuid_to_pcpu[cpuid]); 25787702Sjhb} 25887702Sjhb 259194784Sjeffint 260194784Sjeffsysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) 261194784Sjeff{ 262194935Sjeff uintptr_t dpcpu; 263194784Sjeff int64_t count; 264194784Sjeff int i; 265194784Sjeff 266194784Sjeff count = 0; 267194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 268194784Sjeff dpcpu = dpcpu_off[i]; 269194784Sjeff if (dpcpu == 0) 270194784Sjeff continue; 271194784Sjeff count += *(int64_t *)(dpcpu + (uintptr_t)arg1); 272194784Sjeff } 273194784Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 274194784Sjeff} 275194784Sjeff 276194784Sjeffint 277194935Sjeffsysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) 278194935Sjeff{ 279194935Sjeff uintptr_t dpcpu; 280194935Sjeff long count; 281194935Sjeff int i; 282194935Sjeff 283194935Sjeff count = 0; 284194935Sjeff for (i = 0; i < mp_ncpus; ++i) { 285194935Sjeff dpcpu = dpcpu_off[i]; 286194935Sjeff if (dpcpu == 0) 287194935Sjeff continue; 288194935Sjeff count += *(long *)(dpcpu + (uintptr_t)arg1); 289194935Sjeff } 290194935Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 291194935Sjeff} 292194935Sjeff 293194935Sjeffint 294194784Sjeffsysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) 295194784Sjeff{ 296194935Sjeff uintptr_t dpcpu; 297194784Sjeff int count; 298194784Sjeff int i; 299194784Sjeff 300194784Sjeff count = 0; 301194784Sjeff for (i = 0; i < mp_ncpus; ++i) { 302194784Sjeff dpcpu = dpcpu_off[i]; 303194784Sjeff if (dpcpu == 0) 304194784Sjeff continue; 305194784Sjeff count += *(int *)(dpcpu + (uintptr_t)arg1); 306194784Sjeff } 307194784Sjeff return (SYSCTL_OUT(req, &count, sizeof(count))); 308194784Sjeff} 309194784Sjeff 31087702Sjhb#ifdef DDB 311196132SbzDB_SHOW_COMMAND(dpcpu_off, db_show_dpcpu_off) 312196132Sbz{ 313196132Sbz int id; 314150576Srwatson 315209059Sjhb CPU_FOREACH(id) { 316196132Sbz db_printf("dpcpu_off[%2d] = 0x%jx (+ DPCPU_START = %p)\n", 317196132Sbz id, (uintmax_t)dpcpu_off[id], 318196132Sbz (void *)(uintptr_t)(dpcpu_off[id] + DPCPU_START)); 319196132Sbz } 320196132Sbz} 321196132Sbz 322150576Srwatsonstatic void 323150576Srwatsonshow_pcpu(struct pcpu *pc) 32487702Sjhb{ 32587702Sjhb struct thread *td; 32687702Sjhb 32787702Sjhb db_printf("cpuid = %d\n", pc->pc_cpuid); 328208392Sjhb db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); 32987702Sjhb db_printf("curthread = "); 33087702Sjhb td = pc->pc_curthread; 33187702Sjhb if (td != NULL) 33287702Sjhb db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 333173600Sjulian td->td_name); 33487702Sjhb else 33587702Sjhb db_printf("none\n"); 33687702Sjhb db_printf("curpcb = %p\n", pc->pc_curpcb); 33787702Sjhb db_printf("fpcurthread = "); 33887702Sjhb td = pc->pc_fpcurthread; 33987702Sjhb if (td != NULL) 34087702Sjhb db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, 341173600Sjulian td->td_name); 34287702Sjhb else 34387702Sjhb db_printf("none\n"); 34487702Sjhb db_printf("idlethread = "); 34587702Sjhb td = pc->pc_idlethread; 34687702Sjhb if (td != NULL) 347208392Sjhb db_printf("%p: tid %d \"%s\"\n", td, td->td_tid, td->td_name); 34887702Sjhb else 34987702Sjhb db_printf("none\n"); 35087702Sjhb db_show_mdpcpu(pc); 351208392Sjhb 352191816Szec#ifdef VIMAGE 353191816Szec db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); 354191816Szec#endif 355191816Szec 35687702Sjhb#ifdef WITNESS 35787702Sjhb db_printf("spin locks held:\n"); 358207929Sattilio witness_list_locks(&pc->pc_spinlocks, db_printf); 35987702Sjhb#endif 36087702Sjhb} 361150576Srwatson 362150576SrwatsonDB_SHOW_COMMAND(pcpu, db_show_pcpu) 363150576Srwatson{ 364150576Srwatson struct pcpu *pc; 365150576Srwatson int id; 366150576Srwatson 367150576Srwatson if (have_addr) 368150576Srwatson id = ((addr >> 4) % 16) * 10 + (addr % 16); 369150576Srwatson else 370150576Srwatson id = PCPU_GET(cpuid); 371150576Srwatson pc = pcpu_find(id); 372150576Srwatson if (pc == NULL) { 373150576Srwatson db_printf("CPU %d not found\n", id); 374150576Srwatson return; 375150576Srwatson } 376150576Srwatson show_pcpu(pc); 377150576Srwatson} 378150576Srwatson 379183054SsamDB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) 380150576Srwatson{ 381150576Srwatson struct pcpu *pc; 382150576Srwatson int id; 383150576Srwatson 384150576Srwatson db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); 385152021Sjhb for (id = 0; id <= mp_maxid; id++) { 386150576Srwatson pc = pcpu_find(id); 387150576Srwatson if (pc != NULL) { 388150576Srwatson show_pcpu(pc); 389150576Srwatson db_printf("\n"); 390150576Srwatson } 391150576Srwatson } 392150576Srwatson} 393183054SsamDB_SHOW_ALIAS(allpcpu, db_show_cpu_all); 39487702Sjhb#endif 395