135069Speter/*- 2158502Speter * Copyright (c) Peter Wemm 335069Speter * All rights reserved. 435069Speter * 535069Speter * Redistribution and use in source and binary forms, with or without 635069Speter * modification, are permitted provided that the following conditions 735069Speter * are met: 835069Speter * 1. Redistributions of source code must retain the above copyright 935069Speter * notice, this list of conditions and the following disclaimer. 1035069Speter * 2. Redistributions in binary form must reproduce the above copyright 1135069Speter * notice, this list of conditions and the following disclaimer in the 1235069Speter * documentation and/or other materials provided with the distribution. 1335069Speter * 1435069Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1535069Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1635069Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1735069Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1835069Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1935069Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2035069Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2135069Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2235069Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2335069Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2435069Speter * SUCH DAMAGE. 2535069Speter * 2650477Speter * $FreeBSD$ 2735069Speter */ 2835069Speter 2987702Sjhb#ifndef _MACHINE_PCPU_H_ 30166540Sbde#define _MACHINE_PCPU_H_ 3165557Sjasone 32143063Sjoerg#ifndef _SYS_CDEFS_H_ 33166540Sbde#error "sys/cdefs.h is a prerequisite for this file" 34143063Sjoerg#endif 35143063Sjoerg 3665557Sjasone#include <machine/segments.h> 3765557Sjasone#include <machine/tss.h> 3865557Sjasone 3935069Speter/* 4035069Speter * The SMP parts are setup in pmap.c and locore.s for the BSP, and 4135069Speter * mp_machdep.c sets up the data for the AP's to "see" when they awake. 4235069Speter * The reason for doing it via a struct is so that an array of pointers 4335069Speter * to each CPU's data can be set up for things like "check curproc on all 4435069Speter * other processors" 4535069Speter */ 46181775Skmacy 47216956Srwatson#if defined(XEN) || defined(XENHVM) 48184198Skmacy#ifndef NR_VIRQS 49184198Skmacy#define NR_VIRQS 24 50184198Skmacy#endif 51184198Skmacy#ifndef NR_IPIS 52184198Skmacy#define NR_IPIS 2 53184198Skmacy#endif 54216956Srwatson#endif 55184198Skmacy 56216956Srwatson#if defined(XEN) 57216956Srwatson 58184198Skmacy/* These are peridically updated in shared_info, and then copied here. */ 59184198Skmacystruct shadow_time_info { 60184198Skmacy uint64_t tsc_timestamp; /* TSC at last update of time vals. */ 61184198Skmacy uint64_t system_timestamp; /* Time, in nanosecs, since boot. */ 62184198Skmacy uint32_t tsc_to_nsec_mul; 63184198Skmacy uint32_t tsc_to_usec_mul; 64184198Skmacy int tsc_shift; 65184198Skmacy uint32_t version; 66184198Skmacy}; 67184198Skmacy 68208742Sjhb#define PCPU_XEN_FIELDS \ 69208742Sjhb ; \ 70208742Sjhb u_int pc_cr3; /* track cr3 for R1/R3*/ \ 71208742Sjhb vm_paddr_t *pc_pdir_shadow; \ 72208742Sjhb uint64_t pc_processed_system_time; \ 73184198Skmacy struct shadow_time_info pc_shadow_time; \ 74184198Skmacy int pc_resched_irq; \ 75184198Skmacy int pc_callfunc_irq; \ 76208742Sjhb int pc_virq_to_irq[NR_VIRQS]; \ 77208742Sjhb int pc_ipi_to_irq[NR_IPIS] 78216956Srwatson 79266269Scperciva#elif defined(XENHVM) && !defined(MODXENHVM) 80216956Srwatson 81266268Scperciva/* This is now unused, but remains here for KBI compatibility reasons. */ 82216956Srwatson#define PCPU_XEN_FIELDS \ 83216956Srwatson ; \ 84216956Srwatson unsigned int pc_last_processed_l1i; \ 85216956Srwatson unsigned int pc_last_processed_l2i 86216956Srwatson 87266269Scperciva#else /* !XEN && (!XENHVM || MODXENHVM) */ 88216956Srwatson 89208742Sjhb#define PCPU_XEN_FIELDS 90216956Srwatson 91208742Sjhb#endif 92181775Skmacy 93181775Skmacy#define PCPU_MD_FIELDS \ 94181775Skmacy char pc_monitorbuf[128] __aligned(128); /* cache line */ \ 95181775Skmacy struct pcpu *pc_prvspace; /* Self-reference */ \ 96181775Skmacy struct pmap *pc_curpmap; \ 97181775Skmacy struct i386tss pc_common_tss; \ 98181775Skmacy struct segment_descriptor pc_common_tssd; \ 99181775Skmacy struct segment_descriptor *pc_tss_gdt; \ 100181775Skmacy struct segment_descriptor *pc_fsgs_gdt; \ 101181775Skmacy int pc_currentldt; \ 102181775Skmacy u_int pc_acpi_id; /* ACPI CPU id */ \ 103181775Skmacy u_int pc_apic_id; \ 104208507Sjhb int pc_private_tss; /* Flag indicating private tss*/\ 105208507Sjhb u_int pc_cmci_mask /* MCx banks for CMCI */ \ 106208742Sjhb PCPU_XEN_FIELDS 10735069Speter 108181875Sjhb#ifdef _KERNEL 109181875Sjhb 110166540Sbde#ifdef lint 111166540Sbde 112104291Sphkextern struct pcpu *pcpup; 113104291Sphk 114166540Sbde#define PCPU_GET(member) (pcpup->pc_ ## member) 115184198Skmacy#define PCPU_ADD(member, val) (pcpup->pc_ ## member += (val)) 116170291Sattilio#define PCPU_INC(member) PCPU_ADD(member, 1) 117166540Sbde#define PCPU_PTR(member) (&pcpup->pc_ ## member) 118166540Sbde#define PCPU_SET(member, val) (pcpup->pc_ ## member = (val)) 119166540Sbde 120166540Sbde#elif defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) 121166540Sbde 12287702Sjhb/* 12387702Sjhb * Evaluates to the byte offset of the per-cpu variable name. 12487702Sjhb */ 12587702Sjhb#define __pcpu_offset(name) \ 12687702Sjhb __offsetof(struct pcpu, name) 12787702Sjhb 12887702Sjhb/* 12987702Sjhb * Evaluates to the type of the per-cpu variable name. 13087702Sjhb */ 13187702Sjhb#define __pcpu_type(name) \ 13287702Sjhb __typeof(((struct pcpu *)0)->name) 13387702Sjhb 13487702Sjhb/* 13587702Sjhb * Evaluates to the address of the per-cpu variable name. 13687702Sjhb */ 137122833Sbde#define __PCPU_PTR(name) __extension__ ({ \ 13887702Sjhb __pcpu_type(name) *__p; \ 13987702Sjhb \ 14087702Sjhb __asm __volatile("movl %%fs:%1,%0; addl %2,%0" \ 14187702Sjhb : "=r" (__p) \ 14287702Sjhb : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))), \ 14387702Sjhb "i" (__pcpu_offset(name))); \ 14487702Sjhb \ 14587702Sjhb __p; \ 14687702Sjhb}) 14787702Sjhb 14887702Sjhb/* 14987702Sjhb * Evaluates to the value of the per-cpu variable name. 15087702Sjhb */ 151122833Sbde#define __PCPU_GET(name) __extension__ ({ \ 152166540Sbde __pcpu_type(name) __res; \ 153166540Sbde struct __s { \ 154196816Sjulian u_char __b[MIN(sizeof(__res), 4)]; \ 155166540Sbde } __s; \ 15687702Sjhb \ 157166540Sbde if (sizeof(__res) == 1 || sizeof(__res) == 2 || \ 158166540Sbde sizeof(__res) == 4) { \ 159166536Sbde __asm __volatile("mov %%fs:%1,%0" \ 160166536Sbde : "=r" (__s) \ 161166536Sbde : "m" (*(struct __s *)(__pcpu_offset(name)))); \ 162166540Sbde *(struct __s *)(void *)&__res = __s; \ 16387702Sjhb } else { \ 164166540Sbde __res = *__PCPU_PTR(name); \ 16587702Sjhb } \ 166166540Sbde __res; \ 16787702Sjhb}) 16887702Sjhb 16987702Sjhb/* 170170291Sattilio * Adds a value of the per-cpu counter name. The implementation 171170291Sattilio * must be atomic with respect to interrupts. 172170291Sattilio */ 173170291Sattilio#define __PCPU_ADD(name, val) do { \ 174170291Sattilio __pcpu_type(name) __val; \ 175170291Sattilio struct __s { \ 176196816Sjulian u_char __b[MIN(sizeof(__val), 4)]; \ 177170291Sattilio } __s; \ 178170291Sattilio \ 179170291Sattilio __val = (val); \ 180170291Sattilio if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ 181170291Sattilio sizeof(__val) == 4) { \ 182170291Sattilio __s = *(struct __s *)(void *)&__val; \ 183170291Sattilio __asm __volatile("add %1,%%fs:%0" \ 184170291Sattilio : "=m" (*(struct __s *)(__pcpu_offset(name))) \ 185170291Sattilio : "r" (__s)); \ 186170291Sattilio } else \ 187170291Sattilio *__PCPU_PTR(name) += __val; \ 188170291Sattilio} while (0) 189170291Sattilio 190170291Sattilio/* 191167429Salc * Increments the value of the per-cpu counter name. The implementation 192167429Salc * must be atomic with respect to interrupts. 193167429Salc */ 194170291Sattilio#define __PCPU_INC(name) do { \ 195167429Salc CTASSERT(sizeof(__pcpu_type(name)) == 1 || \ 196167429Salc sizeof(__pcpu_type(name)) == 2 || \ 197167429Salc sizeof(__pcpu_type(name)) == 4); \ 198167429Salc if (sizeof(__pcpu_type(name)) == 1) { \ 199167429Salc __asm __volatile("incb %%fs:%0" \ 200167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 201167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 202167429Salc } else if (sizeof(__pcpu_type(name)) == 2) { \ 203167429Salc __asm __volatile("incw %%fs:%0" \ 204167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 205167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 206167429Salc } else if (sizeof(__pcpu_type(name)) == 4) { \ 207167429Salc __asm __volatile("incl %%fs:%0" \ 208167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 209167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 210167429Salc } \ 211167429Salc} while (0) 212167429Salc 213167429Salc/* 21487702Sjhb * Sets the value of the per-cpu variable name to value val. 21587702Sjhb */ 216196811Sjulian#define __PCPU_SET(name, val) do { \ 217166540Sbde __pcpu_type(name) __val; \ 218166540Sbde struct __s { \ 219196811Sjulian u_char __b[MIN(sizeof(__val), 4)]; \ 220166540Sbde } __s; \ 22187702Sjhb \ 222166540Sbde __val = (val); \ 223166536Sbde if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ 224166536Sbde sizeof(__val) == 4) { \ 225166536Sbde __s = *(struct __s *)(void *)&__val; \ 226166536Sbde __asm __volatile("mov %1,%%fs:%0" \ 227166536Sbde : "=m" (*(struct __s *)(__pcpu_offset(name))) \ 228166536Sbde : "r" (__s)); \ 22987702Sjhb } else { \ 23087702Sjhb *__PCPU_PTR(name) = __val; \ 23187702Sjhb } \ 232196811Sjulian} while (0) 23387702Sjhb 23487702Sjhb#define PCPU_GET(member) __PCPU_GET(pc_ ## member) 235170291Sattilio#define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val) 236170291Sattilio#define PCPU_INC(member) __PCPU_INC(pc_ ## member) 23787702Sjhb#define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) 23887702Sjhb#define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) 23987702Sjhb 240239996Skib#define OFFSETOF_CURTHREAD 0 241239996Skib#ifdef __clang__ 242239996Skib#pragma clang diagnostic push 243239996Skib#pragma clang diagnostic ignored "-Wnull-dereference" 244239996Skib#endif 245210623Sjhbstatic __inline __pure2 struct thread * 246122931Speter__curthread(void) 247122931Speter{ 248122931Speter struct thread *td; 249122931Speter 250239996Skib __asm("movl %%fs:%1,%0" : "=r" (td) 251239996Skib : "m" (*(char *)OFFSETOF_CURTHREAD)); 252122931Speter return (td); 253122931Speter} 254239996Skib#ifdef __clang__ 255239996Skib#pragma clang diagnostic pop 256239996Skib#endif 257166540Sbde#define curthread (__curthread()) 258122931Speter 259239996Skib#define OFFSETOF_CURPCB 16 260239996Skibstatic __inline __pure2 struct pcb * 261239996Skib__curpcb(void) 262239996Skib{ 263239996Skib struct pcb *pcb; 264239996Skib 265239996Skib __asm("movl %%fs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB)); 266239996Skib return (pcb); 267239996Skib} 268239996Skib#define curpcb (__curpcb()) 269239996Skib 270166540Sbde#else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */ 271100079Smarkm 272166540Sbde#error "this file needs to be ported to your compiler" 27381763Sobrien 274166540Sbde#endif /* lint, etc. */ 275166540Sbde 276166540Sbde#endif /* _KERNEL */ 277166540Sbde 278166540Sbde#endif /* !_MACHINE_PCPU_H_ */ 279