135069Speter/*- 235069Speter * Copyright (c) Peter Wemm <peter@netplex.com.au> 335069Speter * All rights reserved. 435069Speter * 535069Speter * Redistribution and use in source and binary forms, with or without 635069Speter * modification, are permitted provided that the following conditions 735069Speter * are met: 835069Speter * 1. Redistributions of source code must retain the above copyright 935069Speter * notice, this list of conditions and the following disclaimer. 1035069Speter * 2. Redistributions in binary form must reproduce the above copyright 1135069Speter * notice, this list of conditions and the following disclaimer in the 1235069Speter * documentation and/or other materials provided with the distribution. 1335069Speter * 1435069Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1535069Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1635069Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1735069Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1835069Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1935069Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2035069Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2135069Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2235069Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2335069Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2435069Speter * SUCH DAMAGE. 2535069Speter * 2650477Speter * $FreeBSD$ 2735069Speter */ 2835069Speter 2987702Sjhb#ifndef _MACHINE_PCPU_H_ 30166540Sbde#define _MACHINE_PCPU_H_ 3165557Sjasone 32143063Sjoerg#ifndef _SYS_CDEFS_H_ 33166540Sbde#error "sys/cdefs.h is a prerequisite for this file" 34143063Sjoerg#endif 35143063Sjoerg 3635069Speter/* 3735069Speter * The SMP parts are setup in pmap.c and locore.s for the BSP, and 3835069Speter * mp_machdep.c sets up the data for the AP's to "see" when they awake. 3935069Speter * The reason for doing it via a struct is so that an array of pointers 4035069Speter * to each CPU's data can be set up for things like "check curproc on all 4135069Speter * other processors" 4235069Speter */ 4387702Sjhb#define PCPU_MD_FIELDS \ 44178471Sjeff char pc_monitorbuf[128] __aligned(128); /* cache line */ \ 45114349Speter struct pcpu *pc_prvspace; /* Self-reference */ \ 46122849Speter struct pmap *pc_curpmap; \ 47190620Skib struct amd64tss *pc_tssp; /* TSS segment active on CPU */ \ 48190620Skib struct amd64tss *pc_commontssp;/* Common TSS for the CPU */ \ 49122849Speter register_t pc_rsp0; \ 50122763Snjl register_t pc_scratch_rsp; /* User %rsp in syscall */ \ 51122849Speter u_int pc_apic_id; \ 52182868Skib u_int pc_acpi_id; /* ACPI CPU id */ \ 53190620Skib /* Pointer to the CPU %fs descriptor */ \ 54190620Skib struct user_segment_descriptor *pc_fs32p; \ 55190620Skib /* Pointer to the CPU %gs descriptor */ \ 56190620Skib struct user_segment_descriptor *pc_gs32p; \ 57190620Skib /* Pointer to the CPU LDT descriptor */ \ 58190620Skib struct system_segment_descriptor *pc_ldt; \ 59190620Skib /* Pointer to the CPU TSS descriptor */ \ 60208507Sjhb struct system_segment_descriptor *pc_tss; \ 61255060Skib uint64_t pc_pm_save_cnt; \ 62255040Sgibbs u_int pc_cmci_mask; /* MCx banks for CMCI */ \ 63250851Skib uint64_t pc_dbreg[16]; /* ddb debugging regs */ \ 64250851Skib int pc_dbreg_cmd; /* ddb debugging reg cmd */ \ 65256073Sgibbs u_int pc_vcpu_id; /* Xen vCPU ID */ \ 66256073Sgibbs char __pad[157] /* be divisor of PAGE_SIZE \ 67249265Sglebius after cache alignment */ 6835069Speter 69250851Skib#define PC_DBREG_CMD_NONE 0 70250851Skib#define PC_DBREG_CMD_LOAD 1 71250851Skib 72181875Sjhb#ifdef _KERNEL 73181875Sjhb 74166540Sbde#ifdef lint 75166540Sbde 76104291Sphkextern struct pcpu *pcpup; 77104291Sphk 78166540Sbde#define PCPU_GET(member) (pcpup->pc_ ## member) 79170291Sattilio#define PCPU_ADD(member, val) (pcpup->pc_ ## member += (val)) 80170291Sattilio#define PCPU_INC(member) PCPU_ADD(member, 1) 81166540Sbde#define PCPU_PTR(member) (&pcpup->pc_ ## member) 82166540Sbde#define PCPU_SET(member, val) (pcpup->pc_ ## member = (val)) 83166540Sbde 84166540Sbde#elif defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) 85166540Sbde 8687702Sjhb/* 8787702Sjhb * Evaluates to the byte offset of the per-cpu variable name. 8887702Sjhb */ 8987702Sjhb#define __pcpu_offset(name) \ 9087702Sjhb __offsetof(struct pcpu, name) 9187702Sjhb 9287702Sjhb/* 9387702Sjhb * Evaluates to the type of the per-cpu variable name. 9487702Sjhb */ 9587702Sjhb#define __pcpu_type(name) \ 9687702Sjhb __typeof(((struct pcpu *)0)->name) 9787702Sjhb 9887702Sjhb/* 9987702Sjhb * Evaluates to the address of the per-cpu variable name. 10087702Sjhb */ 101122833Sbde#define __PCPU_PTR(name) __extension__ ({ \ 10287702Sjhb __pcpu_type(name) *__p; \ 10387702Sjhb \ 104114349Speter __asm __volatile("movq %%gs:%1,%0; addq %2,%0" \ 10587702Sjhb : "=r" (__p) \ 10687702Sjhb : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))), \ 10787702Sjhb "i" (__pcpu_offset(name))); \ 10887702Sjhb \ 10987702Sjhb __p; \ 11087702Sjhb}) 11187702Sjhb 11287702Sjhb/* 11387702Sjhb * Evaluates to the value of the per-cpu variable name. 11487702Sjhb */ 115122833Sbde#define __PCPU_GET(name) __extension__ ({ \ 116166540Sbde __pcpu_type(name) __res; \ 117166540Sbde struct __s { \ 118166540Sbde u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ 119166540Sbde } __s; \ 12087702Sjhb \ 121166540Sbde if (sizeof(__res) == 1 || sizeof(__res) == 2 || \ 122166540Sbde sizeof(__res) == 4 || sizeof(__res) == 8) { \ 123166536Sbde __asm __volatile("mov %%gs:%1,%0" \ 124166536Sbde : "=r" (__s) \ 125166536Sbde : "m" (*(struct __s *)(__pcpu_offset(name)))); \ 126166540Sbde *(struct __s *)(void *)&__res = __s; \ 12787702Sjhb } else { \ 128166540Sbde __res = *__PCPU_PTR(name); \ 12987702Sjhb } \ 130166540Sbde __res; \ 13187702Sjhb}) 13287702Sjhb 13387702Sjhb/* 134170291Sattilio * Adds the value to the per-cpu counter name. The implementation 135170291Sattilio * must be atomic with respect to interrupts. 136170291Sattilio */ 137170291Sattilio#define __PCPU_ADD(name, val) do { \ 138170291Sattilio __pcpu_type(name) __val; \ 139170291Sattilio struct __s { \ 140170291Sattilio u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ 141170291Sattilio } __s; \ 142170291Sattilio \ 143170291Sattilio __val = (val); \ 144170291Sattilio if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ 145170291Sattilio sizeof(__val) == 4 || sizeof(__val) == 8) { \ 146170291Sattilio __s = *(struct __s *)(void *)&__val; \ 147170291Sattilio __asm __volatile("add %1,%%gs:%0" \ 148170291Sattilio : "=m" (*(struct __s *)(__pcpu_offset(name))) \ 149170291Sattilio : "r" (__s)); \ 150170291Sattilio } else \ 151170291Sattilio *__PCPU_PTR(name) += __val; \ 152170291Sattilio} while (0) 153170291Sattilio 154170291Sattilio/* 155167429Salc * Increments the value of the per-cpu counter name. The implementation 156167429Salc * must be atomic with respect to interrupts. 157167429Salc */ 158170291Sattilio#define __PCPU_INC(name) do { \ 159167429Salc CTASSERT(sizeof(__pcpu_type(name)) == 1 || \ 160167429Salc sizeof(__pcpu_type(name)) == 2 || \ 161167429Salc sizeof(__pcpu_type(name)) == 4 || \ 162167429Salc sizeof(__pcpu_type(name)) == 8); \ 163167429Salc if (sizeof(__pcpu_type(name)) == 1) { \ 164167429Salc __asm __volatile("incb %%gs:%0" \ 165167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 166167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 167167429Salc } else if (sizeof(__pcpu_type(name)) == 2) { \ 168167429Salc __asm __volatile("incw %%gs:%0" \ 169167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 170167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 171167429Salc } else if (sizeof(__pcpu_type(name)) == 4) { \ 172167429Salc __asm __volatile("incl %%gs:%0" \ 173167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 174167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 175167429Salc } else if (sizeof(__pcpu_type(name)) == 8) { \ 176167429Salc __asm __volatile("incq %%gs:%0" \ 177167429Salc : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ 178167429Salc : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ 179167429Salc } \ 180167429Salc} while (0) 181167429Salc 182167429Salc/* 18387702Sjhb * Sets the value of the per-cpu variable name to value val. 18487702Sjhb */ 185122833Sbde#define __PCPU_SET(name, val) { \ 186166540Sbde __pcpu_type(name) __val; \ 187166540Sbde struct __s { \ 188166540Sbde u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ 189166540Sbde } __s; \ 19087702Sjhb \ 191166540Sbde __val = (val); \ 192166536Sbde if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ 193166536Sbde sizeof(__val) == 4 || sizeof(__val) == 8) { \ 194166536Sbde __s = *(struct __s *)(void *)&__val; \ 195166536Sbde __asm __volatile("mov %1,%%gs:%0" \ 196166536Sbde : "=m" (*(struct __s *)(__pcpu_offset(name))) \ 197166536Sbde : "r" (__s)); \ 19887702Sjhb } else { \ 19987702Sjhb *__PCPU_PTR(name) = __val; \ 20087702Sjhb } \ 201122833Sbde} 20287702Sjhb 20387702Sjhb#define PCPU_GET(member) __PCPU_GET(pc_ ## member) 204170291Sattilio#define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val) 205170291Sattilio#define PCPU_INC(member) __PCPU_INC(pc_ ## member) 20687702Sjhb#define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) 20787702Sjhb#define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) 20887702Sjhb 209238623Skib#define OFFSETOF_CURTHREAD 0 210238723Skib#ifdef __clang__ 211238723Skib#pragma clang diagnostic push 212238723Skib#pragma clang diagnostic ignored "-Wnull-dereference" 213238723Skib#endif 214210623Sjhbstatic __inline __pure2 struct thread * 215122930Speter__curthread(void) 216122930Speter{ 217122930Speter struct thread *td; 218122930Speter 219238623Skib __asm("movq %%gs:%1,%0" : "=r" (td) 220238623Skib : "m" (*(char *)OFFSETOF_CURTHREAD)); 221122930Speter return (td); 222122930Speter} 223238723Skib#ifdef __clang__ 224238723Skib#pragma clang diagnostic pop 225238723Skib#endif 226166540Sbde#define curthread (__curthread()) 227122930Speter 228238623Skib#define OFFSETOF_CURPCB 32 229238623Skibstatic __inline __pure2 struct pcb * 230238623Skib__curpcb(void) 231238623Skib{ 232238623Skib struct pcb *pcb; 233238623Skib 234238623Skib __asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB)); 235238623Skib return (pcb); 236238623Skib} 237238623Skib#define curpcb (__curpcb()) 238238623Skib 239230260Skib#define IS_BSP() (PCPU_GET(cpuid) == 0) 240230260Skib 241166540Sbde#else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */ 242100079Smarkm 243166540Sbde#error "this file needs to be ported to your compiler" 24481763Sobrien 245166540Sbde#endif /* lint, etc. */ 246166540Sbde 247166540Sbde#endif /* _KERNEL */ 248166540Sbde 249166540Sbde#endif /* !_MACHINE_PCPU_H_ */ 250