1/* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 * 31 */ 32 33#ifndef I386_CPU_DATA 34#define I386_CPU_DATA 35 36#include <mach_assert.h> 37 38#include <kern/assert.h> 39#include <kern/kern_types.h> 40#include <kern/queue.h> 41#include <kern/processor.h> 42#include <kern/pms.h> 43#include <pexpert/pexpert.h> 44#include <mach/i386/thread_status.h> 45#include <mach/i386/vm_param.h> 46#include <i386/rtclock_protos.h> 47#include <i386/pmCPU.h> 48#include <i386/cpu_topology.h> 49 50#if CONFIG_VMX 51#include <i386/vmx/vmx_cpu.h> 52#endif 53 54#include <machine/pal_routines.h> 55 56/* 57 * Data structures referenced (anonymously) from per-cpu data: 58 */ 59struct cpu_cons_buffer; 60struct cpu_desc_table; 61struct mca_state; 62 63/* 64 * Data structures embedded in per-cpu data: 65 */ 66typedef struct rtclock_timer { 67 mpqueue_head_t queue; 68 uint64_t deadline; 69 uint64_t when_set; 70 boolean_t has_expired; 71} rtclock_timer_t; 72 73 74typedef struct { 75 struct x86_64_tss *cdi_ktss; 76 struct __attribute__((packed)) { 77 uint16_t size; 78 void *ptr; 79 } cdi_gdt, cdi_idt; 80 struct fake_descriptor *cdi_ldt; 81 vm_offset_t cdi_sstk; 82} cpu_desc_index_t; 83 84typedef enum { 85 TASK_MAP_32BIT, /* 32-bit user, compatibility mode */ 86 TASK_MAP_64BIT, /* 64-bit user thread, shared space */ 87} task_map_t; 88 89 90/* 91 * This structure is used on entry into the (uber-)kernel on syscall from 92 * a 64-bit user. It contains the address of the machine state save area 93 * for the current thread and a temporary place to save the user's rsp 94 * before loading this address into rsp. 95 */ 96typedef struct { 97 addr64_t cu_isf; /* thread->pcb->iss.isf */ 98 uint64_t cu_tmp; /* temporary scratch */ 99 addr64_t cu_user_gs_base; 100} cpu_uber_t; 101 102typedef uint16_t pcid_t; 103typedef uint8_t pcid_ref_t; 104 105#define CPU_RTIME_BINS (12) 106#define CPU_ITIME_BINS (CPU_RTIME_BINS) 107 108/* 109 * Per-cpu data. 110 * 111 * Each processor has a per-cpu data area which is dereferenced through the 112 * current_cpu_datap() macro. For speed, the %gs segment is based here, and 113 * using this, inlines provides single-instruction access to frequently used 114 * members - such as get_cpu_number()/cpu_number(), and get_active_thread()/ 115 * current_thread(). 116 * 117 * Cpu data owned by another processor can be accessed using the 118 * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu 119 * pointers. 120 */ 121typedef struct cpu_data 122{ 123 struct pal_cpu_data cpu_pal_data; /* PAL-specific data */ 124#define cpu_pd cpu_pal_data /* convenience alias */ 125 struct cpu_data *cpu_this; /* pointer to myself */ 126 thread_t cpu_active_thread; 127 thread_t cpu_nthread; 128 volatile int cpu_preemption_level; 129 int cpu_number; /* Logical CPU */ 130 void *cpu_int_state; /* interrupt state */ 131 vm_offset_t cpu_active_stack; /* kernel stack base */ 132 vm_offset_t cpu_kernel_stack; /* kernel stack top */ 133 vm_offset_t cpu_int_stack_top; 134 int cpu_interrupt_level; 135 int cpu_phys_number; /* Physical CPU */ 136 cpu_id_t cpu_id; /* Platform Expert */ 137 volatile int cpu_signals; /* IPI events */ 138 volatile int cpu_prior_signals; /* Last set of events, 139 * debugging 140 */ 141 ast_t cpu_pending_ast; 142 volatile int cpu_running; 143 boolean_t cpu_fixed_pmcs_enabled; 144 rtclock_timer_t rtclock_timer; 145 volatile addr64_t cpu_active_cr3 __attribute((aligned(64))); 146 union { 147 volatile uint32_t cpu_tlb_invalid; 148 struct { 149 volatile uint16_t cpu_tlb_invalid_local; 150 volatile uint16_t cpu_tlb_invalid_global; 151 }; 152 }; 153 volatile task_map_t cpu_task_map; 154 volatile addr64_t cpu_task_cr3; 155 addr64_t cpu_kernel_cr3; 156 cpu_uber_t cpu_uber; 157 void *cpu_chud; 158 void *cpu_console_buf; 159 struct x86_lcpu lcpu; 160 struct processor *cpu_processor; 161#if NCOPY_WINDOWS > 0 162 struct cpu_pmap *cpu_pmap; 163#endif 164 struct cpu_desc_table *cpu_desc_tablep; 165 struct fake_descriptor *cpu_ldtp; 166 cpu_desc_index_t cpu_desc_index; 167 int cpu_ldt; 168#if NCOPY_WINDOWS > 0 169 vm_offset_t cpu_copywindow_base; 170 uint64_t *cpu_copywindow_pdp; 171 172 vm_offset_t cpu_physwindow_base; 173 uint64_t *cpu_physwindow_ptep; 174#endif 175 176#define HWINTCNT_SIZE 256 177 uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */ 178 uint64_t cpu_hwIntpexits[HWINTCNT_SIZE]; 179 uint64_t cpu_hwIntcexits[HWINTCNT_SIZE]; 180 uint64_t cpu_dr7; /* debug control register */ 181 uint64_t cpu_int_event_time; /* intr entry/exit time */ 182 pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ 183#if CONFIG_COUNTERS 184 thread_t csw_old_thread; 185 thread_t csw_new_thread; 186#endif /* CONFIG COUNTERS */ 187#if KPC 188 /* double-buffered performance counter data */ 189 uint64_t *cpu_kpc_buf[2]; 190 /* PMC shadow and reload value buffers */ 191 uint64_t *cpu_kpc_shadow; 192 uint64_t *cpu_kpc_reload; 193#endif 194 uint32_t cpu_pmap_pcid_enabled; 195 pcid_t cpu_active_pcid; 196 pcid_t cpu_last_pcid; 197 volatile pcid_ref_t *cpu_pmap_pcid_coherentp; 198 volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel; 199#define PMAP_PCID_MAX_PCID (0x1000) 200 pcid_t cpu_pcid_free_hint; 201 pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; 202 pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; 203#ifdef PCID_STATS 204 uint64_t cpu_pmap_pcid_flushes; 205 uint64_t cpu_pmap_pcid_preserves; 206#endif 207 uint64_t cpu_aperf; 208 uint64_t cpu_mperf; 209 uint64_t cpu_c3res; 210 uint64_t cpu_c6res; 211 uint64_t cpu_c7res; 212 uint64_t cpu_itime_total; 213 uint64_t cpu_rtime_total; 214 uint64_t cpu_ixtime; 215 uint64_t cpu_idle_exits; 216 uint64_t cpu_rtimes[CPU_RTIME_BINS]; 217 uint64_t cpu_itimes[CPU_ITIME_BINS]; 218 uint64_t cpu_cur_insns; 219 uint64_t cpu_cur_ucc; 220 uint64_t cpu_cur_urc; 221 uint64_t cpu_max_observed_int_latency; 222 int cpu_max_observed_int_latency_vector; 223 volatile boolean_t cpu_NMI_acknowledged; 224 uint64_t debugger_entry_time; 225 uint64_t debugger_ipi_time; 226 /* A separate nested interrupt stack flag, to account 227 * for non-nested interrupts arriving while on the interrupt stack 228 * Currently only occurs when AICPM enables interrupts on the 229 * interrupt stack during processor offlining. 230 */ 231 uint32_t cpu_nested_istack; 232 uint32_t cpu_nested_istack_events; 233 x86_saved_state64_t *cpu_fatal_trap_state; 234 x86_saved_state64_t *cpu_post_fatal_trap_state; 235#if CONFIG_VMX 236 vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */ 237#endif 238#if CONFIG_MCA 239 struct mca_state *cpu_mca_state; /* State at MC fault */ 240#endif 241 int cpu_type; 242 int cpu_subtype; 243 int cpu_threadtype; 244 boolean_t cpu_iflag; 245 boolean_t cpu_boot_complete; 246 int cpu_hibernate; 247} cpu_data_t; 248 249extern cpu_data_t *cpu_data_ptr[]; 250 251/* Macro to generate inline bodies to retrieve per-cpu data fields. */ 252#if defined(__clang__) 253#define GS_RELATIVE volatile __attribute__((address_space(256))) 254#ifndef offsetof 255#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER) 256#endif 257 258#define CPU_DATA_GET(member,type) \ 259 cpu_data_t GS_RELATIVE *cpu_data = \ 260 (cpu_data_t GS_RELATIVE *)0UL; \ 261 type ret; \ 262 ret = cpu_data->member; \ 263 return ret; 264 265#define CPU_DATA_GET_INDEX(member,index,type) \ 266 cpu_data_t GS_RELATIVE *cpu_data = \ 267 (cpu_data_t GS_RELATIVE *)0UL; \ 268 type ret; \ 269 ret = cpu_data->member[index]; \ 270 return ret; 271 272#define CPU_DATA_SET(member,value) \ 273 cpu_data_t GS_RELATIVE *cpu_data = \ 274 (cpu_data_t GS_RELATIVE *)0UL; \ 275 cpu_data->member = value; 276 277#define CPU_DATA_XCHG(member,value,type) \ 278 cpu_data_t GS_RELATIVE *cpu_data = \ 279 (cpu_data_t GS_RELATIVE *)0UL; \ 280 type ret; \ 281 ret = cpu_data->member; \ 282 cpu_data->member = value; \ 283 return ret; 284 285#else /* !defined(__clang__) */ 286 287#ifndef offsetof 288#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 289#endif /* offsetof */ 290#define CPU_DATA_GET(member,type) \ 291 type ret; \ 292 __asm__ volatile ("mov %%gs:%P1,%0" \ 293 : "=r" (ret) \ 294 : "i" (offsetof(cpu_data_t,member))); \ 295 return ret; 296 297#define CPU_DATA_GET_INDEX(member,index,type) \ 298 type ret; \ 299 __asm__ volatile ("mov %%gs:(%1),%0" \ 300 : "=r" (ret) \ 301 : "r" (offsetof(cpu_data_t,member[index]))); \ 302 return ret; 303 304#define CPU_DATA_SET(member,value) \ 305 __asm__ volatile ("mov %0,%%gs:%P1" \ 306 : \ 307 : "r" (value), "i" (offsetof(cpu_data_t,member))); 308 309#define CPU_DATA_XCHG(member,value,type) \ 310 type ret; \ 311 __asm__ volatile ("xchg %0,%%gs:%P1" \ 312 : "=r" (ret) \ 313 : "i" (offsetof(cpu_data_t,member)), "0" (value)); \ 314 return ret; 315 316#endif /* !defined(__clang__) */ 317 318/* 319 * Everyone within the osfmk part of the kernel can use the fast 320 * inline versions of these routines. Everyone outside, must call 321 * the real thing, 322 */ 323static inline thread_t 324get_active_thread(void) 325{ 326 CPU_DATA_GET(cpu_active_thread,thread_t) 327} 328#define current_thread_fast() get_active_thread() 329#define current_thread() current_thread_fast() 330 331#define cpu_mode_is64bit() TRUE 332 333static inline int 334get_preemption_level(void) 335{ 336 CPU_DATA_GET(cpu_preemption_level,int) 337} 338static inline int 339get_interrupt_level(void) 340{ 341 CPU_DATA_GET(cpu_interrupt_level,int) 342} 343static inline int 344get_cpu_number(void) 345{ 346 CPU_DATA_GET(cpu_number,int) 347} 348static inline int 349get_cpu_phys_number(void) 350{ 351 CPU_DATA_GET(cpu_phys_number,int) 352} 353 354 355static inline void 356disable_preemption(void) 357{ 358#if defined(__clang__) 359 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 360 cpu_data->cpu_preemption_level++; 361#else 362 __asm__ volatile ("incl %%gs:%P0" 363 : 364 : "i" (offsetof(cpu_data_t, cpu_preemption_level))); 365#endif 366} 367 368static inline void 369enable_preemption(void) 370{ 371 assert(get_preemption_level() > 0); 372 373#if defined(__clang__) 374 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 375 if (0 == --cpu_data->cpu_preemption_level) 376 kernel_preempt_check(); 377#else 378 __asm__ volatile ("decl %%gs:%P0 \n\t" 379 "jne 1f \n\t" 380 "call _kernel_preempt_check \n\t" 381 "1:" 382 : /* no outputs */ 383 : "i" (offsetof(cpu_data_t, cpu_preemption_level)) 384 : "eax", "ecx", "edx", "cc", "memory"); 385#endif 386} 387 388static inline void 389enable_preemption_no_check(void) 390{ 391 assert(get_preemption_level() > 0); 392 393#if defined(__clang__) 394 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 395 cpu_data->cpu_preemption_level--; 396#else 397 __asm__ volatile ("decl %%gs:%P0" 398 : /* no outputs */ 399 : "i" (offsetof(cpu_data_t, cpu_preemption_level)) 400 : "cc", "memory"); 401#endif 402} 403 404static inline void 405mp_disable_preemption(void) 406{ 407 disable_preemption(); 408} 409 410static inline void 411mp_enable_preemption(void) 412{ 413 enable_preemption(); 414} 415 416static inline void 417mp_enable_preemption_no_check(void) 418{ 419 enable_preemption_no_check(); 420} 421 422static inline cpu_data_t * 423current_cpu_datap(void) 424{ 425 CPU_DATA_GET(cpu_this, cpu_data_t *); 426} 427 428static inline cpu_data_t * 429cpu_datap(int cpu) 430{ 431 return cpu_data_ptr[cpu]; 432} 433 434extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu); 435extern void cpu_data_realloc(void); 436 437#endif /* I386_CPU_DATA */ 438