1/* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 * 31 */ 32 33#ifndef I386_CPU_DATA 34#define I386_CPU_DATA 35 36#include <mach_assert.h> 37 38#include <kern/assert.h> 39#include <kern/kern_types.h> 40#include <kern/queue.h> 41#include <kern/processor.h> 42#include <kern/pms.h> 43#include <pexpert/pexpert.h> 44#include <mach/i386/thread_status.h> 45#include <mach/i386/vm_param.h> 46#include <i386/locks.h> 47#include <i386/rtclock_protos.h> 48#include <i386/pmCPU.h> 49#include <i386/cpu_topology.h> 50 51#if CONFIG_VMX 52#include <i386/vmx/vmx_cpu.h> 53#endif 54 55#include <machine/pal_routines.h> 56 57/* 58 * Data structures referenced (anonymously) from per-cpu data: 59 */ 60struct cpu_cons_buffer; 61struct cpu_desc_table; 62struct mca_state; 63struct prngContext; 64 65/* 66 * Data structures embedded in per-cpu data: 67 */ 68typedef struct rtclock_timer { 69 mpqueue_head_t queue; 70 uint64_t deadline; 71 uint64_t when_set; 72 boolean_t has_expired; 73} rtclock_timer_t; 74 75 76typedef struct { 77 struct x86_64_tss *cdi_ktss; 78 struct __attribute__((packed)) { 79 uint16_t size; 80 void *ptr; 81 } cdi_gdt, cdi_idt; 82 struct fake_descriptor *cdi_ldt; 83 vm_offset_t cdi_sstk; 84} cpu_desc_index_t; 85 86typedef enum { 87 TASK_MAP_32BIT, /* 32-bit user, compatibility mode */ 88 TASK_MAP_64BIT, /* 64-bit user thread, shared space */ 89} task_map_t; 90 91 92/* 93 * This structure is used on entry into the (uber-)kernel on syscall from 94 * a 64-bit user. It contains the address of the machine state save area 95 * for the current thread and a temporary place to save the user's rsp 96 * before loading this address into rsp. 97 */ 98typedef struct { 99 addr64_t cu_isf; /* thread->pcb->iss.isf */ 100 uint64_t cu_tmp; /* temporary scratch */ 101 addr64_t cu_user_gs_base; 102} cpu_uber_t; 103 104typedef uint16_t pcid_t; 105typedef uint8_t pcid_ref_t; 106 107#define CPU_RTIME_BINS (12) 108#define CPU_ITIME_BINS (CPU_RTIME_BINS) 109 110/* 111 * Per-cpu data. 112 * 113 * Each processor has a per-cpu data area which is dereferenced through the 114 * current_cpu_datap() macro. For speed, the %gs segment is based here, and 115 * using this, inlines provides single-instruction access to frequently used 116 * members - such as get_cpu_number()/cpu_number(), and get_active_thread()/ 117 * current_thread(). 118 * 119 * Cpu data owned by another processor can be accessed using the 120 * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu 121 * pointers. 122 */ 123typedef struct cpu_data 124{ 125 struct pal_cpu_data cpu_pal_data; /* PAL-specific data */ 126#define cpu_pd cpu_pal_data /* convenience alias */ 127 struct cpu_data *cpu_this; /* pointer to myself */ 128 thread_t cpu_active_thread; 129 thread_t cpu_nthread; 130 volatile int cpu_preemption_level; 131 int cpu_number; /* Logical CPU */ 132 void *cpu_int_state; /* interrupt state */ 133 vm_offset_t cpu_active_stack; /* kernel stack base */ 134 vm_offset_t cpu_kernel_stack; /* kernel stack top */ 135 vm_offset_t cpu_int_stack_top; 136 int cpu_interrupt_level; 137 int cpu_phys_number; /* Physical CPU */ 138 cpu_id_t cpu_id; /* Platform Expert */ 139 volatile int cpu_signals; /* IPI events */ 140 volatile int cpu_prior_signals; /* Last set of events, 141 * debugging 142 */ 143 ast_t cpu_pending_ast; 144 volatile int cpu_running; 145 boolean_t cpu_fixed_pmcs_enabled; 146 rtclock_timer_t rtclock_timer; 147 volatile addr64_t cpu_active_cr3 __attribute((aligned(64))); 148 union { 149 volatile uint32_t cpu_tlb_invalid; 150 struct { 151 volatile uint16_t cpu_tlb_invalid_local; 152 volatile uint16_t cpu_tlb_invalid_global; 153 }; 154 }; 155 volatile task_map_t cpu_task_map; 156 volatile addr64_t cpu_task_cr3; 157 addr64_t cpu_kernel_cr3; 158 cpu_uber_t cpu_uber; 159 void *cpu_chud; 160 void *cpu_console_buf; 161 struct x86_lcpu lcpu; 162 struct processor *cpu_processor; 163#if NCOPY_WINDOWS > 0 164 struct cpu_pmap *cpu_pmap; 165#endif 166 struct cpu_desc_table *cpu_desc_tablep; 167 struct fake_descriptor *cpu_ldtp; 168 cpu_desc_index_t cpu_desc_index; 169 int cpu_ldt; 170#if NCOPY_WINDOWS > 0 171 vm_offset_t cpu_copywindow_base; 172 uint64_t *cpu_copywindow_pdp; 173 174 vm_offset_t cpu_physwindow_base; 175 uint64_t *cpu_physwindow_ptep; 176#endif 177 178#define HWINTCNT_SIZE 256 179 uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */ 180 uint64_t cpu_hwIntpexits[HWINTCNT_SIZE]; 181 uint64_t cpu_hwIntcexits[HWINTCNT_SIZE]; 182 uint64_t cpu_dr7; /* debug control register */ 183 uint64_t cpu_int_event_time; /* intr entry/exit time */ 184 pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ 185#if CONFIG_COUNTERS 186 thread_t csw_old_thread; 187 thread_t csw_new_thread; 188#endif /* CONFIG COUNTERS */ 189#if KPC 190 /* double-buffered performance counter data */ 191 uint64_t *cpu_kpc_buf[2]; 192 /* PMC shadow and reload value buffers */ 193 uint64_t *cpu_kpc_shadow; 194 uint64_t *cpu_kpc_reload; 195#endif 196 uint32_t cpu_pmap_pcid_enabled; 197 pcid_t cpu_active_pcid; 198 pcid_t cpu_last_pcid; 199 volatile pcid_ref_t *cpu_pmap_pcid_coherentp; 200 volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel; 201#define PMAP_PCID_MAX_PCID (0x1000) 202 pcid_t cpu_pcid_free_hint; 203 pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; 204 pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; 205#ifdef PCID_STATS 206 uint64_t cpu_pmap_pcid_flushes; 207 uint64_t cpu_pmap_pcid_preserves; 208#endif 209 uint64_t cpu_aperf; 210 uint64_t cpu_mperf; 211 uint64_t cpu_c3res; 212 uint64_t cpu_c6res; 213 uint64_t cpu_c7res; 214 uint64_t cpu_itime_total; 215 uint64_t cpu_rtime_total; 216 uint64_t cpu_ixtime; 217 uint64_t cpu_idle_exits; 218 uint64_t cpu_rtimes[CPU_RTIME_BINS]; 219 uint64_t cpu_itimes[CPU_ITIME_BINS]; 220 uint64_t cpu_cur_insns; 221 uint64_t cpu_cur_ucc; 222 uint64_t cpu_cur_urc; 223 uint64_t cpu_max_observed_int_latency; 224 int cpu_max_observed_int_latency_vector; 225 volatile boolean_t cpu_NMI_acknowledged; 226 uint64_t debugger_entry_time; 227 uint64_t debugger_ipi_time; 228 /* A separate nested interrupt stack flag, to account 229 * for non-nested interrupts arriving while on the interrupt stack 230 * Currently only occurs when AICPM enables interrupts on the 231 * interrupt stack during processor offlining. 232 */ 233 uint32_t cpu_nested_istack; 234 uint32_t cpu_nested_istack_events; 235 x86_saved_state64_t *cpu_fatal_trap_state; 236 x86_saved_state64_t *cpu_post_fatal_trap_state; 237#if CONFIG_VMX 238 vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */ 239#endif 240#if CONFIG_MCA 241 struct mca_state *cpu_mca_state; /* State at MC fault */ 242#endif 243 struct prngContext *cpu_prng; /* PRNG's context */ 244 int cpu_type; 245 int cpu_subtype; 246 int cpu_threadtype; 247 boolean_t cpu_iflag; 248 boolean_t cpu_boot_complete; 249 int cpu_hibernate; 250} cpu_data_t; 251 252extern cpu_data_t *cpu_data_ptr[]; 253 254/* Macro to generate inline bodies to retrieve per-cpu data fields. */ 255#if defined(__clang__) 256#define GS_RELATIVE volatile __attribute__((address_space(256))) 257#ifndef offsetof 258#define offsetof(TYPE,MEMBER) __builtin_offsetof(TYPE,MEMBER) 259#endif 260 261#define CPU_DATA_GET(member,type) \ 262 cpu_data_t GS_RELATIVE *cpu_data = \ 263 (cpu_data_t GS_RELATIVE *)0UL; \ 264 type ret; \ 265 ret = cpu_data->member; \ 266 return ret; 267 268#define CPU_DATA_GET_INDEX(member,index,type) \ 269 cpu_data_t GS_RELATIVE *cpu_data = \ 270 (cpu_data_t GS_RELATIVE *)0UL; \ 271 type ret; \ 272 ret = cpu_data->member[index]; \ 273 return ret; 274 275#define CPU_DATA_SET(member,value) \ 276 cpu_data_t GS_RELATIVE *cpu_data = \ 277 (cpu_data_t GS_RELATIVE *)0UL; \ 278 cpu_data->member = value; 279 280#define CPU_DATA_XCHG(member,value,type) \ 281 cpu_data_t GS_RELATIVE *cpu_data = \ 282 (cpu_data_t GS_RELATIVE *)0UL; \ 283 type ret; \ 284 ret = cpu_data->member; \ 285 cpu_data->member = value; \ 286 return ret; 287 288#else /* !defined(__clang__) */ 289 290#ifndef offsetof 291#define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 292#endif /* offsetof */ 293#define CPU_DATA_GET(member,type) \ 294 type ret; \ 295 __asm__ volatile ("mov %%gs:%P1,%0" \ 296 : "=r" (ret) \ 297 : "i" (offsetof(cpu_data_t,member))); \ 298 return ret; 299 300#define CPU_DATA_GET_INDEX(member,index,type) \ 301 type ret; \ 302 __asm__ volatile ("mov %%gs:(%1),%0" \ 303 : "=r" (ret) \ 304 : "r" (offsetof(cpu_data_t,member[index]))); \ 305 return ret; 306 307#define CPU_DATA_SET(member,value) \ 308 __asm__ volatile ("mov %0,%%gs:%P1" \ 309 : \ 310 : "r" (value), "i" (offsetof(cpu_data_t,member))); 311 312#define CPU_DATA_XCHG(member,value,type) \ 313 type ret; \ 314 __asm__ volatile ("xchg %0,%%gs:%P1" \ 315 : "=r" (ret) \ 316 : "i" (offsetof(cpu_data_t,member)), "0" (value)); \ 317 return ret; 318 319#endif /* !defined(__clang__) */ 320 321/* 322 * Everyone within the osfmk part of the kernel can use the fast 323 * inline versions of these routines. Everyone outside, must call 324 * the real thing, 325 */ 326static inline thread_t 327get_active_thread(void) 328{ 329 CPU_DATA_GET(cpu_active_thread,thread_t) 330} 331#define current_thread_fast() get_active_thread() 332#define current_thread() current_thread_fast() 333 334#define cpu_mode_is64bit() TRUE 335 336static inline int 337get_preemption_level(void) 338{ 339 CPU_DATA_GET(cpu_preemption_level,int) 340} 341static inline int 342get_interrupt_level(void) 343{ 344 CPU_DATA_GET(cpu_interrupt_level,int) 345} 346static inline int 347get_cpu_number(void) 348{ 349 CPU_DATA_GET(cpu_number,int) 350} 351static inline int 352get_cpu_phys_number(void) 353{ 354 CPU_DATA_GET(cpu_phys_number,int) 355} 356 357 358static inline void 359disable_preemption(void) 360{ 361#if defined(__clang__) 362 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 363 cpu_data->cpu_preemption_level++; 364#else 365 __asm__ volatile ("incl %%gs:%P0" 366 : 367 : "i" (offsetof(cpu_data_t, cpu_preemption_level))); 368#endif 369} 370 371static inline void 372enable_preemption(void) 373{ 374 assert(get_preemption_level() > 0); 375 376#if defined(__clang__) 377 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 378 if (0 == --cpu_data->cpu_preemption_level) 379 kernel_preempt_check(); 380#else 381 __asm__ volatile ("decl %%gs:%P0 \n\t" 382 "jne 1f \n\t" 383 "call _kernel_preempt_check \n\t" 384 "1:" 385 : /* no outputs */ 386 : "i" (offsetof(cpu_data_t, cpu_preemption_level)) 387 : "eax", "ecx", "edx", "cc", "memory"); 388#endif 389} 390 391static inline void 392enable_preemption_no_check(void) 393{ 394 assert(get_preemption_level() > 0); 395 396#if defined(__clang__) 397 cpu_data_t GS_RELATIVE *cpu_data = (cpu_data_t GS_RELATIVE *)0UL; 398 cpu_data->cpu_preemption_level--; 399#else 400 __asm__ volatile ("decl %%gs:%P0" 401 : /* no outputs */ 402 : "i" (offsetof(cpu_data_t, cpu_preemption_level)) 403 : "cc", "memory"); 404#endif 405} 406 407static inline void 408mp_disable_preemption(void) 409{ 410 disable_preemption(); 411} 412 413static inline void 414mp_enable_preemption(void) 415{ 416 enable_preemption(); 417} 418 419static inline void 420mp_enable_preemption_no_check(void) 421{ 422 enable_preemption_no_check(); 423} 424 425static inline cpu_data_t * 426current_cpu_datap(void) 427{ 428 CPU_DATA_GET(cpu_this, cpu_data_t *); 429} 430 431static inline cpu_data_t * 432cpu_datap(int cpu) 433{ 434 return cpu_data_ptr[cpu]; 435} 436 437extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu); 438extern void cpu_data_realloc(void); 439 440#endif /* I386_CPU_DATA */ 441