1/* 2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 57/* 58 */ 59#ifdef KERNEL_PRIVATE 60 61#ifndef _I386_MP_H_ 62#define _I386_MP_H_ 63 64#ifndef DEBUG 65#include <debug.h> 66#endif 67//#define MP_DEBUG 1 68 69#include <i386/apic.h> 70#include <i386/mp_events.h> 71 72#define MAX_CPUS 32 /* (8*sizeof(long)) */ 73 74#ifndef ASSEMBLER 75#include <stdint.h> 76#include <sys/cdefs.h> 77#include <mach/boolean.h> 78#include <mach/kern_return.h> 79#include <mach/i386/thread_status.h> 80#include <kern/lock.h> 81 82__BEGIN_DECLS 83 84extern kern_return_t intel_startCPU(int slot_num); 85extern kern_return_t intel_startCPU_fast(int slot_num); 86extern void i386_init_slave(void); 87extern void i386_init_slave_fast(void); 88extern void smp_init(void); 89 90extern void cpu_interrupt(int cpu); 91__END_DECLS 92 93extern unsigned int real_ncpus; /* real number of cpus */ 94extern unsigned int max_ncpus; /* max number of cpus */ 95decl_simple_lock_data(extern,kdb_lock) /* kdb lock */ 96 97__BEGIN_DECLS 98 99extern void console_init(void); 100extern void *console_cpu_alloc(boolean_t boot_cpu); 101extern void console_cpu_free(void *console_buf); 102 103extern int kdb_cpu; /* current cpu running kdb */ 104extern int kdb_debug; 105extern int kdb_active[]; 106 107extern volatile boolean_t mp_kdp_trap; 108extern volatile boolean_t force_immediate_debugger_NMI; 109extern volatile boolean_t pmap_tlb_flush_timeout; 110 111extern void mp_kdp_enter(void); 112extern void mp_kdp_exit(void); 113 114#if MACH_KDB 115extern void mp_kdb_exit(void); 116#endif 117 118/* 119 * All cpu rendezvous: 120 */ 121extern void mp_rendezvous( 122 void (*setup_func)(void *), 123 void (*action_func)(void *), 124 void (*teardown_func)(void *), 125 void *arg); 126extern void mp_rendezvous_no_intrs( 127 void (*action_func)(void *), 128 void *arg); 129extern void mp_rendezvous_break_lock(void); 130 131/* 132 * All cpu broadcast. 133 * Called from thread context, this blocks until all active cpus have 134 * run action_func: 135 */ 136extern void mp_broadcast( 137 void (*action_func)(void *), 138 void *arg); 139 140typedef uint32_t cpu_t; 141typedef uint32_t cpumask_t; 142static inline cpumask_t 143cpu_to_cpumask(cpu_t cpu) 144{ 145 return (cpu < 32) ? (1 << cpu) : 0; 146} 147#define CPUMASK_ALL 0xffffffff 148#define CPUMASK_SELF cpu_to_cpumask(cpu_number()) 149#define CPUMASK_OTHERS (CPUMASK_ALL & ~CPUMASK_SELF) 150 151/* 152 * Invoke a function (possibly NULL) on a set of cpus specified by a mask. 153 * The mask may include the local cpu. 154 * If the mode is: 155 * - ASYNC: other cpus make their calls in parallel. 156 * - SYNC: the calls are performed serially in logical cpu order. 157 * This call returns when the function has been run on all specified cpus. 158 * The return value is the number of cpus on which the call was made. 159 * The action function is called with interrupts disabled. 160 */ 161extern cpu_t mp_cpus_call( 162 cpumask_t cpus, 163 mp_sync_t mode, 164 void (*action_func)(void *), 165 void *arg); 166 167/* 168 * Power-management-specific SPI to: 169 * - register a callout function, and 170 * - request the callout (if registered) on a given cpu. 171 */ 172extern void PM_interrupt_register(void (*fn)(void)); 173extern void cpu_PM_interrupt(int cpu); 174 175 176__END_DECLS 177 178#if MP_DEBUG 179typedef struct { 180 uint64_t time; 181 int cpu; 182 mp_event_t event; 183} cpu_signal_event_t; 184 185#define LOG_NENTRIES 100 186typedef struct { 187 uint64_t count[MP_LAST]; 188 int next_entry; 189 cpu_signal_event_t entry[LOG_NENTRIES]; 190} cpu_signal_event_log_t; 191 192extern cpu_signal_event_log_t *cpu_signal[]; 193extern cpu_signal_event_log_t *cpu_handle[]; 194 195#define DBGLOG(log,_cpu,_event) { \ 196 boolean_t spl = ml_set_interrupts_enabled(FALSE); \ 197 cpu_signal_event_log_t *logp = log[cpu_number()]; \ 198 int next = logp->next_entry; \ 199 cpu_signal_event_t *eventp = &logp->entry[next]; \ 200 \ 201 logp->count[_event]++; \ 202 \ 203 eventp->time = rdtsc64(); \ 204 eventp->cpu = _cpu; \ 205 eventp->event = _event; \ 206 if (next == (LOG_NENTRIES - 1)) \ 207 logp->next_entry = 0; \ 208 else \ 209 logp->next_entry++; \ 210 \ 211 (void) ml_set_interrupts_enabled(spl); \ 212} 213 214#define DBGLOG_CPU_INIT(cpu) { \ 215 cpu_signal_event_log_t **sig_logpp = &cpu_signal[cpu]; \ 216 cpu_signal_event_log_t **hdl_logpp = &cpu_handle[cpu]; \ 217 \ 218 if (*sig_logpp == NULL && \ 219 kmem_alloc(kernel_map, \ 220 (vm_offset_t *) sig_logpp, \ 221 sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\ 222 panic("DBGLOG_CPU_INIT cpu_signal allocation failed\n");\ 223 bzero(*sig_logpp, sizeof(cpu_signal_event_log_t)); \ 224 if (*hdl_logpp == NULL && \ 225 kmem_alloc(kernel_map, \ 226 (vm_offset_t *) hdl_logpp, \ 227 sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\ 228 panic("DBGLOG_CPU_INIT cpu_handle allocation failed\n");\ 229 bzero(*sig_logpp, sizeof(cpu_signal_event_log_t)); \ 230} 231#else /* MP_DEBUG */ 232#define DBGLOG(log,_cpu,_event) 233#define DBGLOG_CPU_INIT(cpu) 234#endif /* MP_DEBUG */ 235 236#endif /* ASSEMBLER */ 237 238#define i_bit(bit, word) ((long)(*(word)) & ((long)1 << (bit))) 239 240 241/* 242 * Device driver synchronization. 243 * 244 * at386_io_lock(op) and at386_io_unlock() are called 245 * by device drivers when accessing H/W. The underlying 246 * Processing is machine dependant. But the op argument 247 * to the at386_io_lock is generic 248 */ 249 250#define MP_DEV_OP_MAX 4 251#define MP_DEV_WAIT MP_DEV_OP_MAX /* Wait for the lock */ 252 253/* 254 * If the caller specifies an op value different than MP_DEV_WAIT, the 255 * at386_io_lock function must return true if lock was successful else 256 * false 257 */ 258 259#define MP_DEV_OP_START 0 /* If lock busy, register a pending start op */ 260#define MP_DEV_OP_INTR 1 /* If lock busy, register a pending intr */ 261#define MP_DEV_OP_TIMEO 2 /* If lock busy, register a pending timeout */ 262#define MP_DEV_OP_CALLB 3 /* If lock busy, register a pending callback */ 263 264#if MACH_RT 265#define _DISABLE_PREEMPTION \ 266 incl %gs:CPU_PREEMPTION_LEVEL 267 268#define _ENABLE_PREEMPTION \ 269 decl %gs:CPU_PREEMPTION_LEVEL ; \ 270 jne 9f ; \ 271 pushl %eax ; \ 272 pushl %ecx ; \ 273 pushl %edx ; \ 274 call EXT(kernel_preempt_check) ; \ 275 popl %edx ; \ 276 popl %ecx ; \ 277 popl %eax ; \ 2789: 279 280#define _ENABLE_PREEMPTION_NO_CHECK \ 281 decl %gs:CPU_PREEMPTION_LEVEL 282 283#if MACH_ASSERT 284#define DISABLE_PREEMPTION \ 285 pushl %eax; \ 286 pushl %ecx; \ 287 pushl %edx; \ 288 call EXT(_disable_preemption); \ 289 popl %edx; \ 290 popl %ecx; \ 291 popl %eax 292#define ENABLE_PREEMPTION \ 293 pushl %eax; \ 294 pushl %ecx; \ 295 pushl %edx; \ 296 call EXT(_enable_preemption); \ 297 popl %edx; \ 298 popl %ecx; \ 299 popl %eax 300#define ENABLE_PREEMPTION_NO_CHECK \ 301 pushl %eax; \ 302 pushl %ecx; \ 303 pushl %edx; \ 304 call EXT(_enable_preemption_no_check); \ 305 popl %edx; \ 306 popl %ecx; \ 307 popl %eax 308#define MP_DISABLE_PREEMPTION \ 309 pushl %eax; \ 310 pushl %ecx; \ 311 pushl %edx; \ 312 call EXT(_mp_disable_preemption); \ 313 popl %edx; \ 314 popl %ecx; \ 315 popl %eax 316#define MP_ENABLE_PREEMPTION \ 317 pushl %eax; \ 318 pushl %ecx; \ 319 pushl %edx; \ 320 call EXT(_mp_enable_preemption); \ 321 popl %edx; \ 322 popl %ecx; \ 323 popl %eax 324#define MP_ENABLE_PREEMPTION_NO_CHECK \ 325 pushl %eax; \ 326 pushl %ecx; \ 327 pushl %edx; \ 328 call EXT(_mp_enable_preemption_no_check); \ 329 popl %edx; \ 330 popl %ecx; \ 331 popl %eax 332#else /* MACH_ASSERT */ 333#define DISABLE_PREEMPTION _DISABLE_PREEMPTION 334#define ENABLE_PREEMPTION _ENABLE_PREEMPTION 335#define ENABLE_PREEMPTION_NO_CHECK _ENABLE_PREEMPTION_NO_CHECK 336#define MP_DISABLE_PREEMPTION _DISABLE_PREEMPTION 337#define MP_ENABLE_PREEMPTION _ENABLE_PREEMPTION 338#define MP_ENABLE_PREEMPTION_NO_CHECK _ENABLE_PREEMPTION_NO_CHECK 339#endif /* MACH_ASSERT */ 340 341#else /* MACH_RT */ 342#define DISABLE_PREEMPTION 343#define ENABLE_PREEMPTION 344#define ENABLE_PREEMPTION_NO_CHECK 345#define MP_DISABLE_PREEMPTION 346#define MP_ENABLE_PREEMPTION 347#define MP_ENABLE_PREEMPTION_NO_CHECK 348#endif /* MACH_RT */ 349 350#endif /* _I386_MP_H_ */ 351 352#endif /* KERNEL_PRIVATE */ 353