1/*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <i386/machine_routines.h>
30#include <i386/io_map_entries.h>
31#include <i386/cpuid.h>
32#include <i386/fpu.h>
33#include <mach/processor.h>
34#include <kern/processor.h>
35#include <kern/machine.h>
36#include <kern/cpu_data.h>
37#include <kern/cpu_number.h>
38#include <kern/thread.h>
39#include <i386/machine_cpu.h>
40#include <i386/lapic.h>
41#include <i386/lock.h>
42#include <i386/mp_events.h>
43#include <i386/pmCPU.h>
44#include <i386/trap.h>
45#include <i386/tsc.h>
46#include <i386/cpu_threads.h>
47#include <i386/proc_reg.h>
48#include <mach/vm_param.h>
49#include <i386/pmap.h>
50#include <i386/pmap_internal.h>
51#include <i386/misc_protos.h>
52
53#if DEBUG
54#define DBG(x...)	kprintf("DBG: " x)
55#else
56#define DBG(x...)
57#endif
58
59extern void 	wakeup(void *);
60
61static int max_cpus_initialized = 0;
62
63unsigned int	LockTimeOut;
64unsigned int	LockTimeOutTSC;
65unsigned int	MutexSpin;
66uint64_t	LastDebuggerEntryAllowance;
67uint64_t	delay_spin_threshold;
68
69extern uint64_t panic_restart_timeout;
70
71boolean_t virtualized = FALSE;
72
73#define MAX_CPUS_SET    0x1
74#define MAX_CPUS_WAIT   0x2
75
76/* IO memory map services */
77
78/* Map memory map IO space */
79vm_offset_t ml_io_map(
80	vm_offset_t phys_addr,
81	vm_size_t size)
82{
83	return(io_map(phys_addr,size,VM_WIMG_IO));
84}
85
86/* boot memory allocation */
87vm_offset_t ml_static_malloc(
88			     __unused vm_size_t size)
89{
90	return((vm_offset_t)NULL);
91}
92
93
94void ml_get_bouncepool_info(vm_offset_t *phys_addr, vm_size_t *size)
95{
96        *phys_addr = 0;
97	*size      = 0;
98}
99
100
101vm_offset_t
102ml_static_ptovirt(
103	vm_offset_t paddr)
104{
105#if defined(__x86_64__)
106	return (vm_offset_t)(((unsigned long) paddr) | VM_MIN_KERNEL_ADDRESS);
107#else
108	return (vm_offset_t)((paddr) | LINEAR_KERNEL_ADDRESS);
109#endif
110}
111
112
113/*
114 *	Routine:        ml_static_mfree
115 *	Function:
116 */
117void
118ml_static_mfree(
119	vm_offset_t vaddr,
120	vm_size_t size)
121{
122	addr64_t vaddr_cur;
123	ppnum_t ppn;
124	uint32_t freed_pages = 0;
125	assert(vaddr >= VM_MIN_KERNEL_ADDRESS);
126
127	assert((vaddr & (PAGE_SIZE-1)) == 0); /* must be page aligned */
128
129	for (vaddr_cur = vaddr;
130 	     vaddr_cur < round_page_64(vaddr+size);
131	     vaddr_cur += PAGE_SIZE) {
132		ppn = pmap_find_phys(kernel_pmap, vaddr_cur);
133		if (ppn != (vm_offset_t)NULL) {
134		        kernel_pmap->stats.resident_count++;
135			if (kernel_pmap->stats.resident_count >
136			    kernel_pmap->stats.resident_max) {
137				kernel_pmap->stats.resident_max =
138					kernel_pmap->stats.resident_count;
139			}
140			pmap_remove(kernel_pmap, vaddr_cur, vaddr_cur+PAGE_SIZE);
141			assert(pmap_valid_page(ppn));
142
143			if (IS_MANAGED_PAGE(ppn)) {
144				vm_page_create(ppn,(ppn+1));
145				vm_page_wire_count--;
146				freed_pages++;
147			}
148		}
149	}
150#if	DEBUG
151	kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn);
152#endif
153}
154
155
156/* virtual to physical on wired pages */
157vm_offset_t ml_vtophys(
158	vm_offset_t vaddr)
159{
160	return	(vm_offset_t)kvtophys(vaddr);
161}
162
163/*
164 *	Routine:        ml_nofault_copy
165 *	Function:	Perform a physical mode copy if the source and
166 *			destination have valid translations in the kernel pmap.
167 *			If translations are present, they are assumed to
168 *			be wired; i.e. no attempt is made to guarantee that the
169 *			translations obtained remained valid for
170 *			the duration of the copy process.
171 */
172
173vm_size_t ml_nofault_copy(
174	vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size)
175{
176	addr64_t cur_phys_dst, cur_phys_src;
177	uint32_t count, nbytes = 0;
178
179	while (size > 0) {
180		if (!(cur_phys_src = kvtophys(virtsrc)))
181			break;
182		if (!(cur_phys_dst = kvtophys(virtdst)))
183			break;
184		if (!pmap_valid_page(i386_btop(cur_phys_dst)) || !pmap_valid_page(i386_btop(cur_phys_src)))
185			break;
186		count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK));
187		if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK)))
188			count = (uint32_t)(PAGE_SIZE - (cur_phys_dst & PAGE_MASK));
189		if (count > size)
190			count = (uint32_t)size;
191
192		bcopy_phys(cur_phys_src, cur_phys_dst, count);
193
194		nbytes += count;
195		virtsrc += count;
196		virtdst += count;
197		size -= count;
198	}
199
200	return nbytes;
201}
202
203/* Interrupt handling */
204
205/* Initialize Interrupts */
206void ml_init_interrupt(void)
207{
208	(void) ml_set_interrupts_enabled(TRUE);
209}
210
211
212/* Get Interrupts Enabled */
213boolean_t ml_get_interrupts_enabled(void)
214{
215  unsigned long flags;
216
217  __asm__ volatile("pushf; pop	%0" :  "=r" (flags));
218  return (flags & EFL_IF) != 0;
219}
220
221/* Set Interrupts Enabled */
222boolean_t ml_set_interrupts_enabled(boolean_t enable)
223{
224	unsigned long flags;
225	boolean_t istate;
226
227	__asm__ volatile("pushf; pop	%0" :  "=r" (flags));
228
229	istate = ((flags & EFL_IF) != 0);
230
231	if (enable) {
232		__asm__ volatile("sti;nop");
233
234		if ((get_preemption_level() == 0) && (*ast_pending() & AST_URGENT))
235			__asm__ volatile ("int $0xff");
236	}
237	else {
238		if (istate)
239			__asm__ volatile("cli");
240	}
241
242	return istate;
243}
244
245/* Check if running at interrupt context */
246boolean_t ml_at_interrupt_context(void)
247{
248	return get_interrupt_level() != 0;
249}
250
251void ml_get_power_state(boolean_t *icp, boolean_t *pidlep) {
252	*icp = (get_interrupt_level() != 0);
253	/* These will be technically inaccurate for interrupts that occur
254	 * successively within a single "idle exit" event, but shouldn't
255	 * matter statistically.
256	 */
257	*pidlep = (current_cpu_datap()->lcpu.package->num_idle == topoParms.nLThreadsPerPackage);
258}
259
260/* Generate a fake interrupt */
261void ml_cause_interrupt(void)
262{
263	panic("ml_cause_interrupt not defined yet on Intel");
264}
265
266void ml_thread_policy(
267	thread_t thread,
268__unused	unsigned policy_id,
269	unsigned policy_info)
270{
271	if (policy_info & MACHINE_NETWORK_WORKLOOP) {
272		spl_t		s = splsched();
273
274		thread_lock(thread);
275
276		set_priority(thread, thread->priority + 1);
277
278		thread_unlock(thread);
279		splx(s);
280	}
281}
282
283/* Initialize Interrupts */
284void ml_install_interrupt_handler(
285	void *nub,
286	int source,
287	void *target,
288	IOInterruptHandler handler,
289	void *refCon)
290{
291	boolean_t current_state;
292
293	current_state = ml_get_interrupts_enabled();
294
295	PE_install_interrupt_handler(nub, source, target,
296	                             (IOInterruptHandler) handler, refCon);
297
298	(void) ml_set_interrupts_enabled(current_state);
299
300	initialize_screen(NULL, kPEAcquireScreen);
301}
302
303
304void
305machine_signal_idle(
306        processor_t processor)
307{
308	cpu_interrupt(processor->cpu_id);
309}
310
311static kern_return_t
312register_cpu(
313        uint32_t        lapic_id,
314	processor_t     *processor_out,
315	boolean_t       boot_cpu )
316{
317	int		target_cpu;
318	cpu_data_t	*this_cpu_datap;
319
320	this_cpu_datap = cpu_data_alloc(boot_cpu);
321	if (this_cpu_datap == NULL) {
322		return KERN_FAILURE;
323	}
324	target_cpu = this_cpu_datap->cpu_number;
325	assert((boot_cpu && (target_cpu == 0)) ||
326	      (!boot_cpu && (target_cpu != 0)));
327
328	lapic_cpu_map(lapic_id, target_cpu);
329
330	/* The cpu_id is not known at registration phase. Just do
331	 * lapic_id for now
332	 */
333	this_cpu_datap->cpu_phys_number = lapic_id;
334
335	this_cpu_datap->cpu_console_buf = console_cpu_alloc(boot_cpu);
336	if (this_cpu_datap->cpu_console_buf == NULL)
337		goto failed;
338
339	this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(boot_cpu);
340	if (this_cpu_datap->cpu_chud == NULL)
341		goto failed;
342
343	if (!boot_cpu) {
344		cpu_thread_alloc(this_cpu_datap->cpu_number);
345		if (this_cpu_datap->lcpu.core == NULL)
346			goto failed;
347
348#if NCOPY_WINDOWS > 0
349		this_cpu_datap->cpu_pmap = pmap_cpu_alloc(boot_cpu);
350		if (this_cpu_datap->cpu_pmap == NULL)
351			goto failed;
352#endif
353
354		this_cpu_datap->cpu_processor = cpu_processor_alloc(boot_cpu);
355		if (this_cpu_datap->cpu_processor == NULL)
356			goto failed;
357		/*
358		 * processor_init() deferred to topology start
359		 * because "slot numbers" a.k.a. logical processor numbers
360	 	 * are not yet finalized.
361		 */
362	}
363
364	*processor_out = this_cpu_datap->cpu_processor;
365
366	return KERN_SUCCESS;
367
368failed:
369	cpu_processor_free(this_cpu_datap->cpu_processor);
370#if NCOPY_WINDOWS > 0
371	pmap_cpu_free(this_cpu_datap->cpu_pmap);
372#endif
373	chudxnu_cpu_free(this_cpu_datap->cpu_chud);
374	console_cpu_free(this_cpu_datap->cpu_console_buf);
375	return KERN_FAILURE;
376}
377
378
379kern_return_t
380ml_processor_register(
381        cpu_id_t        cpu_id,
382        uint32_t        lapic_id,
383        processor_t     *processor_out,
384        boolean_t       boot_cpu,
385	boolean_t       start )
386{
387    static boolean_t done_topo_sort = FALSE;
388    static uint32_t num_registered = 0;
389
390    /* Register all CPUs first, and track max */
391    if( start == FALSE )
392    {
393	num_registered++;
394
395	DBG( "registering CPU lapic id %d\n", lapic_id );
396
397	return register_cpu( lapic_id, processor_out, boot_cpu );
398    }
399
400    /* Sort by topology before we start anything */
401    if( !done_topo_sort )
402    {
403	DBG( "about to start CPUs. %d registered\n", num_registered );
404
405	cpu_topology_sort( num_registered );
406	done_topo_sort = TRUE;
407    }
408
409    /* Assign the cpu ID */
410    uint32_t cpunum = -1;
411    cpu_data_t	*this_cpu_datap = NULL;
412
413    /* find cpu num and pointer */
414    cpunum = ml_get_cpuid( lapic_id );
415
416    if( cpunum == 0xFFFFFFFF ) /* never heard of it? */
417	panic( "trying to start invalid/unregistered CPU %d\n", lapic_id );
418
419    this_cpu_datap = cpu_datap(cpunum);
420
421    /* fix the CPU id */
422    this_cpu_datap->cpu_id = cpu_id;
423
424    /* output arg */
425    *processor_out = this_cpu_datap->cpu_processor;
426
427    /* OK, try and start this CPU */
428    return cpu_topology_start_cpu( cpunum );
429}
430
431
432void
433ml_cpu_get_info(ml_cpu_info_t *cpu_infop)
434{
435	boolean_t	os_supports_sse;
436	i386_cpu_info_t *cpuid_infop;
437
438	if (cpu_infop == NULL)
439		return;
440
441	/*
442	 * Are we supporting MMX/SSE/SSE2/SSE3?
443	 * As distinct from whether the cpu has these capabilities.
444	 */
445	os_supports_sse = !!(get_cr4() & CR4_OSXMM);
446
447	if (ml_fpu_avx_enabled())
448		cpu_infop->vector_unit = 9;
449	else if ((cpuid_features() & CPUID_FEATURE_SSE4_2) && os_supports_sse)
450		cpu_infop->vector_unit = 8;
451	else if ((cpuid_features() & CPUID_FEATURE_SSE4_1) && os_supports_sse)
452		cpu_infop->vector_unit = 7;
453	else if ((cpuid_features() & CPUID_FEATURE_SSSE3) && os_supports_sse)
454		cpu_infop->vector_unit = 6;
455	else if ((cpuid_features() & CPUID_FEATURE_SSE3) && os_supports_sse)
456		cpu_infop->vector_unit = 5;
457	else if ((cpuid_features() & CPUID_FEATURE_SSE2) && os_supports_sse)
458		cpu_infop->vector_unit = 4;
459	else if ((cpuid_features() & CPUID_FEATURE_SSE) && os_supports_sse)
460		cpu_infop->vector_unit = 3;
461	else if (cpuid_features() & CPUID_FEATURE_MMX)
462		cpu_infop->vector_unit = 2;
463	else
464		cpu_infop->vector_unit = 0;
465
466	cpuid_infop  = cpuid_info();
467
468	cpu_infop->cache_line_size = cpuid_infop->cache_linesize;
469
470	cpu_infop->l1_icache_size = cpuid_infop->cache_size[L1I];
471	cpu_infop->l1_dcache_size = cpuid_infop->cache_size[L1D];
472
473        if (cpuid_infop->cache_size[L2U] > 0) {
474            cpu_infop->l2_settings = 1;
475            cpu_infop->l2_cache_size = cpuid_infop->cache_size[L2U];
476        } else {
477            cpu_infop->l2_settings = 0;
478            cpu_infop->l2_cache_size = 0xFFFFFFFF;
479        }
480
481        if (cpuid_infop->cache_size[L3U] > 0) {
482            cpu_infop->l3_settings = 1;
483            cpu_infop->l3_cache_size = cpuid_infop->cache_size[L3U];
484        } else {
485            cpu_infop->l3_settings = 0;
486            cpu_infop->l3_cache_size = 0xFFFFFFFF;
487        }
488}
489
490void
491ml_init_max_cpus(unsigned long max_cpus)
492{
493        boolean_t current_state;
494
495        current_state = ml_set_interrupts_enabled(FALSE);
496        if (max_cpus_initialized != MAX_CPUS_SET) {
497                if (max_cpus > 0 && max_cpus <= MAX_CPUS) {
498			/*
499			 * Note: max_cpus is the number of enabled processors
500			 * that ACPI found; max_ncpus is the maximum number
501			 * that the kernel supports or that the "cpus="
502			 * boot-arg has set. Here we take int minimum.
503			 */
504                        machine_info.max_cpus = (integer_t)MIN(max_cpus, max_ncpus);
505		}
506                if (max_cpus_initialized == MAX_CPUS_WAIT)
507                        wakeup((event_t)&max_cpus_initialized);
508                max_cpus_initialized = MAX_CPUS_SET;
509        }
510        (void) ml_set_interrupts_enabled(current_state);
511}
512
513int
514ml_get_max_cpus(void)
515{
516        boolean_t current_state;
517
518        current_state = ml_set_interrupts_enabled(FALSE);
519        if (max_cpus_initialized != MAX_CPUS_SET) {
520                max_cpus_initialized = MAX_CPUS_WAIT;
521                assert_wait((event_t)&max_cpus_initialized, THREAD_UNINT);
522                (void)thread_block(THREAD_CONTINUE_NULL);
523        }
524        (void) ml_set_interrupts_enabled(current_state);
525        return(machine_info.max_cpus);
526}
527
528/*
529 *	Routine:        ml_init_lock_timeout
530 *	Function:
531 */
532void
533ml_init_lock_timeout(void)
534{
535	uint64_t	abstime;
536	uint32_t	mtxspin;
537	uint64_t	default_timeout_ns = NSEC_PER_SEC>>2;
538	uint32_t	slto;
539	uint32_t	prt;
540
541	if (PE_parse_boot_argn("slto_us", &slto, sizeof (slto)))
542		default_timeout_ns = slto * NSEC_PER_USEC;
543
544	/* LockTimeOut is absolutetime, LockTimeOutTSC is in TSC ticks */
545	nanoseconds_to_absolutetime(default_timeout_ns, &abstime);
546	LockTimeOut = (uint32_t) abstime;
547	LockTimeOutTSC = (uint32_t) tmrCvt(abstime, tscFCvtn2t);
548
549	if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof (mtxspin))) {
550		if (mtxspin > USEC_PER_SEC>>4)
551			mtxspin =  USEC_PER_SEC>>4;
552		nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime);
553	} else {
554		nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime);
555	}
556	MutexSpin = (unsigned int)abstime;
557
558	nanoseconds_to_absolutetime(4ULL * NSEC_PER_SEC, &LastDebuggerEntryAllowance);
559	if (PE_parse_boot_argn("panic_restart_timeout", &prt, sizeof (prt)))
560		nanoseconds_to_absolutetime(prt * NSEC_PER_SEC, &panic_restart_timeout);
561	virtualized = ((cpuid_features() & CPUID_FEATURE_VMM) != 0);
562	interrupt_latency_tracker_setup();
563}
564
565/*
566 * Threshold above which we should attempt to block
567 * instead of spinning for clock_delay_until().
568 */
569void
570ml_init_delay_spin_threshold(int threshold_us)
571{
572	nanoseconds_to_absolutetime(threshold_us * NSEC_PER_USEC, &delay_spin_threshold);
573}
574
575boolean_t
576ml_delay_should_spin(uint64_t interval)
577{
578	return (interval < delay_spin_threshold) ? TRUE : FALSE;
579}
580
581/*
582 * This is called from the machine-independent layer
583 * to perform machine-dependent info updates. Defer to cpu_thread_init().
584 */
585void
586ml_cpu_up(void)
587{
588	return;
589}
590
591/*
592 * This is called from the machine-independent layer
593 * to perform machine-dependent info updates.
594 */
595void
596ml_cpu_down(void)
597{
598	i386_deactivate_cpu();
599
600	return;
601}
602
603/*
604 * The following are required for parts of the kernel
605 * that cannot resolve these functions as inlines:
606 */
607extern thread_t current_act(void);
608thread_t
609current_act(void)
610{
611  return(current_thread_fast());
612}
613
614#undef current_thread
615extern thread_t current_thread(void);
616thread_t
617current_thread(void)
618{
619  return(current_thread_fast());
620}
621
622
623boolean_t ml_is64bit(void) {
624
625        return (cpu_mode_is64bit());
626}
627
628
629boolean_t ml_thread_is64bit(thread_t thread) {
630
631        return (thread_is_64bit(thread));
632}
633
634
635boolean_t ml_state_is64bit(void *saved_state) {
636
637	return is_saved_state64(saved_state);
638}
639
640void ml_cpu_set_ldt(int selector)
641{
642	/*
643	 * Avoid loading the LDT
644	 * if we're setting the KERNEL LDT and it's already set.
645	 */
646	if (selector == KERNEL_LDT &&
647	    current_cpu_datap()->cpu_ldt == KERNEL_LDT)
648		return;
649
650#if defined(__i386__)
651	/*
652 	 * If 64bit this requires a mode switch (and back).
653	 */
654	if (cpu_mode_is64bit())
655		ml_64bit_lldt(selector);
656	else
657		lldt(selector);
658#else
659	lldt(selector);
660#endif
661	current_cpu_datap()->cpu_ldt = selector;
662}
663
664void ml_fp_setvalid(boolean_t value)
665{
666        fp_setvalid(value);
667}
668
669uint64_t ml_cpu_int_event_time(void)
670{
671	return current_cpu_datap()->cpu_int_event_time;
672}
673
674vm_offset_t ml_stack_remaining(void)
675{
676	uintptr_t local = (uintptr_t) &local;
677
678	if (ml_at_interrupt_context() != 0) {
679	    return (local - (current_cpu_datap()->cpu_int_stack_top - INTSTACK_SIZE));
680	} else {
681	    return (local - current_thread()->kernel_stack);
682	}
683}
684
685void
686kernel_preempt_check(void)
687{
688	boolean_t	intr;
689	unsigned long flags;
690
691	assert(get_preemption_level() == 0);
692
693	__asm__ volatile("pushf; pop	%0" :  "=r" (flags));
694
695	intr = ((flags & EFL_IF) != 0);
696
697	if ((*ast_pending() & AST_URGENT) && intr == TRUE) {
698		/*
699		 * can handle interrupts and preemptions
700		 * at this point
701		 */
702
703		/*
704		 * now cause the PRE-EMPTION trap
705		 */
706		__asm__ volatile ("int %0" :: "N" (T_PREEMPT));
707	}
708}
709
710boolean_t machine_timeout_suspended(void) {
711	return (virtualized || pmap_tlb_flush_timeout || spinlock_timed_out || panic_active() || mp_recent_debugger_activity());
712}
713