1/*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach/kern_return.h>
30#include <kern/kalloc.h>
31#include <kern/cpu_number.h>
32#include <kern/cpu_data.h>
33#include <i386/mp.h>
34#include <i386/cpuid.h>
35#include <i386/proc_reg.h>
36#include <i386/mtrr.h>
37
38struct mtrr_var_range {
39	uint64_t  base;		/* in IA32_MTRR_PHYSBASE format */
40	uint64_t  mask;		/* in IA32_MTRR_PHYSMASK format */
41	uint32_t  refcnt;	/* var ranges reference count */
42};
43
44struct mtrr_fix_range {
45	uint64_t  types;	/* fixed-range type octet */
46};
47
48typedef struct mtrr_var_range mtrr_var_range_t;
49typedef struct mtrr_fix_range mtrr_fix_range_t;
50
51static struct {
52	uint64_t            MTRRcap;
53	uint64_t            MTRRdefType;
54	mtrr_var_range_t *  var_range;
55	unsigned int        var_count;
56	mtrr_fix_range_t    fix_range[11];
57} mtrr_state;
58
59static boolean_t mtrr_initialized = FALSE;
60
61decl_simple_lock_data(static, mtrr_lock);
62#define MTRR_LOCK()	simple_lock(&mtrr_lock);
63#define MTRR_UNLOCK()	simple_unlock(&mtrr_lock);
64
65#if	MTRR_DEBUG
66#define DBG(x...)	kprintf(x)
67#else
68#define DBG(x...)
69#endif
70
71/* Private functions */
72static void mtrr_get_var_ranges(mtrr_var_range_t * range, int count);
73static void mtrr_set_var_ranges(const mtrr_var_range_t * range, int count);
74static void mtrr_get_fix_ranges(mtrr_fix_range_t * range);
75static void mtrr_set_fix_ranges(const mtrr_fix_range_t * range);
76static void mtrr_update_setup(void * param);
77static void mtrr_update_teardown(void * param);
78static void mtrr_update_action(void * param);
79static void var_range_encode(mtrr_var_range_t * range, addr64_t address,
80                             uint64_t length, uint32_t type, int valid);
81static int  var_range_overlap(mtrr_var_range_t * range, addr64_t address,
82                              uint64_t length, uint32_t type);
83
84#define CACHE_CONTROL_MTRR		(NULL)
85#define CACHE_CONTROL_PAT		((void *)1)
86
87/*
88 * MTRR MSR bit fields.
89 */
90#define IA32_MTRR_DEF_TYPE_MT		0x000000ff
91#define IA32_MTRR_DEF_TYPE_FE		0x00000400
92#define IA32_MTRR_DEF_TYPE_E		0x00000800
93
94#define IA32_MTRRCAP_VCNT		0x000000ff
95#define IA32_MTRRCAP_FIX		0x00000100
96#define IA32_MTRRCAP_WC			0x00000400
97
98/* 0 < bits <= 64 */
99#define PHYS_BITS_TO_MASK(bits) \
100	((((1ULL << (bits-1)) - 1) << 1) | 1)
101
102/*
103 * Default mask for 36 physical address bits, this can
104 * change depending on the cpu model.
105 */
106static uint64_t mtrr_phys_mask = PHYS_BITS_TO_MASK(36);
107
108#define IA32_MTRR_PHYMASK_VALID		0x0000000000000800ULL
109#define IA32_MTRR_PHYSBASE_MASK		(mtrr_phys_mask & ~0xFFF)
110#define IA32_MTRR_PHYSBASE_TYPE		0x00000000000000FFULL
111
112/*
113 * Variable-range mask to/from length conversions.
114 */
115#define MASK_TO_LEN(mask) \
116	((~((mask) & IA32_MTRR_PHYSBASE_MASK) & mtrr_phys_mask) + 1)
117
118#define LEN_TO_MASK(len)  \
119	(~((len) - 1) & IA32_MTRR_PHYSBASE_MASK)
120
121#define LSB(x)		((x) & (~((x) - 1)))
122
123/*
124 * Fetch variable-range MTRR register pairs.
125 */
126static void
127mtrr_get_var_ranges(mtrr_var_range_t * range, int count)
128{
129	int i;
130
131	for (i = 0; i < count; i++) {
132		range[i].base = rdmsr64(MSR_IA32_MTRR_PHYSBASE(i));
133		range[i].mask = rdmsr64(MSR_IA32_MTRR_PHYSMASK(i));
134
135		/* bump ref count for firmware configured ranges */
136		if (range[i].mask & IA32_MTRR_PHYMASK_VALID)
137			range[i].refcnt = 1;
138		else
139			range[i].refcnt = 0;
140	}
141}
142
143/*
144 * Update variable-range MTRR register pairs.
145 */
146static void
147mtrr_set_var_ranges(const mtrr_var_range_t * range, int count)
148{
149	int i;
150
151	for (i = 0; i < count; i++) {
152		wrmsr64(MSR_IA32_MTRR_PHYSBASE(i), range[i].base);
153		wrmsr64(MSR_IA32_MTRR_PHYSMASK(i), range[i].mask);
154	}
155}
156
157/*
158 * Fetch all fixed-range MTRR's. Note MSR offsets are not consecutive.
159 */
160static void
161mtrr_get_fix_ranges(mtrr_fix_range_t * range)
162{
163	int i;
164
165	/* assume 11 fix range registers */
166	range[0].types = rdmsr64(MSR_IA32_MTRR_FIX64K_00000);
167	range[1].types = rdmsr64(MSR_IA32_MTRR_FIX16K_80000);
168	range[2].types = rdmsr64(MSR_IA32_MTRR_FIX16K_A0000);
169	for (i = 0; i < 8; i++)
170		range[3 + i].types = rdmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i);
171}
172
173/*
174 * Update all fixed-range MTRR's.
175 */
176static void
177mtrr_set_fix_ranges(const struct mtrr_fix_range * range)
178{
179	int i;
180
181	/* assume 11 fix range registers */
182	wrmsr64(MSR_IA32_MTRR_FIX64K_00000, range[0].types);
183	wrmsr64(MSR_IA32_MTRR_FIX16K_80000, range[1].types);
184	wrmsr64(MSR_IA32_MTRR_FIX16K_A0000, range[2].types);
185	for (i = 0; i < 8; i++)
186		wrmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i, range[3 + i].types);
187}
188
189#if MTRR_DEBUG
190static void
191mtrr_msr_dump(void)
192{
193	int i;
194	int count = rdmsr64(MSR_IA32_MTRRCAP) & IA32_MTRRCAP_VCNT;
195
196	DBG("VAR -- BASE -------------- MASK -------------- SIZE\n");
197	for (i = 0; i < count; i++) {
198		DBG(" %02x    0x%016llx  0x%016llx  0x%llx\n", i,
199		    rdmsr64(MSR_IA32_MTRR_PHYSBASE(i)),
200		    rdmsr64(MSR_IA32_MTRR_PHYSMASK(i)),
201		    MASK_TO_LEN(rdmsr64(MSR_IA32_MTRR_PHYSMASK(i))));
202	}
203	DBG("\n");
204
205	DBG("FIX64K_00000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX64K_00000));
206	DBG("FIX16K_80000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_80000));
207	DBG("FIX16K_A0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_A0000));
208	DBG(" FIX4K_C0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C0000));
209	DBG(" FIX4K_C8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C8000));
210	DBG(" FIX4K_D0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D0000));
211	DBG(" FIX4K_D8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D8000));
212	DBG(" FIX4K_E0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E0000));
213	DBG(" FIX4K_E8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E8000));
214	DBG(" FIX4K_F0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F0000));
215	DBG(" FIX4K_F8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F8000));
216
217	DBG("\nMTRRcap = 0x%llx MTRRdefType = 0x%llx\n",
218            rdmsr64(MSR_IA32_MTRRCAP), rdmsr64(MSR_IA32_MTRR_DEF_TYPE));
219}
220#endif /* MTRR_DEBUG */
221
222/*
223 * Called by the boot processor (BP) early during boot to initialize MTRR
224 * support.  The MTRR state on the BP is saved, any additional processors
225 * will have the same settings applied to ensure MTRR consistency.
226 */
227void
228mtrr_init(void)
229{
230	/* no reason to init more than once */
231	if (mtrr_initialized == TRUE)
232		return;
233
234	/* check for presence of MTRR feature on the processor */
235	if ((cpuid_features() & CPUID_FEATURE_MTRR) == 0)
236        	return;  /* no MTRR feature */
237
238	/* use a lock to serialize MTRR changes */
239	bzero((void *)&mtrr_state, sizeof(mtrr_state));
240	simple_lock_init(&mtrr_lock, 0);
241
242	mtrr_state.MTRRcap     = rdmsr64(MSR_IA32_MTRRCAP);
243	mtrr_state.MTRRdefType = rdmsr64(MSR_IA32_MTRR_DEF_TYPE);
244	mtrr_state.var_count   = mtrr_state.MTRRcap & IA32_MTRRCAP_VCNT;
245
246	/* allocate storage for variable ranges (can block?) */
247	if (mtrr_state.var_count) {
248		mtrr_state.var_range = (mtrr_var_range_t *)
249		                       kalloc(sizeof(mtrr_var_range_t) *
250		                              mtrr_state.var_count);
251		if (mtrr_state.var_range == NULL)
252			mtrr_state.var_count = 0;
253	}
254
255	/* fetch the initial firmware configured variable ranges */
256	if (mtrr_state.var_count)
257		mtrr_get_var_ranges(mtrr_state.var_range,
258				    mtrr_state.var_count);
259
260	/* fetch the initial firmware configured fixed ranges */
261	if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX)
262		mtrr_get_fix_ranges(mtrr_state.fix_range);
263
264	mtrr_initialized = TRUE;
265
266#if MTRR_DEBUG
267	mtrr_msr_dump();	/* dump firmware settings */
268#endif
269}
270
271/*
272 * Performs the Intel recommended procedure for changing the MTRR
273 * in a MP system. Leverage rendezvous mechanism for the required
274 * barrier synchronization among all processors. This function is
275 * called from the rendezvous IPI handler, and mtrr_update_cpu().
276 */
277static void
278mtrr_update_action(void * cache_control_type)
279{
280	uint32_t cr0, cr4;
281	uint32_t tmp;
282
283	cr0 = get_cr0();
284	cr4 = get_cr4();
285
286	/* enter no-fill cache mode */
287	tmp = cr0 | CR0_CD;
288	tmp &= ~CR0_NW;
289	set_cr0(tmp);
290
291	/* flush caches */
292	wbinvd();
293
294	/* clear the PGE flag in CR4 */
295	if (cr4 & CR4_PGE)
296		set_cr4(cr4 & ~CR4_PGE);
297
298	/* flush TLBs */
299	flush_tlb();
300
301	if (CACHE_CONTROL_PAT == cache_control_type) {
302		/* Change PA6 attribute field to WC */
303		uint64_t pat = rdmsr64(MSR_IA32_CR_PAT);
304		DBG("CPU%d PAT: was 0x%016llx\n", get_cpu_number(), pat);
305		pat &= ~(0x0FULL << 48);
306		pat |=  (0x01ULL << 48);
307		wrmsr64(MSR_IA32_CR_PAT, pat);
308		DBG("CPU%d PAT: is  0x%016llx\n",
309		    get_cpu_number(), rdmsr64(MSR_IA32_CR_PAT));
310	}
311	else {
312		/* disable all MTRR ranges */
313		wrmsr64(MSR_IA32_MTRR_DEF_TYPE,
314			mtrr_state.MTRRdefType & ~IA32_MTRR_DEF_TYPE_E);
315
316		/* apply MTRR settings */
317		if (mtrr_state.var_count)
318			mtrr_set_var_ranges(mtrr_state.var_range,
319					mtrr_state.var_count);
320
321		if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX)
322			mtrr_set_fix_ranges(mtrr_state.fix_range);
323
324		/* enable all MTRR range registers (what if E was not set?) */
325		wrmsr64(MSR_IA32_MTRR_DEF_TYPE,
326			mtrr_state.MTRRdefType | IA32_MTRR_DEF_TYPE_E);
327	}
328
329	/* flush all caches and TLBs a second time */
330	wbinvd();
331	flush_tlb();
332
333	/* restore normal cache mode */
334	set_cr0(cr0);
335
336	/* restore PGE flag */
337	if (cr4 & CR4_PGE)
338		set_cr4(cr4);
339
340	DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
341}
342
343static void
344mtrr_update_setup(__unused void * param_not_used)
345{
346	/* disable interrupts before the first barrier */
347	current_cpu_datap()->cpu_iflag = ml_set_interrupts_enabled(FALSE);
348	DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
349}
350
351static void
352mtrr_update_teardown(__unused void * param_not_used)
353{
354	/* restore interrupt flag following MTRR changes */
355	ml_set_interrupts_enabled(current_cpu_datap()->cpu_iflag);
356	DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
357}
358
359/*
360 * Update MTRR settings on all processors.
361 */
362kern_return_t
363mtrr_update_all_cpus(void)
364{
365	if (mtrr_initialized == FALSE)
366		return KERN_NOT_SUPPORTED;
367
368	MTRR_LOCK();
369	mp_rendezvous(mtrr_update_setup,
370		      mtrr_update_action,
371		      mtrr_update_teardown, NULL);
372	MTRR_UNLOCK();
373
374	return KERN_SUCCESS;
375}
376
377/*
378 * Update a single CPU with the current MTRR settings. Can be called
379 * during slave processor initialization to mirror the MTRR settings
380 * discovered on the boot processor by mtrr_init().
381 */
382kern_return_t
383mtrr_update_cpu(void)
384{
385	if (mtrr_initialized == FALSE)
386		return KERN_NOT_SUPPORTED;
387
388	MTRR_LOCK();
389	mtrr_update_setup(NULL);
390	mtrr_update_action(NULL);
391	mtrr_update_teardown(NULL);
392	MTRR_UNLOCK();
393
394	return KERN_SUCCESS;
395}
396
397/*
398 * Add a MTRR range to associate the physical memory range specified
399 * with a given memory caching type.
400 */
401kern_return_t
402mtrr_range_add(addr64_t address, uint64_t length, uint32_t type)
403{
404	mtrr_var_range_t * vr;
405	mtrr_var_range_t * free_range;
406	kern_return_t      ret = KERN_NO_SPACE;
407	int                overlap;
408	unsigned int       i;
409
410	DBG("mtrr_range_add base = 0x%llx, size = 0x%llx, type = %d\n",
411            address, length, type);
412
413	if (mtrr_initialized == FALSE) {
414		return KERN_NOT_SUPPORTED;
415	}
416
417	/* check memory type (GPF exception for undefined types) */
418	if ((type != MTRR_TYPE_UNCACHEABLE)  &&
419	    (type != MTRR_TYPE_WRITECOMBINE) &&
420	    (type != MTRR_TYPE_WRITETHROUGH) &&
421	    (type != MTRR_TYPE_WRITEPROTECT) &&
422	    (type != MTRR_TYPE_WRITEBACK)) {
423		return KERN_INVALID_ARGUMENT;
424	}
425
426	/* check WC support if requested */
427	if ((type == MTRR_TYPE_WRITECOMBINE) &&
428	    (mtrr_state.MTRRcap & IA32_MTRRCAP_WC) == 0) {
429		return KERN_NOT_SUPPORTED;
430	}
431
432	/* leave the fix range area below 1MB alone */
433	if (address < 0x100000 || mtrr_state.var_count == 0) {
434		return KERN_NOT_SUPPORTED;
435	}
436
437	/*
438	 * Length must be a power of 2 given by 2^n, where n >= 12.
439	 * Base address alignment must be larger than or equal to length.
440	 */
441	if ((length < 0x1000)       ||
442	    (LSB(length) != length) ||
443            (address && (length > LSB(address)))) {
444		return KERN_INVALID_ARGUMENT;
445	}
446
447	MTRR_LOCK();
448
449	/*
450	 * Check for overlap and locate a free range.
451	 */
452	for (i = 0, free_range = NULL; i < mtrr_state.var_count; i++)
453	{
454		vr = &mtrr_state.var_range[i];
455
456		if (vr->refcnt == 0) {
457			/* free range candidate if no overlaps are found */
458			free_range = vr;
459			continue;
460		}
461
462		overlap = var_range_overlap(vr, address, length, type);
463		if (overlap > 0) {
464			/*
465			 * identical overlap permitted, increment ref count.
466			 * no hardware update required.
467			 */
468			free_range = vr;
469			break;
470		}
471		if (overlap < 0) {
472			/* unsupported overlapping of memory types */
473			free_range = NULL;
474			break;
475		}
476	}
477
478	if (free_range) {
479		if (free_range->refcnt++ == 0) {
480			var_range_encode(free_range, address, length, type, 1);
481			mp_rendezvous(mtrr_update_setup,
482				      mtrr_update_action,
483				      mtrr_update_teardown, NULL);
484		}
485		ret = KERN_SUCCESS;
486	}
487
488#if MTRR_DEBUG
489	mtrr_msr_dump();
490#endif
491
492	MTRR_UNLOCK();
493
494	return ret;
495}
496
497/*
498 * Remove a previously added MTRR range. The same arguments used for adding
499 * the memory range must be supplied again.
500 */
501kern_return_t
502mtrr_range_remove(addr64_t address, uint64_t length, uint32_t type)
503{
504	mtrr_var_range_t * vr;
505	int                result = KERN_FAILURE;
506	int                cpu_update = 0;
507	unsigned int       i;
508
509	DBG("mtrr_range_remove base = 0x%llx, size = 0x%llx, type = %d\n",
510            address, length, type);
511
512	if (mtrr_initialized == FALSE) {
513		return KERN_NOT_SUPPORTED;
514	}
515
516	MTRR_LOCK();
517
518	for (i = 0; i < mtrr_state.var_count; i++) {
519		vr = &mtrr_state.var_range[i];
520
521		if (vr->refcnt &&
522		    var_range_overlap(vr, address, length, type) > 0) {
523			/* found specified variable range */
524			if (--mtrr_state.var_range[i].refcnt == 0) {
525				var_range_encode(vr, address, length, type, 0);
526				cpu_update = 1;
527			}
528			result = KERN_SUCCESS;
529			break;
530		}
531	}
532
533	if (cpu_update) {
534		mp_rendezvous(mtrr_update_setup,
535			      mtrr_update_action,
536			      mtrr_update_teardown, NULL);
537		result = KERN_SUCCESS;
538	}
539
540#if MTRR_DEBUG
541	mtrr_msr_dump();
542#endif
543
544	MTRR_UNLOCK();
545
546	return result;
547}
548
549/*
550 * Variable range helper routines
551 */
552static void
553var_range_encode(mtrr_var_range_t * range, addr64_t address,
554		 uint64_t length, uint32_t type, int valid)
555{
556	range->base = (address & IA32_MTRR_PHYSBASE_MASK) |
557		      (type    & IA32_MTRR_PHYSBASE_TYPE);
558
559	range->mask = LEN_TO_MASK(length) |
560		      (valid ? IA32_MTRR_PHYMASK_VALID : 0);
561}
562
563static int
564var_range_overlap(mtrr_var_range_t * range, addr64_t address,
565		  uint64_t length, uint32_t type)
566{
567	uint64_t  v_address, v_length;
568	uint32_t  v_type;
569	int       result = 0;  /* no overlap, or overlap ok */
570
571	v_address = range->base & IA32_MTRR_PHYSBASE_MASK;
572	v_type    = range->base & IA32_MTRR_PHYSBASE_TYPE;
573	v_length  = MASK_TO_LEN(range->mask);
574
575	/* detect range overlap */
576	if ((v_address >= address && v_address < (address + length)) ||
577	    (address >= v_address && address < (v_address + v_length))) {
578
579		if (v_address == address && v_length == length && v_type == type)
580			result = 1; /* identical overlap ok */
581		else if ( v_type == MTRR_TYPE_UNCACHEABLE &&
582			    type == MTRR_TYPE_UNCACHEABLE ) {
583			/* UC ranges can overlap */
584		}
585		else if ((v_type == MTRR_TYPE_UNCACHEABLE &&
586		            type == MTRR_TYPE_WRITEBACK)  ||
587			 (v_type == MTRR_TYPE_WRITEBACK &&
588			    type == MTRR_TYPE_UNCACHEABLE)) {
589			/* UC/WB can overlap - effective type becomes UC */
590		}
591		else {
592			/* anything else may cause undefined behavior */
593			result = -1;
594		}
595	}
596
597	return result;
598}
599
600/*
601 * Initialize PAT (Page Attribute Table)
602 */
603void
604pat_init(void)
605{
606	if (cpuid_features() & CPUID_FEATURE_PAT)
607	{
608		boolean_t istate = ml_set_interrupts_enabled(FALSE);
609		mtrr_update_action(CACHE_CONTROL_PAT);
610		ml_set_interrupts_enabled(istate);
611	}
612}
613