1/* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <mach/kern_return.h> 30#include <kern/kalloc.h> 31#include <kern/cpu_number.h> 32#include <kern/cpu_data.h> 33#include <i386/mp.h> 34#include <i386/cpuid.h> 35#include <i386/proc_reg.h> 36#include <i386/mtrr.h> 37 38struct mtrr_var_range { 39 uint64_t base; /* in IA32_MTRR_PHYSBASE format */ 40 uint64_t mask; /* in IA32_MTRR_PHYSMASK format */ 41 uint32_t refcnt; /* var ranges reference count */ 42}; 43 44struct mtrr_fix_range { 45 uint64_t types; /* fixed-range type octet */ 46}; 47 48typedef struct mtrr_var_range mtrr_var_range_t; 49typedef struct mtrr_fix_range mtrr_fix_range_t; 50 51static struct { 52 uint64_t MTRRcap; 53 uint64_t MTRRdefType; 54 mtrr_var_range_t * var_range; 55 unsigned int var_count; 56 mtrr_fix_range_t fix_range[11]; 57} mtrr_state; 58 59static boolean_t mtrr_initialized = FALSE; 60 61decl_simple_lock_data(static, mtrr_lock); 62#define MTRR_LOCK() simple_lock(&mtrr_lock); 63#define MTRR_UNLOCK() simple_unlock(&mtrr_lock); 64 65#if MTRR_DEBUG 66#define DBG(x...) kprintf(x) 67#else 68#define DBG(x...) 69#endif 70 71/* Private functions */ 72static void mtrr_get_var_ranges(mtrr_var_range_t * range, int count); 73static void mtrr_set_var_ranges(const mtrr_var_range_t * range, int count); 74static void mtrr_get_fix_ranges(mtrr_fix_range_t * range); 75static void mtrr_set_fix_ranges(const mtrr_fix_range_t * range); 76static void mtrr_update_setup(void * param); 77static void mtrr_update_teardown(void * param); 78static void mtrr_update_action(void * param); 79static void var_range_encode(mtrr_var_range_t * range, addr64_t address, 80 uint64_t length, uint32_t type, int valid); 81static int var_range_overlap(mtrr_var_range_t * range, addr64_t address, 82 uint64_t length, uint32_t type); 83 84#define CACHE_CONTROL_MTRR (NULL) 85#define CACHE_CONTROL_PAT ((void *)1) 86 87/* 88 * MTRR MSR bit fields. 89 */ 90#define IA32_MTRR_DEF_TYPE_MT 0x000000ff 91#define IA32_MTRR_DEF_TYPE_FE 0x00000400 92#define IA32_MTRR_DEF_TYPE_E 0x00000800 93 94#define IA32_MTRRCAP_VCNT 0x000000ff 95#define IA32_MTRRCAP_FIX 0x00000100 96#define IA32_MTRRCAP_WC 0x00000400 97 98/* 0 < bits <= 64 */ 99#define PHYS_BITS_TO_MASK(bits) \ 100 ((((1ULL << (bits-1)) - 1) << 1) | 1) 101 102/* 103 * Default mask for 36 physical address bits, this can 104 * change depending on the cpu model. 105 */ 106static uint64_t mtrr_phys_mask = PHYS_BITS_TO_MASK(36); 107 108#define IA32_MTRR_PHYMASK_VALID 0x0000000000000800ULL 109#define IA32_MTRR_PHYSBASE_MASK (mtrr_phys_mask & ~0xFFF) 110#define IA32_MTRR_PHYSBASE_TYPE 0x00000000000000FFULL 111 112/* 113 * Variable-range mask to/from length conversions. 114 */ 115#define MASK_TO_LEN(mask) \ 116 ((~((mask) & IA32_MTRR_PHYSBASE_MASK) & mtrr_phys_mask) + 1) 117 118#define LEN_TO_MASK(len) \ 119 (~((len) - 1) & IA32_MTRR_PHYSBASE_MASK) 120 121#define LSB(x) ((x) & (~((x) - 1))) 122 123/* 124 * Fetch variable-range MTRR register pairs. 125 */ 126static void 127mtrr_get_var_ranges(mtrr_var_range_t * range, int count) 128{ 129 int i; 130 131 for (i = 0; i < count; i++) { 132 range[i].base = rdmsr64(MSR_IA32_MTRR_PHYSBASE(i)); 133 range[i].mask = rdmsr64(MSR_IA32_MTRR_PHYSMASK(i)); 134 135 /* bump ref count for firmware configured ranges */ 136 if (range[i].mask & IA32_MTRR_PHYMASK_VALID) 137 range[i].refcnt = 1; 138 else 139 range[i].refcnt = 0; 140 } 141} 142 143/* 144 * Update variable-range MTRR register pairs. 145 */ 146static void 147mtrr_set_var_ranges(const mtrr_var_range_t * range, int count) 148{ 149 int i; 150 151 for (i = 0; i < count; i++) { 152 wrmsr64(MSR_IA32_MTRR_PHYSBASE(i), range[i].base); 153 wrmsr64(MSR_IA32_MTRR_PHYSMASK(i), range[i].mask); 154 } 155} 156 157/* 158 * Fetch all fixed-range MTRR's. Note MSR offsets are not consecutive. 159 */ 160static void 161mtrr_get_fix_ranges(mtrr_fix_range_t * range) 162{ 163 int i; 164 165 /* assume 11 fix range registers */ 166 range[0].types = rdmsr64(MSR_IA32_MTRR_FIX64K_00000); 167 range[1].types = rdmsr64(MSR_IA32_MTRR_FIX16K_80000); 168 range[2].types = rdmsr64(MSR_IA32_MTRR_FIX16K_A0000); 169 for (i = 0; i < 8; i++) 170 range[3 + i].types = rdmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i); 171} 172 173/* 174 * Update all fixed-range MTRR's. 175 */ 176static void 177mtrr_set_fix_ranges(const struct mtrr_fix_range * range) 178{ 179 int i; 180 181 /* assume 11 fix range registers */ 182 wrmsr64(MSR_IA32_MTRR_FIX64K_00000, range[0].types); 183 wrmsr64(MSR_IA32_MTRR_FIX16K_80000, range[1].types); 184 wrmsr64(MSR_IA32_MTRR_FIX16K_A0000, range[2].types); 185 for (i = 0; i < 8; i++) 186 wrmsr64(MSR_IA32_MTRR_FIX4K_C0000 + i, range[3 + i].types); 187} 188 189#if MTRR_DEBUG 190static void 191mtrr_msr_dump(void) 192{ 193 int i; 194 int count = rdmsr64(MSR_IA32_MTRRCAP) & IA32_MTRRCAP_VCNT; 195 196 DBG("VAR -- BASE -------------- MASK -------------- SIZE\n"); 197 for (i = 0; i < count; i++) { 198 DBG(" %02x 0x%016llx 0x%016llx 0x%llx\n", i, 199 rdmsr64(MSR_IA32_MTRR_PHYSBASE(i)), 200 rdmsr64(MSR_IA32_MTRR_PHYSMASK(i)), 201 MASK_TO_LEN(rdmsr64(MSR_IA32_MTRR_PHYSMASK(i)))); 202 } 203 DBG("\n"); 204 205 DBG("FIX64K_00000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX64K_00000)); 206 DBG("FIX16K_80000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_80000)); 207 DBG("FIX16K_A0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX16K_A0000)); 208 DBG(" FIX4K_C0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C0000)); 209 DBG(" FIX4K_C8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_C8000)); 210 DBG(" FIX4K_D0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D0000)); 211 DBG(" FIX4K_D8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_D8000)); 212 DBG(" FIX4K_E0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E0000)); 213 DBG(" FIX4K_E8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_E8000)); 214 DBG(" FIX4K_F0000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F0000)); 215 DBG(" FIX4K_F8000: 0x%016llx\n", rdmsr64(MSR_IA32_MTRR_FIX4K_F8000)); 216 217 DBG("\nMTRRcap = 0x%llx MTRRdefType = 0x%llx\n", 218 rdmsr64(MSR_IA32_MTRRCAP), rdmsr64(MSR_IA32_MTRR_DEF_TYPE)); 219} 220#endif /* MTRR_DEBUG */ 221 222/* 223 * Called by the boot processor (BP) early during boot to initialize MTRR 224 * support. The MTRR state on the BP is saved, any additional processors 225 * will have the same settings applied to ensure MTRR consistency. 226 */ 227void 228mtrr_init(void) 229{ 230 /* no reason to init more than once */ 231 if (mtrr_initialized == TRUE) 232 return; 233 234 /* check for presence of MTRR feature on the processor */ 235 if ((cpuid_features() & CPUID_FEATURE_MTRR) == 0) 236 return; /* no MTRR feature */ 237 238 /* use a lock to serialize MTRR changes */ 239 bzero((void *)&mtrr_state, sizeof(mtrr_state)); 240 simple_lock_init(&mtrr_lock, 0); 241 242 mtrr_state.MTRRcap = rdmsr64(MSR_IA32_MTRRCAP); 243 mtrr_state.MTRRdefType = rdmsr64(MSR_IA32_MTRR_DEF_TYPE); 244 mtrr_state.var_count = mtrr_state.MTRRcap & IA32_MTRRCAP_VCNT; 245 246 /* allocate storage for variable ranges (can block?) */ 247 if (mtrr_state.var_count) { 248 mtrr_state.var_range = (mtrr_var_range_t *) 249 kalloc(sizeof(mtrr_var_range_t) * 250 mtrr_state.var_count); 251 if (mtrr_state.var_range == NULL) 252 mtrr_state.var_count = 0; 253 } 254 255 /* fetch the initial firmware configured variable ranges */ 256 if (mtrr_state.var_count) 257 mtrr_get_var_ranges(mtrr_state.var_range, 258 mtrr_state.var_count); 259 260 /* fetch the initial firmware configured fixed ranges */ 261 if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX) 262 mtrr_get_fix_ranges(mtrr_state.fix_range); 263 264 mtrr_initialized = TRUE; 265 266#if MTRR_DEBUG 267 mtrr_msr_dump(); /* dump firmware settings */ 268#endif 269} 270 271/* 272 * Performs the Intel recommended procedure for changing the MTRR 273 * in a MP system. Leverage rendezvous mechanism for the required 274 * barrier synchronization among all processors. This function is 275 * called from the rendezvous IPI handler, and mtrr_update_cpu(). 276 */ 277static void 278mtrr_update_action(void * cache_control_type) 279{ 280 uint32_t cr0, cr4; 281 uint32_t tmp; 282 283 cr0 = get_cr0(); 284 cr4 = get_cr4(); 285 286 /* enter no-fill cache mode */ 287 tmp = cr0 | CR0_CD; 288 tmp &= ~CR0_NW; 289 set_cr0(tmp); 290 291 /* flush caches */ 292 wbinvd(); 293 294 /* clear the PGE flag in CR4 */ 295 if (cr4 & CR4_PGE) 296 set_cr4(cr4 & ~CR4_PGE); 297 298 /* flush TLBs */ 299 flush_tlb(); 300 301 if (CACHE_CONTROL_PAT == cache_control_type) { 302 /* Change PA6 attribute field to WC */ 303 uint64_t pat = rdmsr64(MSR_IA32_CR_PAT); 304 DBG("CPU%d PAT: was 0x%016llx\n", get_cpu_number(), pat); 305 pat &= ~(0x0FULL << 48); 306 pat |= (0x01ULL << 48); 307 wrmsr64(MSR_IA32_CR_PAT, pat); 308 DBG("CPU%d PAT: is 0x%016llx\n", 309 get_cpu_number(), rdmsr64(MSR_IA32_CR_PAT)); 310 } 311 else { 312 /* disable all MTRR ranges */ 313 wrmsr64(MSR_IA32_MTRR_DEF_TYPE, 314 mtrr_state.MTRRdefType & ~IA32_MTRR_DEF_TYPE_E); 315 316 /* apply MTRR settings */ 317 if (mtrr_state.var_count) 318 mtrr_set_var_ranges(mtrr_state.var_range, 319 mtrr_state.var_count); 320 321 if (mtrr_state.MTRRcap & IA32_MTRRCAP_FIX) 322 mtrr_set_fix_ranges(mtrr_state.fix_range); 323 324 /* enable all MTRR range registers (what if E was not set?) */ 325 wrmsr64(MSR_IA32_MTRR_DEF_TYPE, 326 mtrr_state.MTRRdefType | IA32_MTRR_DEF_TYPE_E); 327 } 328 329 /* flush all caches and TLBs a second time */ 330 wbinvd(); 331 flush_tlb(); 332 333 /* restore normal cache mode */ 334 set_cr0(cr0); 335 336 /* restore PGE flag */ 337 if (cr4 & CR4_PGE) 338 set_cr4(cr4); 339 340 DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__); 341} 342 343static void 344mtrr_update_setup(__unused void * param_not_used) 345{ 346 /* disable interrupts before the first barrier */ 347 current_cpu_datap()->cpu_iflag = ml_set_interrupts_enabled(FALSE); 348 DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__); 349} 350 351static void 352mtrr_update_teardown(__unused void * param_not_used) 353{ 354 /* restore interrupt flag following MTRR changes */ 355 ml_set_interrupts_enabled(current_cpu_datap()->cpu_iflag); 356 DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__); 357} 358 359/* 360 * Update MTRR settings on all processors. 361 */ 362kern_return_t 363mtrr_update_all_cpus(void) 364{ 365 if (mtrr_initialized == FALSE) 366 return KERN_NOT_SUPPORTED; 367 368 MTRR_LOCK(); 369 mp_rendezvous(mtrr_update_setup, 370 mtrr_update_action, 371 mtrr_update_teardown, NULL); 372 MTRR_UNLOCK(); 373 374 return KERN_SUCCESS; 375} 376 377/* 378 * Update a single CPU with the current MTRR settings. Can be called 379 * during slave processor initialization to mirror the MTRR settings 380 * discovered on the boot processor by mtrr_init(). 381 */ 382kern_return_t 383mtrr_update_cpu(void) 384{ 385 if (mtrr_initialized == FALSE) 386 return KERN_NOT_SUPPORTED; 387 388 MTRR_LOCK(); 389 mtrr_update_setup(NULL); 390 mtrr_update_action(NULL); 391 mtrr_update_teardown(NULL); 392 MTRR_UNLOCK(); 393 394 return KERN_SUCCESS; 395} 396 397/* 398 * Add a MTRR range to associate the physical memory range specified 399 * with a given memory caching type. 400 */ 401kern_return_t 402mtrr_range_add(addr64_t address, uint64_t length, uint32_t type) 403{ 404 mtrr_var_range_t * vr; 405 mtrr_var_range_t * free_range; 406 kern_return_t ret = KERN_NO_SPACE; 407 int overlap; 408 unsigned int i; 409 410 DBG("mtrr_range_add base = 0x%llx, size = 0x%llx, type = %d\n", 411 address, length, type); 412 413 if (mtrr_initialized == FALSE) { 414 return KERN_NOT_SUPPORTED; 415 } 416 417 /* check memory type (GPF exception for undefined types) */ 418 if ((type != MTRR_TYPE_UNCACHEABLE) && 419 (type != MTRR_TYPE_WRITECOMBINE) && 420 (type != MTRR_TYPE_WRITETHROUGH) && 421 (type != MTRR_TYPE_WRITEPROTECT) && 422 (type != MTRR_TYPE_WRITEBACK)) { 423 return KERN_INVALID_ARGUMENT; 424 } 425 426 /* check WC support if requested */ 427 if ((type == MTRR_TYPE_WRITECOMBINE) && 428 (mtrr_state.MTRRcap & IA32_MTRRCAP_WC) == 0) { 429 return KERN_NOT_SUPPORTED; 430 } 431 432 /* leave the fix range area below 1MB alone */ 433 if (address < 0x100000 || mtrr_state.var_count == 0) { 434 return KERN_NOT_SUPPORTED; 435 } 436 437 /* 438 * Length must be a power of 2 given by 2^n, where n >= 12. 439 * Base address alignment must be larger than or equal to length. 440 */ 441 if ((length < 0x1000) || 442 (LSB(length) != length) || 443 (address && (length > LSB(address)))) { 444 return KERN_INVALID_ARGUMENT; 445 } 446 447 MTRR_LOCK(); 448 449 /* 450 * Check for overlap and locate a free range. 451 */ 452 for (i = 0, free_range = NULL; i < mtrr_state.var_count; i++) 453 { 454 vr = &mtrr_state.var_range[i]; 455 456 if (vr->refcnt == 0) { 457 /* free range candidate if no overlaps are found */ 458 free_range = vr; 459 continue; 460 } 461 462 overlap = var_range_overlap(vr, address, length, type); 463 if (overlap > 0) { 464 /* 465 * identical overlap permitted, increment ref count. 466 * no hardware update required. 467 */ 468 free_range = vr; 469 break; 470 } 471 if (overlap < 0) { 472 /* unsupported overlapping of memory types */ 473 free_range = NULL; 474 break; 475 } 476 } 477 478 if (free_range) { 479 if (free_range->refcnt++ == 0) { 480 var_range_encode(free_range, address, length, type, 1); 481 mp_rendezvous(mtrr_update_setup, 482 mtrr_update_action, 483 mtrr_update_teardown, NULL); 484 } 485 ret = KERN_SUCCESS; 486 } 487 488#if MTRR_DEBUG 489 mtrr_msr_dump(); 490#endif 491 492 MTRR_UNLOCK(); 493 494 return ret; 495} 496 497/* 498 * Remove a previously added MTRR range. The same arguments used for adding 499 * the memory range must be supplied again. 500 */ 501kern_return_t 502mtrr_range_remove(addr64_t address, uint64_t length, uint32_t type) 503{ 504 mtrr_var_range_t * vr; 505 int result = KERN_FAILURE; 506 int cpu_update = 0; 507 unsigned int i; 508 509 DBG("mtrr_range_remove base = 0x%llx, size = 0x%llx, type = %d\n", 510 address, length, type); 511 512 if (mtrr_initialized == FALSE) { 513 return KERN_NOT_SUPPORTED; 514 } 515 516 MTRR_LOCK(); 517 518 for (i = 0; i < mtrr_state.var_count; i++) { 519 vr = &mtrr_state.var_range[i]; 520 521 if (vr->refcnt && 522 var_range_overlap(vr, address, length, type) > 0) { 523 /* found specified variable range */ 524 if (--mtrr_state.var_range[i].refcnt == 0) { 525 var_range_encode(vr, address, length, type, 0); 526 cpu_update = 1; 527 } 528 result = KERN_SUCCESS; 529 break; 530 } 531 } 532 533 if (cpu_update) { 534 mp_rendezvous(mtrr_update_setup, 535 mtrr_update_action, 536 mtrr_update_teardown, NULL); 537 result = KERN_SUCCESS; 538 } 539 540#if MTRR_DEBUG 541 mtrr_msr_dump(); 542#endif 543 544 MTRR_UNLOCK(); 545 546 return result; 547} 548 549/* 550 * Variable range helper routines 551 */ 552static void 553var_range_encode(mtrr_var_range_t * range, addr64_t address, 554 uint64_t length, uint32_t type, int valid) 555{ 556 range->base = (address & IA32_MTRR_PHYSBASE_MASK) | 557 (type & IA32_MTRR_PHYSBASE_TYPE); 558 559 range->mask = LEN_TO_MASK(length) | 560 (valid ? IA32_MTRR_PHYMASK_VALID : 0); 561} 562 563static int 564var_range_overlap(mtrr_var_range_t * range, addr64_t address, 565 uint64_t length, uint32_t type) 566{ 567 uint64_t v_address, v_length; 568 uint32_t v_type; 569 int result = 0; /* no overlap, or overlap ok */ 570 571 v_address = range->base & IA32_MTRR_PHYSBASE_MASK; 572 v_type = range->base & IA32_MTRR_PHYSBASE_TYPE; 573 v_length = MASK_TO_LEN(range->mask); 574 575 /* detect range overlap */ 576 if ((v_address >= address && v_address < (address + length)) || 577 (address >= v_address && address < (v_address + v_length))) { 578 579 if (v_address == address && v_length == length && v_type == type) 580 result = 1; /* identical overlap ok */ 581 else if ( v_type == MTRR_TYPE_UNCACHEABLE && 582 type == MTRR_TYPE_UNCACHEABLE ) { 583 /* UC ranges can overlap */ 584 } 585 else if ((v_type == MTRR_TYPE_UNCACHEABLE && 586 type == MTRR_TYPE_WRITEBACK) || 587 (v_type == MTRR_TYPE_WRITEBACK && 588 type == MTRR_TYPE_UNCACHEABLE)) { 589 /* UC/WB can overlap - effective type becomes UC */ 590 } 591 else { 592 /* anything else may cause undefined behavior */ 593 result = -1; 594 } 595 } 596 597 return result; 598} 599 600/* 601 * Initialize PAT (Page Attribute Table) 602 */ 603void 604pat_init(void) 605{ 606 if (cpuid_features() & CPUID_FEATURE_PAT) 607 { 608 boolean_t istate = ml_set_interrupts_enabled(FALSE); 609 mtrr_update_action(CACHE_CONTROL_PAT); 610 ml_set_interrupts_enabled(istate); 611 } 612} 613