1/*- 2 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> 3 * Copyright (c) 2017 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Konstantin Belousov 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/11/sys/x86/x86/x86_mem.c 314591 2017-03-03 10:30:30Z kib $"); 33 34#include <sys/param.h> 35#include <sys/kernel.h> 36#include <sys/systm.h> 37#include <sys/malloc.h> 38#include <sys/memrange.h> 39#include <sys/smp.h> 40#include <sys/sysctl.h> 41 42#include <vm/vm.h> 43#include <vm/vm_param.h> 44#include <vm/pmap.h> 45 46#include <machine/cputypes.h> 47#include <machine/md_var.h> 48#include <machine/specialreg.h> 49 50/* 51 * Pentium Pro+ memory range operations 52 * 53 * This code will probably be impenetrable without reference to the 54 * Intel Pentium Pro documentation or x86-64 programmers manual vol 2. 55 */ 56 57static char *mem_owner_bios = "BIOS"; 58 59#define MR686_FIXMTRR (1<<0) 60 61#define mrwithin(mr, a) \ 62 (((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len))) 63#define mroverlap(mra, mrb) \ 64 (mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base)) 65 66#define mrvalid(base, len) \ 67 ((!(base & ((1 << 12) - 1))) && /* base is multiple of 4k */ \ 68 ((len) >= (1 << 12)) && /* length is >= 4k */ \ 69 powerof2((len)) && /* ... and power of two */ \ 70 !((base) & ((len) - 1))) /* range is not discontiuous */ 71 72#define mrcopyflags(curr, new) \ 73 (((curr) & ~MDF_ATTRMASK) | ((new) & MDF_ATTRMASK)) 74 75static int mtrrs_disabled; 76SYSCTL_INT(_machdep, OID_AUTO, disable_mtrrs, CTLFLAG_RDTUN, 77 &mtrrs_disabled, 0, 78 "Disable MTRRs."); 79 80static void x86_mrinit(struct mem_range_softc *sc); 81static int x86_mrset(struct mem_range_softc *sc, 82 struct mem_range_desc *mrd, int *arg); 83static void x86_mrAPinit(struct mem_range_softc *sc); 84static void x86_mrreinit(struct mem_range_softc *sc); 85 86static struct mem_range_ops x86_mrops = { 87 x86_mrinit, 88 x86_mrset, 89 x86_mrAPinit, 90 x86_mrreinit 91}; 92 93/* XXX for AP startup hook */ 94static u_int64_t mtrrcap, mtrrdef; 95 96/* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */ 97static u_int64_t mtrr_physmask; 98 99static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc, 100 struct mem_range_desc *mrd); 101static void x86_mrfetch(struct mem_range_softc *sc); 102static int x86_mtrrtype(int flags); 103static int x86_mrt2mtrr(int flags, int oldval); 104static int x86_mtrrconflict(int flag1, int flag2); 105static void x86_mrstore(struct mem_range_softc *sc); 106static void x86_mrstoreone(void *arg); 107static struct mem_range_desc *x86_mtrrfixsearch(struct mem_range_softc *sc, 108 u_int64_t addr); 109static int x86_mrsetlow(struct mem_range_softc *sc, 110 struct mem_range_desc *mrd, int *arg); 111static int x86_mrsetvariable(struct mem_range_softc *sc, 112 struct mem_range_desc *mrd, int *arg); 113 114/* ia32 MTRR type to memory range type conversion */ 115static int x86_mtrrtomrt[] = { 116 MDF_UNCACHEABLE, 117 MDF_WRITECOMBINE, 118 MDF_UNKNOWN, 119 MDF_UNKNOWN, 120 MDF_WRITETHROUGH, 121 MDF_WRITEPROTECT, 122 MDF_WRITEBACK 123}; 124 125#define MTRRTOMRTLEN nitems(x86_mtrrtomrt) 126 127static int 128x86_mtrr2mrt(int val) 129{ 130 131 if (val < 0 || val >= MTRRTOMRTLEN) 132 return (MDF_UNKNOWN); 133 return (x86_mtrrtomrt[val]); 134} 135 136/* 137 * x86 MTRR conflicts. Writeback and uncachable may overlap. 138 */ 139static int 140x86_mtrrconflict(int flag1, int flag2) 141{ 142 143 flag1 &= MDF_ATTRMASK; 144 flag2 &= MDF_ATTRMASK; 145 if ((flag1 & MDF_UNKNOWN) || (flag2 & MDF_UNKNOWN)) 146 return (1); 147 if (flag1 == flag2 || 148 (flag1 == MDF_WRITEBACK && flag2 == MDF_UNCACHEABLE) || 149 (flag2 == MDF_WRITEBACK && flag1 == MDF_UNCACHEABLE)) 150 return (0); 151 return (1); 152} 153 154/* 155 * Look for an exactly-matching range. 156 */ 157static struct mem_range_desc * 158mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd) 159{ 160 struct mem_range_desc *cand; 161 int i; 162 163 for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++) 164 if ((cand->mr_base == mrd->mr_base) && 165 (cand->mr_len == mrd->mr_len)) 166 return (cand); 167 return (NULL); 168} 169 170/* 171 * Ensure that the direct map region does not contain any mappings 172 * that span MTRRs of different types. However, the fixed MTRRs can 173 * be ignored, because a large page mapping the first 1 MB of physical 174 * memory is a special case that the processor handles. Invalidate 175 * any old TLB entries that might hold inconsistent memory type 176 * information. 177 */ 178static void 179x86_mr_split_dmap(struct mem_range_softc *sc __unused) 180{ 181#ifdef __amd64__ 182 struct mem_range_desc *mrd; 183 int i; 184 185 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 186 mrd = sc->mr_desc + i; 187 for (; i < sc->mr_ndesc; i++, mrd++) { 188 if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE) 189 pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, TRUE); 190 } 191#endif 192} 193 194/* 195 * Fetch the current mtrr settings from the current CPU (assumed to 196 * all be in sync in the SMP case). Note that if we are here, we 197 * assume that MTRRs are enabled, and we may or may not have fixed 198 * MTRRs. 199 */ 200static void 201x86_mrfetch(struct mem_range_softc *sc) 202{ 203 struct mem_range_desc *mrd; 204 u_int64_t msrv; 205 int i, j, msr; 206 207 mrd = sc->mr_desc; 208 209 /* Get fixed-range MTRRs. */ 210 if (sc->mr_cap & MR686_FIXMTRR) { 211 msr = MSR_MTRR64kBase; 212 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 213 msrv = rdmsr(msr); 214 for (j = 0; j < 8; j++, mrd++) { 215 mrd->mr_flags = 216 (mrd->mr_flags & ~MDF_ATTRMASK) | 217 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 218 if (mrd->mr_owner[0] == 0) 219 strcpy(mrd->mr_owner, mem_owner_bios); 220 msrv = msrv >> 8; 221 } 222 } 223 msr = MSR_MTRR16kBase; 224 for (i = 0; i < MTRR_N16K / 8; i++, msr++) { 225 msrv = rdmsr(msr); 226 for (j = 0; j < 8; j++, mrd++) { 227 mrd->mr_flags = 228 (mrd->mr_flags & ~MDF_ATTRMASK) | 229 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 230 if (mrd->mr_owner[0] == 0) 231 strcpy(mrd->mr_owner, mem_owner_bios); 232 msrv = msrv >> 8; 233 } 234 } 235 msr = MSR_MTRR4kBase; 236 for (i = 0; i < MTRR_N4K / 8; i++, msr++) { 237 msrv = rdmsr(msr); 238 for (j = 0; j < 8; j++, mrd++) { 239 mrd->mr_flags = 240 (mrd->mr_flags & ~MDF_ATTRMASK) | 241 x86_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE; 242 if (mrd->mr_owner[0] == 0) 243 strcpy(mrd->mr_owner, mem_owner_bios); 244 msrv = msrv >> 8; 245 } 246 } 247 } 248 249 /* Get remainder which must be variable MTRRs. */ 250 msr = MSR_MTRRVarBase; 251 for (; mrd - sc->mr_desc < sc->mr_ndesc; msr += 2, mrd++) { 252 msrv = rdmsr(msr); 253 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 254 x86_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE); 255 mrd->mr_base = msrv & mtrr_physmask; 256 msrv = rdmsr(msr + 1); 257 mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ? 258 (mrd->mr_flags | MDF_ACTIVE) : 259 (mrd->mr_flags & ~MDF_ACTIVE); 260 261 /* Compute the range from the mask. Ick. */ 262 mrd->mr_len = (~(msrv & mtrr_physmask) & 263 (mtrr_physmask | 0xfff)) + 1; 264 if (!mrvalid(mrd->mr_base, mrd->mr_len)) 265 mrd->mr_flags |= MDF_BOGUS; 266 267 /* If unclaimed and active, must be the BIOS. */ 268 if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0)) 269 strcpy(mrd->mr_owner, mem_owner_bios); 270 } 271} 272 273/* 274 * Return the MTRR memory type matching a region's flags 275 */ 276static int 277x86_mtrrtype(int flags) 278{ 279 int i; 280 281 flags &= MDF_ATTRMASK; 282 283 for (i = 0; i < MTRRTOMRTLEN; i++) { 284 if (x86_mtrrtomrt[i] == MDF_UNKNOWN) 285 continue; 286 if (flags == x86_mtrrtomrt[i]) 287 return (i); 288 } 289 return (-1); 290} 291 292static int 293x86_mrt2mtrr(int flags, int oldval) 294{ 295 int val; 296 297 if ((val = x86_mtrrtype(flags)) == -1) 298 return (oldval & 0xff); 299 return (val & 0xff); 300} 301 302/* 303 * Update running CPU(s) MTRRs to match the ranges in the descriptor 304 * list. 305 * 306 * Must be called with interrupts enabled. 307 */ 308static void 309x86_mrstore(struct mem_range_softc *sc) 310{ 311 312 smp_rendezvous(NULL, x86_mrstoreone, NULL, sc); 313} 314 315/* 316 * Update the current CPU's MTRRs with those represented in the 317 * descriptor list. Note that we do this wholesale rather than just 318 * stuffing one entry; this is simpler (but slower, of course). 319 */ 320static void 321x86_mrstoreone(void *arg) 322{ 323 struct mem_range_softc *sc = arg; 324 struct mem_range_desc *mrd; 325 u_int64_t omsrv, msrv; 326 int i, j, msr; 327 u_long cr0, cr4; 328 329 mrd = sc->mr_desc; 330 331 critical_enter(); 332 333 /* Disable PGE. */ 334 cr4 = rcr4(); 335 load_cr4(cr4 & ~CR4_PGE); 336 337 /* Disable caches (CD = 1, NW = 0). */ 338 cr0 = rcr0(); 339 load_cr0((cr0 & ~CR0_NW) | CR0_CD); 340 341 /* Flushes caches and TLBs. */ 342 wbinvd(); 343 invltlb(); 344 345 /* Disable MTRRs (E = 0). */ 346 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~MTRR_DEF_ENABLE); 347 348 /* Set fixed-range MTRRs. */ 349 if (sc->mr_cap & MR686_FIXMTRR) { 350 msr = MSR_MTRR64kBase; 351 for (i = 0; i < MTRR_N64K / 8; i++, msr++) { 352 msrv = 0; 353 omsrv = rdmsr(msr); 354 for (j = 7; j >= 0; j--) { 355 msrv = msrv << 8; 356 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 357 omsrv >> (j * 8)); 358 } 359 wrmsr(msr, msrv); 360 mrd += 8; 361 } 362 msr = MSR_MTRR16kBase; 363 for (i = 0; i < MTRR_N16K / 8; i++, msr++) { 364 msrv = 0; 365 omsrv = rdmsr(msr); 366 for (j = 7; j >= 0; j--) { 367 msrv = msrv << 8; 368 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 369 omsrv >> (j * 8)); 370 } 371 wrmsr(msr, msrv); 372 mrd += 8; 373 } 374 msr = MSR_MTRR4kBase; 375 for (i = 0; i < MTRR_N4K / 8; i++, msr++) { 376 msrv = 0; 377 omsrv = rdmsr(msr); 378 for (j = 7; j >= 0; j--) { 379 msrv = msrv << 8; 380 msrv |= x86_mrt2mtrr((mrd + j)->mr_flags, 381 omsrv >> (j * 8)); 382 } 383 wrmsr(msr, msrv); 384 mrd += 8; 385 } 386 } 387 388 /* Set remainder which must be variable MTRRs. */ 389 msr = MSR_MTRRVarBase; 390 for (; mrd - sc->mr_desc < sc->mr_ndesc; msr += 2, mrd++) { 391 /* base/type register */ 392 omsrv = rdmsr(msr); 393 if (mrd->mr_flags & MDF_ACTIVE) { 394 msrv = mrd->mr_base & mtrr_physmask; 395 msrv |= x86_mrt2mtrr(mrd->mr_flags, omsrv); 396 } else { 397 msrv = 0; 398 } 399 wrmsr(msr, msrv); 400 401 /* mask/active register */ 402 if (mrd->mr_flags & MDF_ACTIVE) { 403 msrv = MTRR_PHYSMASK_VALID | 404 rounddown2(mtrr_physmask, mrd->mr_len); 405 } else { 406 msrv = 0; 407 } 408 wrmsr(msr + 1, msrv); 409 } 410 411 /* Flush caches and TLBs. */ 412 wbinvd(); 413 invltlb(); 414 415 /* Enable MTRRs. */ 416 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | MTRR_DEF_ENABLE); 417 418 /* Restore caches and PGE. */ 419 load_cr0(cr0); 420 load_cr4(cr4); 421 422 critical_exit(); 423} 424 425/* 426 * Hunt for the fixed MTRR referencing (addr) 427 */ 428static struct mem_range_desc * 429x86_mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr) 430{ 431 struct mem_range_desc *mrd; 432 int i; 433 434 for (i = 0, mrd = sc->mr_desc; i < MTRR_N64K + MTRR_N16K + MTRR_N4K; 435 i++, mrd++) 436 if (addr >= mrd->mr_base && 437 addr < mrd->mr_base + mrd->mr_len) 438 return (mrd); 439 return (NULL); 440} 441 442/* 443 * Try to satisfy the given range request by manipulating the fixed 444 * MTRRs that cover low memory. 445 * 446 * Note that we try to be generous here; we'll bloat the range out to 447 * the next higher/lower boundary to avoid the consumer having to know 448 * too much about the mechanisms here. 449 * 450 * XXX note that this will have to be updated when we start supporting 451 * "busy" ranges. 452 */ 453static int 454x86_mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 455{ 456 struct mem_range_desc *first_md, *last_md, *curr_md; 457 458 /* Range check. */ 459 if ((first_md = x86_mtrrfixsearch(sc, mrd->mr_base)) == NULL || 460 (last_md = x86_mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) 461 == NULL) 462 return (EINVAL); 463 464 /* Check that we aren't doing something risky. */ 465 if ((mrd->mr_flags & MDF_FORCE) == 0) { 466 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 467 if ((curr_md->mr_flags & MDF_ATTRMASK) == MDF_UNKNOWN) 468 return (EACCES); 469 } 470 } 471 472 /* Set flags, clear set-by-firmware flag. */ 473 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 474 curr_md->mr_flags = mrcopyflags(curr_md->mr_flags & 475 ~MDF_FIRMWARE, mrd->mr_flags); 476 bcopy(mrd->mr_owner, curr_md->mr_owner, sizeof(mrd->mr_owner)); 477 } 478 479 return (0); 480} 481 482/* 483 * Modify/add a variable MTRR to satisfy the request. 484 * 485 * XXX needs to be updated to properly support "busy" ranges. 486 */ 487static int 488x86_mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd, 489 int *arg) 490{ 491 struct mem_range_desc *curr_md, *free_md; 492 int i; 493 494 /* 495 * Scan the currently active variable descriptors, look for 496 * one we exactly match (straight takeover) and for possible 497 * accidental overlaps. 498 * 499 * Keep track of the first empty variable descriptor in case 500 * we can't perform a takeover. 501 */ 502 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 503 curr_md = sc->mr_desc + i; 504 free_md = NULL; 505 for (; i < sc->mr_ndesc; i++, curr_md++) { 506 if (curr_md->mr_flags & MDF_ACTIVE) { 507 /* Exact match? */ 508 if (curr_md->mr_base == mrd->mr_base && 509 curr_md->mr_len == mrd->mr_len) { 510 511 /* Whoops, owned by someone. */ 512 if (curr_md->mr_flags & MDF_BUSY) 513 return (EBUSY); 514 515 /* Check that we aren't doing something risky */ 516 if (!(mrd->mr_flags & MDF_FORCE) && 517 (curr_md->mr_flags & MDF_ATTRMASK) == 518 MDF_UNKNOWN) 519 return (EACCES); 520 521 /* Ok, just hijack this entry. */ 522 free_md = curr_md; 523 break; 524 } 525 526 /* Non-exact overlap? */ 527 if (mroverlap(curr_md, mrd)) { 528 /* Between conflicting region types? */ 529 if (x86_mtrrconflict(curr_md->mr_flags, 530 mrd->mr_flags)) 531 return (EINVAL); 532 } 533 } else if (free_md == NULL) { 534 free_md = curr_md; 535 } 536 } 537 538 /* Got somewhere to put it? */ 539 if (free_md == NULL) 540 return (ENOSPC); 541 542 /* Set up new descriptor. */ 543 free_md->mr_base = mrd->mr_base; 544 free_md->mr_len = mrd->mr_len; 545 free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags); 546 bcopy(mrd->mr_owner, free_md->mr_owner, sizeof(mrd->mr_owner)); 547 return (0); 548} 549 550/* 551 * Handle requests to set memory range attributes by manipulating MTRRs. 552 */ 553static int 554x86_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 555{ 556 struct mem_range_desc *targ; 557 int error; 558 559 switch (*arg) { 560 case MEMRANGE_SET_UPDATE: 561 /* 562 * Make sure that what's being asked for is even 563 * possible at all. 564 */ 565 if (!mrvalid(mrd->mr_base, mrd->mr_len) || 566 x86_mtrrtype(mrd->mr_flags) == -1) 567 return (EINVAL); 568 569#define FIXTOP \ 570 ((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + (MTRR_N4K * 0x1000)) 571 572 /* Are the "low memory" conditions applicable? */ 573 if ((sc->mr_cap & MR686_FIXMTRR) != 0 && 574 mrd->mr_base + mrd->mr_len <= FIXTOP) { 575 if ((error = x86_mrsetlow(sc, mrd, arg)) != 0) 576 return (error); 577 } else { 578 /* It's time to play with variable MTRRs. */ 579 if ((error = x86_mrsetvariable(sc, mrd, arg)) != 0) 580 return (error); 581 } 582 break; 583 584 case MEMRANGE_SET_REMOVE: 585 if ((targ = mem_range_match(sc, mrd)) == NULL) 586 return (ENOENT); 587 if (targ->mr_flags & MDF_FIXACTIVE) 588 return (EPERM); 589 if (targ->mr_flags & MDF_BUSY) 590 return (EBUSY); 591 targ->mr_flags &= ~MDF_ACTIVE; 592 targ->mr_owner[0] = 0; 593 break; 594 595 default: 596 return (EOPNOTSUPP); 597 } 598 599 x86_mr_split_dmap(sc); 600 601 /* Update the hardware. */ 602 x86_mrstore(sc); 603 604 /* Refetch to see where we're at. */ 605 x86_mrfetch(sc); 606 return (0); 607} 608 609/* 610 * Work out how many ranges we support, initialise storage for them, 611 * and fetch the initial settings. 612 */ 613static void 614x86_mrinit(struct mem_range_softc *sc) 615{ 616 struct mem_range_desc *mrd; 617 int i, nmdesc; 618 619 if (sc->mr_desc != NULL) 620 /* Already initialized. */ 621 return; 622 623 nmdesc = 0; 624 mtrrcap = rdmsr(MSR_MTRRcap); 625 mtrrdef = rdmsr(MSR_MTRRdefType); 626 627 /* For now, bail out if MTRRs are not enabled. */ 628 if (!(mtrrdef & MTRR_DEF_ENABLE)) { 629 if (bootverbose) 630 printf("CPU supports MTRRs but not enabled\n"); 631 return; 632 } 633 nmdesc = mtrrcap & MTRR_CAP_VCNT; 634 if (bootverbose) 635 printf("Pentium Pro MTRR support enabled\n"); 636 637 /* 638 * Determine the size of the PhysMask and PhysBase fields in 639 * the variable range MTRRs. 640 */ 641 mtrr_physmask = (((uint64_t)1 << cpu_maxphyaddr) - 1) & 642 ~(uint64_t)0xfff; 643 644 /* If fixed MTRRs supported and enabled. */ 645 if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) { 646 sc->mr_cap = MR686_FIXMTRR; 647 nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K; 648 } 649 650 sc->mr_desc = malloc(nmdesc * sizeof(struct mem_range_desc), M_MEMDESC, 651 M_WAITOK | M_ZERO); 652 sc->mr_ndesc = nmdesc; 653 654 mrd = sc->mr_desc; 655 656 /* Populate the fixed MTRR entries' base/length. */ 657 if (sc->mr_cap & MR686_FIXMTRR) { 658 for (i = 0; i < MTRR_N64K; i++, mrd++) { 659 mrd->mr_base = i * 0x10000; 660 mrd->mr_len = 0x10000; 661 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 662 MDF_FIXACTIVE; 663 } 664 for (i = 0; i < MTRR_N16K; i++, mrd++) { 665 mrd->mr_base = i * 0x4000 + 0x80000; 666 mrd->mr_len = 0x4000; 667 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 668 MDF_FIXACTIVE; 669 } 670 for (i = 0; i < MTRR_N4K; i++, mrd++) { 671 mrd->mr_base = i * 0x1000 + 0xc0000; 672 mrd->mr_len = 0x1000; 673 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | 674 MDF_FIXACTIVE; 675 } 676 } 677 678 /* 679 * Get current settings, anything set now is considered to 680 * have been set by the firmware. (XXX has something already 681 * played here?) 682 */ 683 x86_mrfetch(sc); 684 mrd = sc->mr_desc; 685 for (i = 0; i < sc->mr_ndesc; i++, mrd++) { 686 if (mrd->mr_flags & MDF_ACTIVE) 687 mrd->mr_flags |= MDF_FIRMWARE; 688 } 689 690 x86_mr_split_dmap(sc); 691} 692 693/* 694 * Initialise MTRRs on an AP after the BSP has run the init code. 695 */ 696static void 697x86_mrAPinit(struct mem_range_softc *sc) 698{ 699 700 x86_mrstoreone(sc); 701 wrmsr(MSR_MTRRdefType, mtrrdef); 702} 703 704/* 705 * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor 706 * list. 707 * 708 * Must be called with interrupts enabled. 709 */ 710static void 711x86_mrreinit(struct mem_range_softc *sc) 712{ 713 714 smp_rendezvous(NULL, (void (*)(void *))x86_mrAPinit, NULL, sc); 715} 716 717static void 718x86_mem_drvinit(void *unused) 719{ 720 721 if (mtrrs_disabled) 722 return; 723 if (!(cpu_feature & CPUID_MTRR)) 724 return; 725 mem_range_softc.mr_op = &x86_mrops; 726 x86_mrinit(&mem_range_softc); 727} 728SYSINIT(x86memdev, SI_SUB_CPU, SI_ORDER_ANY, x86_mem_drvinit, NULL); 729