x86_mem.c revision 45405
1/*- 2 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $Id$ 27 */ 28 29#include "opt_smp.h" 30 31#include <sys/param.h> 32#include <sys/kernel.h> 33#include <sys/systm.h> 34#include <sys/ioccom.h> 35#include <sys/malloc.h> 36#include <sys/memrange.h> 37 38#include <machine/md_var.h> 39#include <machine/specialreg.h> 40 41/* 42 * i686 memory range operations 43 * 44 * This code will probably be impenetrable without reference to the 45 * Intel Pentium Pro documentation. 46 */ 47 48static char *mem_owner_bios = "BIOS"; 49 50#define MR686_FIXMTRR (1<<0) 51 52#define mrwithin(mr, a) \ 53 (((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len))) 54#define mroverlap(mra, mrb) \ 55 (mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base)) 56 57#define mrvalid(base, len) \ 58 ((!(base & ((1 << 12) - 1))) && /* base is multiple of 4k */ \ 59 ((len) >= (1 << 12)) && /* length is >= 4k */ \ 60 powerof2((len)) && /* ... and power of two */ \ 61 !((base) & ((len) - 1))) /* range is not discontiuous */ 62 63#define mrcopyflags(curr, new) (((curr) & ~MDF_ATTRMASK) | ((new) & MDF_ATTRMASK)) 64 65static void i686_mrinit(struct mem_range_softc *); 66static int i686_mrset(struct mem_range_softc *, 67 struct mem_range_desc *, 68 int *); 69 70static struct mem_range_ops i686_mrops = { 71 i686_mrinit, 72 i686_mrset 73}; 74 75static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc, 76 struct mem_range_desc *mrd); 77static void i686_mrfetch(struct mem_range_softc *sc); 78static int i686_mtrrtype(int flags); 79static int i686_mrstore(struct mem_range_softc *sc); 80static struct mem_range_desc *i686_mtrrfixsearch(struct mem_range_softc *sc, 81 u_int64_t addr); 82static int i686_mrsetlow(struct mem_range_softc *sc, 83 struct mem_range_desc *mrd, 84 int *arg); 85static int i686_mrsetvariable(struct mem_range_softc *sc, 86 struct mem_range_desc *mrd, 87 int *arg); 88 89/* i686 MTRR type to memory range type conversion */ 90static int i686_mtrrtomrt[] = { 91 MDF_UNCACHEABLE, 92 MDF_WRITECOMBINE, 93 0, 94 0, 95 MDF_WRITETHROUGH, 96 MDF_WRITEPROTECT, 97 MDF_WRITEBACK 98}; 99 100/* MTRR type to text conversion */ 101static char *i686_mtrrtotext[] = { 102 "uncacheable", 103 "write-combine", 104 "invalid", 105 "invalid", 106 "write-through" 107 "write-protect", 108 "write-back" 109}; 110 111/* 112 * Look for an exactly-matching range. 113 */ 114static struct mem_range_desc * 115mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd) 116{ 117 struct mem_range_desc *cand; 118 int i; 119 120 for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++) 121 if ((cand->mr_base == mrd->mr_base) && 122 (cand->mr_len == mrd->mr_len)) 123 return(cand); 124 return(NULL); 125} 126 127/* 128 * Fetch the current mtrr settings from the current CPU (assumed to all 129 * be in sync in the SMP case). Note that if we are here, we assume 130 * that MTRRs are enabled, and we may or may not have fixed MTRRs. 131 */ 132static void 133i686_mrfetch(struct mem_range_softc *sc) 134{ 135 struct mem_range_desc *mrd; 136 u_int64_t msrv; 137 int i, j, msr; 138 139 mrd = sc->mr_desc; 140 141 /* Get fixed-range MTRRs */ 142 if (sc->mr_cap & MR686_FIXMTRR) { 143 msr = MSR_MTRR64kBase; 144 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 145 msrv = rdmsr(msr); 146 for (j = 0; j < 8; j++, mrd++) { 147 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 148 i686_mtrrtomrt[msrv & 0xff] | 149 MDF_ACTIVE; 150 if (mrd->mr_owner[0] == 0) 151 strcpy(mrd->mr_owner, mem_owner_bios); 152 msrv = msrv >> 8; 153 } 154 } 155 msr = MSR_MTRR16kBase; 156 for (i = 0; i < (MTRR_N16K / 8); i++, msr++) { 157 msrv = rdmsr(msr); 158 for (j = 0; j < 8; j++, mrd++) { 159 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 160 i686_mtrrtomrt[msrv & 0xff] | 161 MDF_ACTIVE; 162 if (mrd->mr_owner[0] == 0) 163 strcpy(mrd->mr_owner, mem_owner_bios); 164 msrv = msrv >> 8; 165 } 166 } 167 msr = MSR_MTRR4kBase; 168 for (i = 0; i < (MTRR_N4K / 8); i++, msr++) { 169 msrv = rdmsr(msr); 170 for (j = 0; j < 8; j++, mrd++) { 171 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 172 i686_mtrrtomrt[msrv & 0xff] | 173 MDF_ACTIVE; 174 if (mrd->mr_owner[0] == 0) 175 strcpy(mrd->mr_owner, mem_owner_bios); 176 msrv = msrv >> 8; 177 } 178 } 179 } 180 181 /* Get remainder which must be variable MTRRs */ 182 msr = MSR_MTRRVarBase; 183 for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) { 184 msrv = rdmsr(msr); 185 mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) | 186 i686_mtrrtomrt[msrv & 0xff]; 187 mrd->mr_base = msrv & 0x0000000ffffff000LL; 188 msrv = rdmsr(msr + 1); 189 mrd->mr_flags = (msrv & 0x800) ? 190 (mrd->mr_flags | MDF_ACTIVE) : 191 (mrd->mr_flags & ~MDF_ACTIVE); 192 /* Compute the range from the mask. Ick. */ 193 mrd->mr_len = (~(msrv & 0x0000000ffffff000LL) & 0x0000000fffffffffLL) + 1; 194 if (!mrvalid(mrd->mr_base, mrd->mr_len)) 195 mrd->mr_flags |= MDF_BOGUS; 196 /* If unclaimed and active, must be the BIOS */ 197 if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0)) 198 strcpy(mrd->mr_owner, mem_owner_bios); 199 } 200} 201 202/* 203 * Return the MTRR memory type matching a region's flags 204 */ 205static int 206i686_mtrrtype(int flags) 207{ 208 int i; 209 210 flags &= MDF_ATTRMASK; 211 212 for (i = 0; i < (sizeof(i686_mtrrtomrt) / sizeof(i686_mtrrtomrt[0])); i++) { 213 if (i686_mtrrtomrt[i] == 0) 214 continue; 215 if (flags == i686_mtrrtomrt[i]) 216 return(i); 217 } 218 return(-1); 219} 220 221 222/* 223 * Update the current CPU's MTRRs with those represented in the 224 * descriptor list. 225 */ 226static int 227i686_mrstore(struct mem_range_softc *sc) 228{ 229 struct mem_range_desc *mrd; 230 u_int64_t msrv; 231 int i, j, msr; 232 u_int cr4save; 233 234#ifdef SMP 235 /* 236 * We should use all_but_self_ipi() to call other CPUs into a 237 * locking gate, then call a target function to do this work. 238 * The "proper" solution involves a generalised locking gate 239 * implementation, not ready yet. 240 */ 241 return(EOPNOTSUPP); 242#endif 243 244 disable_intr(); /* disable interrupts */ 245 cr4save = rcr4(); /* save cr4 */ 246 if (cr4save & CR4_PGE) 247 load_cr4(cr4save & ~CR4_PGE); 248 load_cr0((rcr0() & ~CR0_NW) | CR0_CD); /* disable caches (CD = 1, NW = 0) */ 249 wbinvd(); /* flush caches */ 250 invltlb(); /* flush TLBs */ 251 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~0x800); /* disable MTRRs (E = 0) */ 252 253 mrd = sc->mr_desc; 254 255 /* Set fixed-range MTRRs */ 256 if (sc->mr_cap & MR686_FIXMTRR) { 257 msr = MSR_MTRR64kBase; 258 for (i = 0; i < (MTRR_N64K / 8); i++, msr++) { 259 msrv = 0; 260 for (j = 7; j >= 0; j--) { 261 msrv = msrv << 8; 262 msrv |= (i686_mtrrtype((mrd + j)->mr_flags) & 0xff); 263 } 264 wrmsr(msr, msrv); 265 mrd += 8; 266 } 267 msr = MSR_MTRR16kBase; 268 for (i = 0; i < (MTRR_N16K / 8); i++, msr++) { 269 msrv = 0; 270 for (j = 7; j >= 0; j--) { 271 msrv = msrv << 8; 272 msrv |= (i686_mtrrtype((mrd + j)->mr_flags) & 0xff); 273 } 274 wrmsr(msr, msrv); 275 mrd += 8; 276 } 277 msr = MSR_MTRR4kBase; 278 for (i = 0; i < (MTRR_N4K / 8); i++, msr++) { 279 msrv = 0; 280 for (j = 7; j >= 0; j--) { 281 msrv = msrv << 8; 282 msrv |= (i686_mtrrtype((mrd + j)->mr_flags) & 0xff); 283 } 284 wrmsr(msr, msrv); 285 mrd += 8; 286 } 287 } 288 289 /* Set remainder which must be variable MTRRs */ 290 msr = MSR_MTRRVarBase; 291 for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) { 292 /* base/type register */ 293 if (mrd->mr_flags & MDF_ACTIVE) { 294 msrv = mrd->mr_base & 0x0000000ffffff000LL; 295 msrv |= (i686_mtrrtype(mrd->mr_flags) & 0xff); 296 } else { 297 msrv = 0; 298 } 299 wrmsr(msr, msrv); 300 301 /* mask/active register */ 302 if (mrd->mr_flags & MDF_ACTIVE) { 303 msrv = 0x800 | (~(mrd->mr_len - 1) & 0x0000000ffffff000LL); 304 } else { 305 msrv = 0; 306 } 307 wrmsr(msr + 1, msrv); 308 } 309 310 wbinvd(); /* flush caches */ 311 invltlb(); /* flush TLB */ 312 wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | 0x800); /* restore MTRR state */ 313 load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* enable caches CD = 0 and NW = 0 */ 314 load_cr4(cr4save); /* restore cr4 */ 315 enable_intr(); /* enable interrupts */ 316 return(0); 317} 318 319/* 320 * Hunt for the fixed MTRR referencing (addr) 321 */ 322static struct mem_range_desc * 323i686_mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr) 324{ 325 struct mem_range_desc *mrd; 326 int i; 327 328 for (i = 0, mrd = sc->mr_desc; i < (MTRR_N64K + MTRR_N16K + MTRR_N4K); i++, mrd++) 329 if ((addr >= mrd->mr_base) && (addr < (mrd->mr_base + mrd->mr_len))) 330 return(mrd); 331 return(NULL); 332} 333 334/* 335 * Try to satisfy the given range request by manipulating the fixed MTRRs that 336 * cover low memory. 337 * 338 * Note that we try to be generous here; we'll bloat the range out to the 339 * next higher/lower boundary to avoid the consumer having to know too much 340 * about the mechanisms here. 341 * 342 * XXX note that this will have to be updated when we start supporting "busy" ranges. 343 */ 344static int 345i686_mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 346{ 347 struct mem_range_desc *first_md, *last_md, *curr_md; 348 349 /* range check */ 350 if (((first_md = i686_mtrrfixsearch(sc, mrd->mr_base)) == NULL) || 351 ((last_md = i686_mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) == NULL)) 352 return(EINVAL); 353 354 /* set flags, clear set-by-firmware flag */ 355 for (curr_md = first_md; curr_md <= last_md; curr_md++) { 356 curr_md->mr_flags = mrcopyflags(curr_md->mr_flags & ~MDF_FIRMWARE, mrd->mr_flags); 357 bcopy(mrd->mr_owner, curr_md->mr_owner, sizeof(mrd->mr_owner)); 358 } 359 360 return(0); 361} 362 363 364/* 365 * Modify/add a variable MTRR to satisfy the request. 366 * 367 * XXX needs to be updated to properly support "busy" ranges. 368 */ 369static int 370i686_mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 371{ 372 struct mem_range_desc *curr_md, *free_md; 373 int i; 374 375 /* 376 * Scan the currently active variable descriptors, look for 377 * one we exactly match (straight takeover) and for possible 378 * accidental overlaps. 379 * Keep track of the first empty variable descriptor in case we 380 * can't perform a takeover. 381 */ 382 i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; 383 curr_md = sc->mr_desc + i; 384 free_md = NULL; 385 for (; i < sc->mr_ndesc; i++, curr_md++) { 386 if (curr_md->mr_flags & MDF_ACTIVE) { 387 /* exact match? */ 388 if ((curr_md->mr_base == mrd->mr_base) && 389 (curr_md->mr_len == mrd->mr_len)) { 390 /* whoops, owned by someone */ 391 if (curr_md->mr_flags & MDF_BUSY) 392 return(EBUSY); 393 /* Ok, just hijack this entry */ 394 free_md = curr_md; 395 break; 396 } 397 /* non-exact overlap? */ 398 if (mroverlap(curr_md, mrd)) 399 return(EINVAL); 400 } else if (free_md == NULL) { 401 free_md = curr_md; 402 } 403 } 404 /* got somewhere to put it? */ 405 if (free_md == NULL) 406 return(ENOSPC); 407 408 /* Set up new descriptor */ 409 free_md->mr_base = mrd->mr_base; 410 free_md->mr_len = mrd->mr_len; 411 free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags); 412 bcopy(mrd->mr_owner, free_md->mr_owner, sizeof(mrd->mr_owner)); 413 return(0); 414} 415 416/* 417 * Handle requests to set memory range attributes by manipulating MTRRs. 418 * 419 * Note that we're not too smart here; we'll split a range to insert a 420 * region inside, and coalesce regions to make smaller ones, but nothing 421 * really fancy. 422 */ 423static int 424i686_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) 425{ 426 struct mem_range_desc *targ; 427 int error = 0; 428 429 switch(*arg) { 430 case MEMRANGE_SET_UPDATE: 431 /* make sure that what's being asked for is even possible at all */ 432 if (!mrvalid(mrd->mr_base, mrd->mr_len) || 433 (i686_mtrrtype(mrd->mr_flags & MDF_ATTRMASK) == -1)) 434 return(EINVAL); 435 436#define FIXTOP ((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + (MTRR_N4K * 0x1000)) 437 438 /* are the "low memory" conditions applicable? */ 439 if ((sc->mr_cap & MR686_FIXMTRR) && 440 ((mrd->mr_base + mrd->mr_len) <= FIXTOP)) { 441 if ((error = i686_mrsetlow(sc, mrd, arg)) != 0) 442 return(error); 443 } else { 444 /* it's time to play with variable MTRRs */ 445 if ((error = i686_mrsetvariable(sc, mrd, arg)) != 0) 446 return(error); 447 } 448 break; 449 450 case MEMRANGE_SET_REMOVE: 451 if ((targ = mem_range_match(sc, mrd)) == NULL) 452 return(ENOENT); 453 if (targ->mr_flags & MDF_FIXACTIVE) 454 return(EPERM); 455 if (targ->mr_flags & MDF_BUSY) 456 return(EBUSY); 457 targ->mr_flags &= ~MDF_ACTIVE; 458 targ->mr_owner[0] = 0; 459 break; 460 461 default: 462 return(EOPNOTSUPP); 463 } 464 465 /* update the hardware */ 466 if (i686_mrstore(sc)) 467 error = EIO; 468 i686_mrfetch(sc); /* refetch to see where we're at */ 469 return(error); 470} 471 472/* 473 * Work out how many ranges we support, initialise storage for them, 474 * fetch the initial settings. 475 */ 476static void 477i686_mrinit(struct mem_range_softc *sc) 478{ 479 struct mem_range_desc *mrd; 480 u_int64_t mtrrcap, mtrrdef; 481 int nmdesc = 0; 482 int i; 483 484 mtrrcap = rdmsr(MSR_MTRRcap); 485 mtrrdef = rdmsr(MSR_MTRRdefType); 486 487 /* For now, bail out if MTRRs are not enabled */ 488 if (!(mtrrdef & 0x800)) { 489 if (bootverbose) 490 printf("CPU supports MTRRs but not enabled\n"); 491 return; 492 } 493 nmdesc = mtrrcap & 0xff; 494 printf("Pentium Pro MTRR support enabled, default memory type is %s\n", 495 i686_mtrrtotext[mtrrdef & 0xff]); 496 497 /* If fixed MTRRs supported and enabled */ 498 if ((mtrrcap & 0x100) && (mtrrdef & 0x400)) { 499 sc->mr_cap = MR686_FIXMTRR; 500 nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K; 501 } 502 503 sc->mr_desc = 504 (struct mem_range_desc *)malloc(nmdesc * sizeof(struct mem_range_desc), 505 M_MEMDESC, M_WAITOK); 506 bzero(sc->mr_desc, nmdesc * sizeof(struct mem_range_desc)); 507 sc->mr_ndesc = nmdesc; 508 509 mrd = sc->mr_desc; 510 511 /* Populate the fixed MTRR entries' base/length */ 512 if (sc->mr_cap & MR686_FIXMTRR) { 513 for (i = 0; i < MTRR_N64K; i++, mrd++) { 514 mrd->mr_base = i * 0x10000; 515 mrd->mr_len = 0x10000; 516 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE; 517 } 518 for (i = 0; i < MTRR_N16K; i++, mrd++) { 519 mrd->mr_base = i * 0x4000 + 0x80000; 520 mrd->mr_len = 0x4000; 521 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE; 522 } 523 for (i = 0; i < MTRR_N4K; i++, mrd++) { 524 mrd->mr_base = i * 0x1000 + 0xc0000; 525 mrd->mr_len = 0x1000; 526 mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN | MDF_FIXACTIVE; 527 } 528 } 529 530 /* 531 * Get current settings, anything set now is considered to have 532 * been set by the firmware. (XXX has something already played here?) 533 */ 534 i686_mrfetch(sc); 535 mrd = sc->mr_desc; 536 for (i = 0; i < sc->mr_ndesc; i++, mrd++) { 537 if (mrd->mr_flags & MDF_ACTIVE) 538 mrd->mr_flags |= MDF_FIRMWARE; 539 } 540} 541 542static void 543i686_mem_drvinit(void *unused) 544{ 545 546 /* Try for i686 MTRRs */ 547 if (cpu_feature & CPUID_MTRR) { 548 mem_range_softc.mr_op = &i686_mrops; 549 } 550} 551 552SYSINIT(i686memdev,SI_SUB_DRIVERS,SI_ORDER_FIRST,i686_mem_drvinit,NULL) 553 554 555