1/*-
2 * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: releng/10.3/sys/amd64/amd64/amd64_mem.c 217506 2011-01-17 17:30:35Z jkim $");
29
30#include <sys/param.h>
31#include <sys/kernel.h>
32#include <sys/systm.h>
33#include <sys/malloc.h>
34#include <sys/memrange.h>
35#include <sys/smp.h>
36#include <sys/sysctl.h>
37
38#include <vm/vm.h>
39#include <vm/vm_param.h>
40#include <vm/pmap.h>
41
42#include <machine/cputypes.h>
43#include <machine/md_var.h>
44#include <machine/specialreg.h>
45
46/*
47 * amd64 memory range operations
48 *
49 * This code will probably be impenetrable without reference to the
50 * Intel Pentium Pro documentation or x86-64 programmers manual vol 2.
51 */
52
53static char *mem_owner_bios = "BIOS";
54
55#define	MR686_FIXMTRR	(1<<0)
56
57#define	mrwithin(mr, a)							\
58	(((a) >= (mr)->mr_base) && ((a) < ((mr)->mr_base + (mr)->mr_len)))
59#define	mroverlap(mra, mrb)						\
60	(mrwithin(mra, mrb->mr_base) || mrwithin(mrb, mra->mr_base))
61
62#define	mrvalid(base, len) 						\
63	((!(base & ((1 << 12) - 1))) &&	/* base is multiple of 4k */	\
64	    ((len) >= (1 << 12)) &&	/* length is >= 4k */		\
65	    powerof2((len)) &&		/* ... and power of two */	\
66	    !((base) & ((len) - 1)))	/* range is not discontiuous */
67
68#define	mrcopyflags(curr, new)						\
69	(((curr) & ~MDF_ATTRMASK) | ((new) & MDF_ATTRMASK))
70
71static int mtrrs_disabled;
72TUNABLE_INT("machdep.disable_mtrrs", &mtrrs_disabled);
73SYSCTL_INT(_machdep, OID_AUTO, disable_mtrrs, CTLFLAG_RDTUN,
74    &mtrrs_disabled, 0, "Disable amd64 MTRRs.");
75
76static void	amd64_mrinit(struct mem_range_softc *sc);
77static int	amd64_mrset(struct mem_range_softc *sc,
78		    struct mem_range_desc *mrd, int *arg);
79static void	amd64_mrAPinit(struct mem_range_softc *sc);
80static void	amd64_mrreinit(struct mem_range_softc *sc);
81
82static struct mem_range_ops amd64_mrops = {
83	amd64_mrinit,
84	amd64_mrset,
85	amd64_mrAPinit,
86	amd64_mrreinit
87};
88
89/* XXX for AP startup hook */
90static u_int64_t mtrrcap, mtrrdef;
91
92/* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */
93static u_int64_t mtrr_physmask;
94
95static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc,
96		    struct mem_range_desc *mrd);
97static void	amd64_mrfetch(struct mem_range_softc *sc);
98static int	amd64_mtrrtype(int flags);
99static int	amd64_mrt2mtrr(int flags, int oldval);
100static int	amd64_mtrrconflict(int flag1, int flag2);
101static void	amd64_mrstore(struct mem_range_softc *sc);
102static void	amd64_mrstoreone(void *arg);
103static struct mem_range_desc *amd64_mtrrfixsearch(struct mem_range_softc *sc,
104		    u_int64_t addr);
105static int	amd64_mrsetlow(struct mem_range_softc *sc,
106		    struct mem_range_desc *mrd, int *arg);
107static int	amd64_mrsetvariable(struct mem_range_softc *sc,
108		    struct mem_range_desc *mrd, int *arg);
109
110/* amd64 MTRR type to memory range type conversion */
111static int amd64_mtrrtomrt[] = {
112	MDF_UNCACHEABLE,
113	MDF_WRITECOMBINE,
114	MDF_UNKNOWN,
115	MDF_UNKNOWN,
116	MDF_WRITETHROUGH,
117	MDF_WRITEPROTECT,
118	MDF_WRITEBACK
119};
120
121#define	MTRRTOMRTLEN (sizeof(amd64_mtrrtomrt) / sizeof(amd64_mtrrtomrt[0]))
122
123static int
124amd64_mtrr2mrt(int val)
125{
126
127	if (val < 0 || val >= MTRRTOMRTLEN)
128		return (MDF_UNKNOWN);
129	return (amd64_mtrrtomrt[val]);
130}
131
132/*
133 * amd64 MTRR conflicts. Writeback and uncachable may overlap.
134 */
135static int
136amd64_mtrrconflict(int flag1, int flag2)
137{
138
139	flag1 &= MDF_ATTRMASK;
140	flag2 &= MDF_ATTRMASK;
141	if ((flag1 & MDF_UNKNOWN) || (flag2 & MDF_UNKNOWN))
142		return (1);
143	if (flag1 == flag2 ||
144	    (flag1 == MDF_WRITEBACK && flag2 == MDF_UNCACHEABLE) ||
145	    (flag2 == MDF_WRITEBACK && flag1 == MDF_UNCACHEABLE))
146		return (0);
147	return (1);
148}
149
150/*
151 * Look for an exactly-matching range.
152 */
153static struct mem_range_desc *
154mem_range_match(struct mem_range_softc *sc, struct mem_range_desc *mrd)
155{
156	struct mem_range_desc *cand;
157	int i;
158
159	for (i = 0, cand = sc->mr_desc; i < sc->mr_ndesc; i++, cand++)
160		if ((cand->mr_base == mrd->mr_base) &&
161		    (cand->mr_len == mrd->mr_len))
162			return (cand);
163	return (NULL);
164}
165
166/*
167 * Fetch the current mtrr settings from the current CPU (assumed to
168 * all be in sync in the SMP case).  Note that if we are here, we
169 * assume that MTRRs are enabled, and we may or may not have fixed
170 * MTRRs.
171 */
172static void
173amd64_mrfetch(struct mem_range_softc *sc)
174{
175	struct mem_range_desc *mrd;
176	u_int64_t msrv;
177	int i, j, msr;
178
179	mrd = sc->mr_desc;
180
181	/* Get fixed-range MTRRs. */
182	if (sc->mr_cap & MR686_FIXMTRR) {
183		msr = MSR_MTRR64kBase;
184		for (i = 0; i < (MTRR_N64K / 8); i++, msr++) {
185			msrv = rdmsr(msr);
186			for (j = 0; j < 8; j++, mrd++) {
187				mrd->mr_flags =
188				    (mrd->mr_flags & ~MDF_ATTRMASK) |
189				    amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE;
190				if (mrd->mr_owner[0] == 0)
191					strcpy(mrd->mr_owner, mem_owner_bios);
192				msrv = msrv >> 8;
193			}
194		}
195		msr = MSR_MTRR16kBase;
196		for (i = 0; i < (MTRR_N16K / 8); i++, msr++) {
197			msrv = rdmsr(msr);
198			for (j = 0; j < 8; j++, mrd++) {
199				mrd->mr_flags =
200				    (mrd->mr_flags & ~MDF_ATTRMASK) |
201				    amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE;
202				if (mrd->mr_owner[0] == 0)
203					strcpy(mrd->mr_owner, mem_owner_bios);
204				msrv = msrv >> 8;
205			}
206		}
207		msr = MSR_MTRR4kBase;
208		for (i = 0; i < (MTRR_N4K / 8); i++, msr++) {
209			msrv = rdmsr(msr);
210			for (j = 0; j < 8; j++, mrd++) {
211				mrd->mr_flags =
212				    (mrd->mr_flags & ~MDF_ATTRMASK) |
213				    amd64_mtrr2mrt(msrv & 0xff) | MDF_ACTIVE;
214				if (mrd->mr_owner[0] == 0)
215					strcpy(mrd->mr_owner, mem_owner_bios);
216				msrv = msrv >> 8;
217			}
218		}
219	}
220
221	/* Get remainder which must be variable MTRRs. */
222	msr = MSR_MTRRVarBase;
223	for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) {
224		msrv = rdmsr(msr);
225		mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
226		    amd64_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE);
227		mrd->mr_base = msrv & mtrr_physmask;
228		msrv = rdmsr(msr + 1);
229		mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ?
230		    (mrd->mr_flags | MDF_ACTIVE) :
231		    (mrd->mr_flags & ~MDF_ACTIVE);
232
233		/* Compute the range from the mask. Ick. */
234		mrd->mr_len = (~(msrv & mtrr_physmask) &
235		    (mtrr_physmask | 0xfffL)) + 1;
236		if (!mrvalid(mrd->mr_base, mrd->mr_len))
237			mrd->mr_flags |= MDF_BOGUS;
238
239		/* If unclaimed and active, must be the BIOS. */
240		if ((mrd->mr_flags & MDF_ACTIVE) && (mrd->mr_owner[0] == 0))
241			strcpy(mrd->mr_owner, mem_owner_bios);
242	}
243}
244
245/*
246 * Return the MTRR memory type matching a region's flags
247 */
248static int
249amd64_mtrrtype(int flags)
250{
251	int i;
252
253	flags &= MDF_ATTRMASK;
254
255	for (i = 0; i < MTRRTOMRTLEN; i++) {
256		if (amd64_mtrrtomrt[i] == MDF_UNKNOWN)
257			continue;
258		if (flags == amd64_mtrrtomrt[i])
259			return (i);
260	}
261	return (-1);
262}
263
264static int
265amd64_mrt2mtrr(int flags, int oldval)
266{
267	int val;
268
269	if ((val = amd64_mtrrtype(flags)) == -1)
270		return (oldval & 0xff);
271	return (val & 0xff);
272}
273
274/*
275 * Update running CPU(s) MTRRs to match the ranges in the descriptor
276 * list.
277 *
278 * XXX Must be called with interrupts enabled.
279 */
280static void
281amd64_mrstore(struct mem_range_softc *sc)
282{
283#ifdef SMP
284	/*
285	 * We should use ipi_all_but_self() to call other CPUs into a
286	 * locking gate, then call a target function to do this work.
287	 * The "proper" solution involves a generalised locking gate
288	 * implementation, not ready yet.
289	 */
290	smp_rendezvous(NULL, amd64_mrstoreone, NULL, sc);
291#else
292	disable_intr();				/* disable interrupts */
293	amd64_mrstoreone(sc);
294	enable_intr();
295#endif
296}
297
298/*
299 * Update the current CPU's MTRRs with those represented in the
300 * descriptor list.  Note that we do this wholesale rather than just
301 * stuffing one entry; this is simpler (but slower, of course).
302 */
303static void
304amd64_mrstoreone(void *arg)
305{
306	struct mem_range_softc *sc = arg;
307	struct mem_range_desc *mrd;
308	u_int64_t omsrv, msrv;
309	int i, j, msr;
310	u_long cr0, cr4;
311
312	mrd = sc->mr_desc;
313
314	critical_enter();
315
316	/* Disable PGE. */
317	cr4 = rcr4();
318	load_cr4(cr4 & ~CR4_PGE);
319
320	/* Disable caches (CD = 1, NW = 0). */
321	cr0 = rcr0();
322	load_cr0((cr0 & ~CR0_NW) | CR0_CD);
323
324	/* Flushes caches and TLBs. */
325	wbinvd();
326	invltlb();
327
328	/* Disable MTRRs (E = 0). */
329	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~MTRR_DEF_ENABLE);
330
331	/* Set fixed-range MTRRs. */
332	if (sc->mr_cap & MR686_FIXMTRR) {
333		msr = MSR_MTRR64kBase;
334		for (i = 0; i < (MTRR_N64K / 8); i++, msr++) {
335			msrv = 0;
336			omsrv = rdmsr(msr);
337			for (j = 7; j >= 0; j--) {
338				msrv = msrv << 8;
339				msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags,
340				    omsrv >> (j * 8));
341			}
342			wrmsr(msr, msrv);
343			mrd += 8;
344		}
345		msr = MSR_MTRR16kBase;
346		for (i = 0; i < (MTRR_N16K / 8); i++, msr++) {
347			msrv = 0;
348			omsrv = rdmsr(msr);
349			for (j = 7; j >= 0; j--) {
350				msrv = msrv << 8;
351				msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags,
352				    omsrv >> (j * 8));
353			}
354			wrmsr(msr, msrv);
355			mrd += 8;
356		}
357		msr = MSR_MTRR4kBase;
358		for (i = 0; i < (MTRR_N4K / 8); i++, msr++) {
359			msrv = 0;
360			omsrv = rdmsr(msr);
361			for (j = 7; j >= 0; j--) {
362				msrv = msrv << 8;
363				msrv |= amd64_mrt2mtrr((mrd + j)->mr_flags,
364				    omsrv >> (j * 8));
365			}
366			wrmsr(msr, msrv);
367			mrd += 8;
368		}
369	}
370
371	/* Set remainder which must be variable MTRRs. */
372	msr = MSR_MTRRVarBase;
373	for (; (mrd - sc->mr_desc) < sc->mr_ndesc; msr += 2, mrd++) {
374		/* base/type register */
375		omsrv = rdmsr(msr);
376		if (mrd->mr_flags & MDF_ACTIVE) {
377			msrv = mrd->mr_base & mtrr_physmask;
378			msrv |= amd64_mrt2mtrr(mrd->mr_flags, omsrv);
379		} else {
380			msrv = 0;
381		}
382		wrmsr(msr, msrv);
383
384		/* mask/active register */
385		if (mrd->mr_flags & MDF_ACTIVE) {
386			msrv = MTRR_PHYSMASK_VALID |
387			    (~(mrd->mr_len - 1) & mtrr_physmask);
388		} else {
389			msrv = 0;
390		}
391		wrmsr(msr + 1, msrv);
392	}
393
394	/* Flush caches and TLBs. */
395	wbinvd();
396	invltlb();
397
398	/* Enable MTRRs. */
399	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | MTRR_DEF_ENABLE);
400
401	/* Restore caches and PGE. */
402	load_cr0(cr0);
403	load_cr4(cr4);
404
405	critical_exit();
406}
407
408/*
409 * Hunt for the fixed MTRR referencing (addr)
410 */
411static struct mem_range_desc *
412amd64_mtrrfixsearch(struct mem_range_softc *sc, u_int64_t addr)
413{
414	struct mem_range_desc *mrd;
415	int i;
416
417	for (i = 0, mrd = sc->mr_desc; i < (MTRR_N64K + MTRR_N16K + MTRR_N4K);
418	     i++, mrd++)
419		if ((addr >= mrd->mr_base) &&
420		    (addr < (mrd->mr_base + mrd->mr_len)))
421			return (mrd);
422	return (NULL);
423}
424
425/*
426 * Try to satisfy the given range request by manipulating the fixed
427 * MTRRs that cover low memory.
428 *
429 * Note that we try to be generous here; we'll bloat the range out to
430 * the next higher/lower boundary to avoid the consumer having to know
431 * too much about the mechanisms here.
432 *
433 * XXX note that this will have to be updated when we start supporting
434 * "busy" ranges.
435 */
436static int
437amd64_mrsetlow(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg)
438{
439	struct mem_range_desc *first_md, *last_md, *curr_md;
440
441	/* Range check. */
442	if (((first_md = amd64_mtrrfixsearch(sc, mrd->mr_base)) == NULL) ||
443	    ((last_md = amd64_mtrrfixsearch(sc, mrd->mr_base + mrd->mr_len - 1)) == NULL))
444		return (EINVAL);
445
446	/* Check that we aren't doing something risky. */
447	if (!(mrd->mr_flags & MDF_FORCE))
448		for (curr_md = first_md; curr_md <= last_md; curr_md++) {
449			if ((curr_md->mr_flags & MDF_ATTRMASK) == MDF_UNKNOWN)
450				return (EACCES);
451		}
452
453	/* Set flags, clear set-by-firmware flag. */
454	for (curr_md = first_md; curr_md <= last_md; curr_md++) {
455		curr_md->mr_flags = mrcopyflags(curr_md->mr_flags &
456		    ~MDF_FIRMWARE, mrd->mr_flags);
457		bcopy(mrd->mr_owner, curr_md->mr_owner, sizeof(mrd->mr_owner));
458	}
459
460	return (0);
461}
462
463/*
464 * Modify/add a variable MTRR to satisfy the request.
465 *
466 * XXX needs to be updated to properly support "busy" ranges.
467 */
468static int
469amd64_mrsetvariable(struct mem_range_softc *sc, struct mem_range_desc *mrd,
470    int *arg)
471{
472	struct mem_range_desc *curr_md, *free_md;
473	int i;
474
475	/*
476	 * Scan the currently active variable descriptors, look for
477	 * one we exactly match (straight takeover) and for possible
478	 * accidental overlaps.
479	 *
480	 * Keep track of the first empty variable descriptor in case
481	 * we can't perform a takeover.
482	 */
483	i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0;
484	curr_md = sc->mr_desc + i;
485	free_md = NULL;
486	for (; i < sc->mr_ndesc; i++, curr_md++) {
487		if (curr_md->mr_flags & MDF_ACTIVE) {
488			/* Exact match? */
489			if ((curr_md->mr_base == mrd->mr_base) &&
490			    (curr_md->mr_len == mrd->mr_len)) {
491
492				/* Whoops, owned by someone. */
493				if (curr_md->mr_flags & MDF_BUSY)
494					return (EBUSY);
495
496				/* Check that we aren't doing something risky */
497				if (!(mrd->mr_flags & MDF_FORCE) &&
498				    ((curr_md->mr_flags & MDF_ATTRMASK) ==
499				    MDF_UNKNOWN))
500					return (EACCES);
501
502				/* Ok, just hijack this entry. */
503				free_md = curr_md;
504				break;
505			}
506
507			/* Non-exact overlap? */
508			if (mroverlap(curr_md, mrd)) {
509				/* Between conflicting region types? */
510				if (amd64_mtrrconflict(curr_md->mr_flags,
511				    mrd->mr_flags))
512					return (EINVAL);
513			}
514		} else if (free_md == NULL) {
515			free_md = curr_md;
516		}
517	}
518
519	/* Got somewhere to put it? */
520	if (free_md == NULL)
521		return (ENOSPC);
522
523	/* Set up new descriptor. */
524	free_md->mr_base = mrd->mr_base;
525	free_md->mr_len = mrd->mr_len;
526	free_md->mr_flags = mrcopyflags(MDF_ACTIVE, mrd->mr_flags);
527	bcopy(mrd->mr_owner, free_md->mr_owner, sizeof(mrd->mr_owner));
528	return (0);
529}
530
531/*
532 * Handle requests to set memory range attributes by manipulating MTRRs.
533 */
534static int
535amd64_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg)
536{
537	struct mem_range_desc *targ;
538	int error, i;
539
540	switch (*arg) {
541	case MEMRANGE_SET_UPDATE:
542		/*
543		 * Make sure that what's being asked for is even
544		 * possible at all.
545		 */
546		if (!mrvalid(mrd->mr_base, mrd->mr_len) ||
547		    amd64_mtrrtype(mrd->mr_flags) == -1)
548			return (EINVAL);
549
550#define	FIXTOP	((MTRR_N64K * 0x10000) + (MTRR_N16K * 0x4000) + (MTRR_N4K * 0x1000))
551
552		/* Are the "low memory" conditions applicable? */
553		if ((sc->mr_cap & MR686_FIXMTRR) &&
554		    ((mrd->mr_base + mrd->mr_len) <= FIXTOP)) {
555			if ((error = amd64_mrsetlow(sc, mrd, arg)) != 0)
556				return (error);
557		} else {
558			/* It's time to play with variable MTRRs. */
559			if ((error = amd64_mrsetvariable(sc, mrd, arg)) != 0)
560				return (error);
561		}
562		break;
563
564	case MEMRANGE_SET_REMOVE:
565		if ((targ = mem_range_match(sc, mrd)) == NULL)
566			return (ENOENT);
567		if (targ->mr_flags & MDF_FIXACTIVE)
568			return (EPERM);
569		if (targ->mr_flags & MDF_BUSY)
570			return (EBUSY);
571		targ->mr_flags &= ~MDF_ACTIVE;
572		targ->mr_owner[0] = 0;
573		break;
574
575	default:
576		return (EOPNOTSUPP);
577	}
578
579	/*
580	 * Ensure that the direct map region does not contain any mappings
581	 * that span MTRRs of different types.  However, the fixed MTRRs can
582	 * be ignored, because a large page mapping the first 1 MB of physical
583	 * memory is a special case that the processor handles.  The entire
584	 * TLB will be invalidated by amd64_mrstore(), so pmap_demote_DMAP()
585	 * needn't do it.
586	 */
587	i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0;
588	mrd = sc->mr_desc + i;
589	for (; i < sc->mr_ndesc; i++, mrd++) {
590		if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE)
591			pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, FALSE);
592	}
593
594	/* Update the hardware. */
595	amd64_mrstore(sc);
596
597	/* Refetch to see where we're at. */
598	amd64_mrfetch(sc);
599	return (0);
600}
601
602/*
603 * Work out how many ranges we support, initialise storage for them,
604 * and fetch the initial settings.
605 */
606static void
607amd64_mrinit(struct mem_range_softc *sc)
608{
609	struct mem_range_desc *mrd;
610	u_int regs[4];
611	int i, nmdesc = 0, pabits;
612
613	mtrrcap = rdmsr(MSR_MTRRcap);
614	mtrrdef = rdmsr(MSR_MTRRdefType);
615
616	/* For now, bail out if MTRRs are not enabled. */
617	if (!(mtrrdef & MTRR_DEF_ENABLE)) {
618		if (bootverbose)
619			printf("CPU supports MTRRs but not enabled\n");
620		return;
621	}
622	nmdesc = mtrrcap & MTRR_CAP_VCNT;
623
624	/*
625	 * Determine the size of the PhysMask and PhysBase fields in
626	 * the variable range MTRRs.  If the extended CPUID 0x80000008
627	 * is present, use that to figure out how many physical
628	 * address bits the CPU supports.  Otherwise, default to 36
629	 * address bits.
630	 */
631	if (cpu_exthigh >= 0x80000008) {
632		do_cpuid(0x80000008, regs);
633		pabits = regs[0] & 0xff;
634	} else
635		pabits = 36;
636	mtrr_physmask = ((1UL << pabits) - 1) & ~0xfffUL;
637
638	/* If fixed MTRRs supported and enabled. */
639	if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) {
640		sc->mr_cap = MR686_FIXMTRR;
641		nmdesc += MTRR_N64K + MTRR_N16K + MTRR_N4K;
642	}
643
644	sc->mr_desc = malloc(nmdesc * sizeof(struct mem_range_desc), M_MEMDESC,
645	    M_WAITOK | M_ZERO);
646	sc->mr_ndesc = nmdesc;
647
648	mrd = sc->mr_desc;
649
650	/* Populate the fixed MTRR entries' base/length. */
651	if (sc->mr_cap & MR686_FIXMTRR) {
652		for (i = 0; i < MTRR_N64K; i++, mrd++) {
653			mrd->mr_base = i * 0x10000;
654			mrd->mr_len = 0x10000;
655			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN |
656			    MDF_FIXACTIVE;
657		}
658		for (i = 0; i < MTRR_N16K; i++, mrd++) {
659			mrd->mr_base = i * 0x4000 + 0x80000;
660			mrd->mr_len = 0x4000;
661			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN |
662			    MDF_FIXACTIVE;
663		}
664		for (i = 0; i < MTRR_N4K; i++, mrd++) {
665			mrd->mr_base = i * 0x1000 + 0xc0000;
666			mrd->mr_len = 0x1000;
667			mrd->mr_flags = MDF_FIXBASE | MDF_FIXLEN |
668			    MDF_FIXACTIVE;
669		}
670	}
671
672	/*
673	 * Get current settings, anything set now is considered to
674	 * have been set by the firmware. (XXX has something already
675	 * played here?)
676	 */
677	amd64_mrfetch(sc);
678	mrd = sc->mr_desc;
679	for (i = 0; i < sc->mr_ndesc; i++, mrd++) {
680		if (mrd->mr_flags & MDF_ACTIVE)
681			mrd->mr_flags |= MDF_FIRMWARE;
682	}
683
684	/*
685	 * Ensure that the direct map region does not contain any mappings
686	 * that span MTRRs of different types.  However, the fixed MTRRs can
687	 * be ignored, because a large page mapping the first 1 MB of physical
688	 * memory is a special case that the processor handles.  Invalidate
689	 * any old TLB entries that might hold inconsistent memory type
690	 * information.
691	 */
692	i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0;
693	mrd = sc->mr_desc + i;
694	for (; i < sc->mr_ndesc; i++, mrd++) {
695		if ((mrd->mr_flags & (MDF_ACTIVE | MDF_BOGUS)) == MDF_ACTIVE)
696			pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, TRUE);
697	}
698}
699
700/*
701 * Initialise MTRRs on an AP after the BSP has run the init code.
702 */
703static void
704amd64_mrAPinit(struct mem_range_softc *sc)
705{
706
707	amd64_mrstoreone(sc);
708	wrmsr(MSR_MTRRdefType, mtrrdef);
709}
710
711/*
712 * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor
713 * list.
714 *
715 * XXX Must be called with interrupts enabled.
716 */
717static void
718amd64_mrreinit(struct mem_range_softc *sc)
719{
720#ifdef SMP
721	/*
722	 * We should use ipi_all_but_self() to call other CPUs into a
723	 * locking gate, then call a target function to do this work.
724	 * The "proper" solution involves a generalised locking gate
725	 * implementation, not ready yet.
726	 */
727	smp_rendezvous(NULL, (void *)amd64_mrAPinit, NULL, sc);
728#else
729	disable_intr();				/* disable interrupts */
730	amd64_mrAPinit(sc);
731	enable_intr();
732#endif
733}
734
735static void
736amd64_mem_drvinit(void *unused)
737{
738
739	if (mtrrs_disabled)
740		return;
741	if (!(cpu_feature & CPUID_MTRR))
742		return;
743	if ((cpu_id & 0xf00) != 0x600 && (cpu_id & 0xf00) != 0xf00)
744		return;
745	switch (cpu_vendor_id) {
746	case CPU_VENDOR_INTEL:
747	case CPU_VENDOR_AMD:
748	case CPU_VENDOR_CENTAUR:
749		break;
750	default:
751		return;
752	}
753	mem_range_softc.mr_op = &amd64_mrops;
754}
755SYSINIT(amd64memdev, SI_SUB_DRIVERS, SI_ORDER_FIRST, amd64_mem_drvinit, NULL);
756