mca.c revision 218221 
1/*-
2 * Copyright (c) 2009 Advanced Computing Technologies LLC
3 * Written by: John H. Baldwin <jhb@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28/*
29 * Support for x86 machine check architecture.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/x86/x86/mca.c 218221 2011-02-03 13:09:22Z jhb $");
34
35#ifdef __amd64__
36#define	DEV_APIC
37#else
38#include "opt_apic.h"
39#endif
40
41#include <sys/param.h>
42#include <sys/bus.h>
43#include <sys/interrupt.h>
44#include <sys/kernel.h>
45#include <sys/lock.h>
46#include <sys/malloc.h>
47#include <sys/mutex.h>
48#include <sys/proc.h>
49#include <sys/sched.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/systm.h>
53#include <sys/taskqueue.h>
54#include <machine/intr_machdep.h>
55#include <machine/apicvar.h>
56#include <machine/cputypes.h>
57#include <x86/mca.h>
58#include <machine/md_var.h>
59#include <machine/specialreg.h>
60
61/* Modes for mca_scan() */
62enum scan_mode {
63	POLLED,
64	MCE,
65	CMCI,
66};
67
68#ifdef DEV_APIC
69/*
70 * State maintained for each monitored MCx bank to control the
71 * corrected machine check interrupt threshold.
72 */
73struct cmc_state {
74	int	max_threshold;
75	int	last_intr;
76};
77#endif
78
79struct mca_internal {
80	struct mca_record rec;
81	int		logged;
82	STAILQ_ENTRY(mca_internal) link;
83};
84
85static MALLOC_DEFINE(M_MCA, "MCA", "Machine Check Architecture");
86
87static int mca_count;		/* Number of records stored. */
88
89SYSCTL_NODE(_hw, OID_AUTO, mca, CTLFLAG_RD, NULL, "Machine Check Architecture");
90
91static int mca_enabled = 1;
92TUNABLE_INT("hw.mca.enabled", &mca_enabled);
93SYSCTL_INT(_hw_mca, OID_AUTO, enabled, CTLFLAG_RDTUN, &mca_enabled, 0,
94    "Administrative toggle for machine check support");
95
96static int amd10h_L1TP = 1;
97TUNABLE_INT("hw.mca.amd10h_L1TP", &amd10h_L1TP);
98SYSCTL_INT(_hw_mca, OID_AUTO, amd10h_L1TP, CTLFLAG_RDTUN, &amd10h_L1TP, 0,
99    "Administrative toggle for logging of level one TLB parity (L1TP) errors");
100
101int workaround_erratum383;
102SYSCTL_INT(_hw_mca, OID_AUTO, erratum383, CTLFLAG_RD, &workaround_erratum383, 0,
103    "Is the workaround for Erratum 383 on AMD Family 10h processors enabled?");
104
105static STAILQ_HEAD(, mca_internal) mca_records;
106static struct callout mca_timer;
107static int mca_ticks = 3600;	/* Check hourly by default. */
108static struct taskqueue *mca_tq;
109static struct task mca_task;
110static struct mtx mca_lock;
111
112#ifdef DEV_APIC
113static struct cmc_state **cmc_state;	/* Indexed by cpuid, bank */
114static int cmc_banks;
115static int cmc_throttle = 60;	/* Time in seconds to throttle CMCI. */
116#endif
117
118static int
119sysctl_positive_int(SYSCTL_HANDLER_ARGS)
120{
121	int error, value;
122
123	value = *(int *)arg1;
124	error = sysctl_handle_int(oidp, &value, 0, req);
125	if (error || req->newptr == NULL)
126		return (error);
127	if (value <= 0)
128		return (EINVAL);
129	*(int *)arg1 = value;
130	return (0);
131}
132
133static int
134sysctl_mca_records(SYSCTL_HANDLER_ARGS)
135{
136	int *name = (int *)arg1;
137	u_int namelen = arg2;
138	struct mca_record record;
139	struct mca_internal *rec;
140	int i;
141
142	if (namelen != 1)
143		return (EINVAL);
144
145	if (name[0] < 0 || name[0] >= mca_count)
146		return (EINVAL);
147
148	mtx_lock_spin(&mca_lock);
149	if (name[0] >= mca_count) {
150		mtx_unlock_spin(&mca_lock);
151		return (EINVAL);
152	}
153	i = 0;
154	STAILQ_FOREACH(rec, &mca_records, link) {
155		if (i == name[0]) {
156			record = rec->rec;
157			break;
158		}
159		i++;
160	}
161	mtx_unlock_spin(&mca_lock);
162	return (SYSCTL_OUT(req, &record, sizeof(record)));
163}
164
165static const char *
166mca_error_ttype(uint16_t mca_error)
167{
168
169	switch ((mca_error & 0x000c) >> 2) {
170	case 0:
171		return ("I");
172	case 1:
173		return ("D");
174	case 2:
175		return ("G");
176	}
177	return ("?");
178}
179
180static const char *
181mca_error_level(uint16_t mca_error)
182{
183
184	switch (mca_error & 0x0003) {
185	case 0:
186		return ("L0");
187	case 1:
188		return ("L1");
189	case 2:
190		return ("L2");
191	case 3:
192		return ("LG");
193	}
194	return ("L?");
195}
196
197static const char *
198mca_error_request(uint16_t mca_error)
199{
200
201	switch ((mca_error & 0x00f0) >> 4) {
202	case 0x0:
203		return ("ERR");
204	case 0x1:
205		return ("RD");
206	case 0x2:
207		return ("WR");
208	case 0x3:
209		return ("DRD");
210	case 0x4:
211		return ("DWR");
212	case 0x5:
213		return ("IRD");
214	case 0x6:
215		return ("PREFETCH");
216	case 0x7:
217		return ("EVICT");
218	case 0x8:
219		return ("SNOOP");
220	}
221	return ("???");
222}
223
224static const char *
225mca_error_mmtype(uint16_t mca_error)
226{
227
228	switch ((mca_error & 0x70) >> 4) {
229	case 0x0:
230		return ("GEN");
231	case 0x1:
232		return ("RD");
233	case 0x2:
234		return ("WR");
235	case 0x3:
236		return ("AC");
237	case 0x4:
238		return ("MS");
239	}
240	return ("???");
241}
242
243/* Dump details about a single machine check. */
244static void __nonnull(1)
245mca_log(const struct mca_record *rec)
246{
247	uint16_t mca_error;
248
249	printf("MCA: Bank %d, Status 0x%016llx\n", rec->mr_bank,
250	    (long long)rec->mr_status);
251	printf("MCA: Global Cap 0x%016llx, Status 0x%016llx\n",
252	    (long long)rec->mr_mcg_cap, (long long)rec->mr_mcg_status);
253	printf("MCA: Vendor \"%s\", ID 0x%x, APIC ID %d\n", cpu_vendor,
254	    rec->mr_cpu_id, rec->mr_apic_id);
255	printf("MCA: CPU %d ", rec->mr_cpu);
256	if (rec->mr_status & MC_STATUS_UC)
257		printf("UNCOR ");
258	else {
259		printf("COR ");
260		if (rec->mr_mcg_cap & MCG_CAP_CMCI_P)
261			printf("(%lld) ", ((long long)rec->mr_status &
262			    MC_STATUS_COR_COUNT) >> 38);
263	}
264	if (rec->mr_status & MC_STATUS_PCC)
265		printf("PCC ");
266	if (rec->mr_status & MC_STATUS_OVER)
267		printf("OVER ");
268	mca_error = rec->mr_status & MC_STATUS_MCA_ERROR;
269	switch (mca_error) {
270		/* Simple error codes. */
271	case 0x0000:
272		printf("no error");
273		break;
274	case 0x0001:
275		printf("unclassified error");
276		break;
277	case 0x0002:
278		printf("ucode ROM parity error");
279		break;
280	case 0x0003:
281		printf("external error");
282		break;
283	case 0x0004:
284		printf("FRC error");
285		break;
286	case 0x0005:
287		printf("internal parity error");
288		break;
289	case 0x0400:
290		printf("internal timer error");
291		break;
292	default:
293		if ((mca_error & 0xfc00) == 0x0400) {
294			printf("internal error %x", mca_error & 0x03ff);
295			break;
296		}
297
298		/* Compound error codes. */
299
300		/* Memory hierarchy error. */
301		if ((mca_error & 0xeffc) == 0x000c) {
302			printf("%s memory error", mca_error_level(mca_error));
303			break;
304		}
305
306		/* TLB error. */
307		if ((mca_error & 0xeff0) == 0x0010) {
308			printf("%sTLB %s error", mca_error_ttype(mca_error),
309			    mca_error_level(mca_error));
310			break;
311		}
312
313		/* Memory controller error. */
314		if ((mca_error & 0xef80) == 0x0080) {
315			printf("%s channel ", mca_error_mmtype(mca_error));
316			if ((mca_error & 0x000f) != 0x000f)
317				printf("%d", mca_error & 0x000f);
318			else
319				printf("??");
320			printf(" memory error");
321			break;
322		}
323
324		/* Cache error. */
325		if ((mca_error & 0xef00) == 0x0100) {
326			printf("%sCACHE %s %s error",
327			    mca_error_ttype(mca_error),
328			    mca_error_level(mca_error),
329			    mca_error_request(mca_error));
330			break;
331		}
332
333		/* Bus and/or Interconnect error. */
334		if ((mca_error & 0xe800) == 0x0800) {
335			printf("BUS%s ", mca_error_level(mca_error));
336			switch ((mca_error & 0x0600) >> 9) {
337			case 0:
338				printf("Source");
339				break;
340			case 1:
341				printf("Responder");
342				break;
343			case 2:
344				printf("Observer");
345				break;
346			default:
347				printf("???");
348				break;
349			}
350			printf(" %s ", mca_error_request(mca_error));
351			switch ((mca_error & 0x000c) >> 2) {
352			case 0:
353				printf("Memory");
354				break;
355			case 2:
356				printf("I/O");
357				break;
358			case 3:
359				printf("Other");
360				break;
361			default:
362				printf("???");
363				break;
364			}
365			if (mca_error & 0x0100)
366				printf(" timed out");
367			break;
368		}
369
370		printf("unknown error %x", mca_error);
371		break;
372	}
373	printf("\n");
374	if (rec->mr_status & MC_STATUS_ADDRV)
375		printf("MCA: Address 0x%llx\n", (long long)rec->mr_addr);
376	if (rec->mr_status & MC_STATUS_MISCV)
377		printf("MCA: Misc 0x%llx\n", (long long)rec->mr_misc);
378}
379
380static int __nonnull(2)
381mca_check_status(int bank, struct mca_record *rec)
382{
383	uint64_t status;
384	u_int p[4];
385
386	status = rdmsr(MSR_MC_STATUS(bank));
387	if (!(status & MC_STATUS_VAL))
388		return (0);
389
390	/* Save exception information. */
391	rec->mr_status = status;
392	rec->mr_bank = bank;
393	rec->mr_addr = 0;
394	if (status & MC_STATUS_ADDRV)
395		rec->mr_addr = rdmsr(MSR_MC_ADDR(bank));
396	rec->mr_misc = 0;
397	if (status & MC_STATUS_MISCV)
398		rec->mr_misc = rdmsr(MSR_MC_MISC(bank));
399	rec->mr_tsc = rdtsc();
400	rec->mr_apic_id = PCPU_GET(apic_id);
401	rec->mr_mcg_cap = rdmsr(MSR_MCG_CAP);
402	rec->mr_mcg_status = rdmsr(MSR_MCG_STATUS);
403	rec->mr_cpu_id = cpu_id;
404	rec->mr_cpu_vendor_id = cpu_vendor_id;
405	rec->mr_cpu = PCPU_GET(cpuid);
406
407	/*
408	 * Clear machine check.  Don't do this for uncorrectable
409	 * errors so that the BIOS can see them.
410	 */
411	if (!(rec->mr_status & (MC_STATUS_PCC | MC_STATUS_UC))) {
412		wrmsr(MSR_MC_STATUS(bank), 0);
413		do_cpuid(0, p);
414	}
415	return (1);
416}
417
418static void __nonnull(1)
419mca_record_entry(const struct mca_record *record)
420{
421	struct mca_internal *rec;
422
423	rec = malloc(sizeof(*rec), M_MCA, M_NOWAIT);
424	if (rec == NULL) {
425		printf("MCA: Unable to allocate space for an event.\n");
426		mca_log(record);
427		return;
428	}
429
430	rec->rec = *record;
431	rec->logged = 0;
432	mtx_lock_spin(&mca_lock);
433	STAILQ_INSERT_TAIL(&mca_records, rec, link);
434	mca_count++;
435	mtx_unlock_spin(&mca_lock);
436}
437
438#ifdef DEV_APIC
439/*
440 * Update the interrupt threshold for a CMCI.  The strategy is to use
441 * a low trigger that interrupts as soon as the first event occurs.
442 * However, if a steady stream of events arrive, the threshold is
443 * increased until the interrupts are throttled to once every
444 * cmc_throttle seconds or the periodic scan.  If a periodic scan
445 * finds that the threshold is too high, it is lowered.
446 */
447static void
448cmci_update(enum scan_mode mode, int bank, int valid, struct mca_record *rec)
449{
450	struct cmc_state *cc;
451	uint64_t ctl;
452	u_int delta;
453	int count, limit;
454
455	/* Fetch the current limit for this bank. */
456	cc = &cmc_state[PCPU_GET(cpuid)][bank];
457	ctl = rdmsr(MSR_MC_CTL2(bank));
458	count = (rec->mr_status & MC_STATUS_COR_COUNT) >> 38;
459	delta = (u_int)(ticks - cc->last_intr);
460
461	/*
462	 * If an interrupt was received less than cmc_throttle seconds
463	 * since the previous interrupt and the count from the current
464	 * event is greater than or equal to the current threshold,
465	 * double the threshold up to the max.
466	 */
467	if (mode == CMCI && valid) {
468		limit = ctl & MC_CTL2_THRESHOLD;
469		if (delta < cmc_throttle && count >= limit &&
470		    limit < cc->max_threshold) {
471			limit = min(limit << 1, cc->max_threshold);
472			ctl &= ~MC_CTL2_THRESHOLD;
473			ctl |= limit;
474			wrmsr(MSR_MC_CTL2(bank), limit);
475		}
476		cc->last_intr = ticks;
477		return;
478	}
479
480	/*
481	 * When the banks are polled, check to see if the threshold
482	 * should be lowered.
483	 */
484	if (mode != POLLED)
485		return;
486
487	/* If a CMCI occured recently, do nothing for now. */
488	if (delta < cmc_throttle)
489		return;
490
491	/*
492	 * Compute a new limit based on the average rate of events per
493	 * cmc_throttle seconds since the last interrupt.
494	 */
495	if (valid) {
496		count = (rec->mr_status & MC_STATUS_COR_COUNT) >> 38;
497		limit = count * cmc_throttle / delta;
498		if (limit <= 0)
499			limit = 1;
500		else if (limit > cc->max_threshold)
501			limit = cc->max_threshold;
502	} else
503		limit = 1;
504	if ((ctl & MC_CTL2_THRESHOLD) != limit) {
505		ctl &= ~MC_CTL2_THRESHOLD;
506		ctl |= limit;
507		wrmsr(MSR_MC_CTL2(bank), limit);
508	}
509}
510#endif
511
512/*
513 * This scans all the machine check banks of the current CPU to see if
514 * there are any machine checks.  Any non-recoverable errors are
515 * reported immediately via mca_log().  The current thread must be
516 * pinned when this is called.  The 'mode' parameter indicates if we
517 * are being called from the MC exception handler, the CMCI handler,
518 * or the periodic poller.  In the MC exception case this function
519 * returns true if the system is restartable.  Otherwise, it returns a
520 * count of the number of valid MC records found.
521 */
522static int
523mca_scan(enum scan_mode mode)
524{
525	struct mca_record rec;
526	uint64_t mcg_cap, ucmask;
527	int count, i, recoverable, valid;
528
529	count = 0;
530	recoverable = 1;
531	ucmask = MC_STATUS_UC | MC_STATUS_PCC;
532
533	/* When handling a MCE#, treat the OVER flag as non-restartable. */
534	if (mode == MCE)
535		ucmask |= MC_STATUS_OVER;
536	mcg_cap = rdmsr(MSR_MCG_CAP);
537	for (i = 0; i < (mcg_cap & MCG_CAP_COUNT); i++) {
538#ifdef DEV_APIC
539		/*
540		 * For a CMCI, only check banks this CPU is
541		 * responsible for.
542		 */
543		if (mode == CMCI && !(PCPU_GET(cmci_mask) & 1 << i))
544			continue;
545#endif
546
547		valid = mca_check_status(i, &rec);
548		if (valid) {
549			count++;
550			if (rec.mr_status & ucmask) {
551				recoverable = 0;
552				mca_log(&rec);
553			}
554			mca_record_entry(&rec);
555		}
556
557#ifdef DEV_APIC
558		/*
559		 * If this is a bank this CPU monitors via CMCI,
560		 * update the threshold.
561		 */
562		if (PCPU_GET(cmci_mask) & 1 << i)
563			cmci_update(mode, i, valid, &rec);
564#endif
565	}
566	return (mode == MCE ? recoverable : count);
567}
568
569/*
570 * Scan the machine check banks on all CPUs by binding to each CPU in
571 * turn.  If any of the CPUs contained new machine check records, log
572 * them to the console.
573 */
574static void
575mca_scan_cpus(void *context, int pending)
576{
577	struct mca_internal *mca;
578	struct thread *td;
579	int count, cpu;
580
581	td = curthread;
582	count = 0;
583	thread_lock(td);
584	CPU_FOREACH(cpu) {
585		sched_bind(td, cpu);
586		thread_unlock(td);
587		count += mca_scan(POLLED);
588		thread_lock(td);
589		sched_unbind(td);
590	}
591	thread_unlock(td);
592	if (count != 0) {
593		mtx_lock_spin(&mca_lock);
594		STAILQ_FOREACH(mca, &mca_records, link) {
595			if (!mca->logged) {
596				mca->logged = 1;
597				mtx_unlock_spin(&mca_lock);
598				mca_log(&mca->rec);
599				mtx_lock_spin(&mca_lock);
600			}
601		}
602		mtx_unlock_spin(&mca_lock);
603	}
604}
605
606static void
607mca_periodic_scan(void *arg)
608{
609
610	taskqueue_enqueue(mca_tq, &mca_task);
611	callout_reset(&mca_timer, mca_ticks * hz, mca_periodic_scan, NULL);
612}
613
614static int
615sysctl_mca_scan(SYSCTL_HANDLER_ARGS)
616{
617	int error, i;
618
619	i = 0;
620	error = sysctl_handle_int(oidp, &i, 0, req);
621	if (error)
622		return (error);
623	if (i)
624		taskqueue_enqueue(mca_tq, &mca_task);
625	return (0);
626}
627
628static void
629mca_startup(void *dummy)
630{
631
632	if (!mca_enabled || !(cpu_feature & CPUID_MCA))
633		return;
634
635	mca_tq = taskqueue_create("mca", M_WAITOK, taskqueue_thread_enqueue,
636	    &mca_tq);
637	taskqueue_start_threads(&mca_tq, 1, PI_SWI(SWI_TQ), "mca taskq");
638	callout_reset(&mca_timer, mca_ticks * hz, mca_periodic_scan,
639		    NULL);
640}
641SYSINIT(mca_startup, SI_SUB_SMP, SI_ORDER_ANY, mca_startup, NULL);
642
643#ifdef DEV_APIC
644static void
645cmci_setup(uint64_t mcg_cap)
646{
647	int i;
648
649	cmc_state = malloc((mp_maxid + 1) * sizeof(struct cmc_state **),
650	    M_MCA, M_WAITOK);
651	cmc_banks = mcg_cap & MCG_CAP_COUNT;
652	for (i = 0; i <= mp_maxid; i++)
653		cmc_state[i] = malloc(sizeof(struct cmc_state) * cmc_banks,
654		    M_MCA, M_WAITOK | M_ZERO);
655	SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
656	    "cmc_throttle", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
657	    &cmc_throttle, 0, sysctl_positive_int, "I",
658	    "Interval in seconds to throttle corrected MC interrupts");
659}
660#endif
661
662static void
663mca_setup(uint64_t mcg_cap)
664{
665
666	/*
667	 * On AMD Family 10h processors, unless logging of level one TLB
668	 * parity (L1TP) errors is disabled, enable the recommended workaround
669	 * for Erratum 383.
670	 */
671	if (cpu_vendor_id == CPU_VENDOR_AMD &&
672	    CPUID_TO_FAMILY(cpu_id) == 0x10 && amd10h_L1TP)
673		workaround_erratum383 = 1;
674
675	mtx_init(&mca_lock, "mca", NULL, MTX_SPIN);
676	STAILQ_INIT(&mca_records);
677	TASK_INIT(&mca_task, 0, mca_scan_cpus, NULL);
678	callout_init(&mca_timer, CALLOUT_MPSAFE);
679	SYSCTL_ADD_INT(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
680	    "count", CTLFLAG_RD, &mca_count, 0, "Record count");
681	SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
682	    "interval", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &mca_ticks,
683	    0, sysctl_positive_int, "I",
684	    "Periodic interval in seconds to scan for machine checks");
685	SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
686	    "records", CTLFLAG_RD, sysctl_mca_records, "Machine check records");
687	SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_mca), OID_AUTO,
688	    "force_scan", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
689	    sysctl_mca_scan, "I", "Force an immediate scan for machine checks");
690#ifdef DEV_APIC
691	if (mcg_cap & MCG_CAP_CMCI_P)
692		cmci_setup(mcg_cap);
693#endif
694}
695
696#ifdef DEV_APIC
697/*
698 * See if we should monitor CMCI for this bank.  If CMCI_EN is already
699 * set in MC_CTL2, then another CPU is responsible for this bank, so
700 * ignore it.  If CMCI_EN returns zero after being set, then this bank
701 * does not support CMCI_EN.  If this CPU sets CMCI_EN, then it should
702 * now monitor this bank.
703 */
704static void
705cmci_monitor(int i)
706{
707	struct cmc_state *cc;
708	uint64_t ctl;
709
710	KASSERT(i < cmc_banks, ("CPU %d has more MC banks", PCPU_GET(cpuid)));
711
712	ctl = rdmsr(MSR_MC_CTL2(i));
713	if (ctl & MC_CTL2_CMCI_EN)
714		/* Already monitored by another CPU. */
715		return;
716
717	/* Set the threshold to one event for now. */
718	ctl &= ~MC_CTL2_THRESHOLD;
719	ctl |= MC_CTL2_CMCI_EN | 1;
720	wrmsr(MSR_MC_CTL2(i), ctl);
721	ctl = rdmsr(MSR_MC_CTL2(i));
722	if (!(ctl & MC_CTL2_CMCI_EN))
723		/* This bank does not support CMCI. */
724		return;
725
726	cc = &cmc_state[PCPU_GET(cpuid)][i];
727
728	/* Determine maximum threshold. */
729	ctl &= ~MC_CTL2_THRESHOLD;
730	ctl |= 0x7fff;
731	wrmsr(MSR_MC_CTL2(i), ctl);
732	ctl = rdmsr(MSR_MC_CTL2(i));
733	cc->max_threshold = ctl & MC_CTL2_THRESHOLD;
734
735	/* Start off with a threshold of 1. */
736	ctl &= ~MC_CTL2_THRESHOLD;
737	ctl |= 1;
738	wrmsr(MSR_MC_CTL2(i), ctl);
739
740	/* Mark this bank as monitored. */
741	PCPU_SET(cmci_mask, PCPU_GET(cmci_mask) | 1 << i);
742}
743
744/*
745 * For resume, reset the threshold for any banks we monitor back to
746 * one and throw away the timestamp of the last interrupt.
747 */
748static void
749cmci_resume(int i)
750{
751	struct cmc_state *cc;
752	uint64_t ctl;
753
754	KASSERT(i < cmc_banks, ("CPU %d has more MC banks", PCPU_GET(cpuid)));
755
756	/* Ignore banks not monitored by this CPU. */
757	if (!(PCPU_GET(cmci_mask) & 1 << i))
758		return;
759
760	cc = &cmc_state[PCPU_GET(cpuid)][i];
761	cc->last_intr = -ticks;
762	ctl = rdmsr(MSR_MC_CTL2(i));
763	ctl &= ~MC_CTL2_THRESHOLD;
764	ctl |= MC_CTL2_CMCI_EN | 1;
765	wrmsr(MSR_MC_CTL2(i), ctl);
766}
767#endif
768
769/*
770 * Initializes per-CPU machine check registers and enables corrected
771 * machine check interrupts.
772 */
773static void
774_mca_init(int boot)
775{
776	uint64_t mcg_cap;
777	uint64_t ctl, mask;
778	int i, skip;
779
780	/* MCE is required. */
781	if (!mca_enabled || !(cpu_feature & CPUID_MCE))
782		return;
783
784	if (cpu_feature & CPUID_MCA) {
785		if (boot)
786			PCPU_SET(cmci_mask, 0);
787
788		mcg_cap = rdmsr(MSR_MCG_CAP);
789		if (mcg_cap & MCG_CAP_CTL_P)
790			/* Enable MCA features. */
791			wrmsr(MSR_MCG_CTL, MCG_CTL_ENABLE);
792		if (PCPU_GET(cpuid) == 0 && boot)
793			mca_setup(mcg_cap);
794
795		/*
796		 * Disable logging of level one TLB parity (L1TP) errors by
797		 * the data cache as an alternative workaround for AMD Family
798		 * 10h Erratum 383.  Unlike the recommended workaround, there
799		 * is no performance penalty to this workaround.  However,
800		 * L1TP errors will go unreported.
801		 */
802		if (cpu_vendor_id == CPU_VENDOR_AMD &&
803		    CPUID_TO_FAMILY(cpu_id) == 0x10 && !amd10h_L1TP) {
804			mask = rdmsr(MSR_MC0_CTL_MASK);
805			if ((mask & (1UL << 5)) == 0)
806				wrmsr(MSR_MC0_CTL_MASK, mask | (1UL << 5));
807		}
808		for (i = 0; i < (mcg_cap & MCG_CAP_COUNT); i++) {
809			/* By default enable logging of all errors. */
810			ctl = 0xffffffffffffffffUL;
811			skip = 0;
812
813			if (cpu_vendor_id == CPU_VENDOR_INTEL) {
814				/*
815				 * For P6 models before Nehalem MC0_CTL is
816				 * always enabled and reserved.
817				 */
818				if (i == 0 && CPUID_TO_FAMILY(cpu_id) == 0x6
819				    && CPUID_TO_MODEL(cpu_id) < 0x1a)
820					skip = 1;
821			} else if (cpu_vendor_id == CPU_VENDOR_AMD) {
822				/* BKDG for Family 10h: unset GartTblWkEn. */
823				if (i == 4 && CPUID_TO_FAMILY(cpu_id) >= 0xf)
824					ctl &= ~(1UL << 10);
825			}
826
827			if (!skip)
828				wrmsr(MSR_MC_CTL(i), ctl);
829
830#ifdef DEV_APIC
831			if (mcg_cap & MCG_CAP_CMCI_P) {
832				if (boot)
833					cmci_monitor(i);
834				else
835					cmci_resume(i);
836			}
837#endif
838
839			/* Clear all errors. */
840			wrmsr(MSR_MC_STATUS(i), 0);
841		}
842
843#ifdef DEV_APIC
844		if (PCPU_GET(cmci_mask) != 0 && boot)
845			lapic_enable_cmc();
846#endif
847	}
848
849	load_cr4(rcr4() | CR4_MCE);
850}
851
852/* Must be executed on each CPU during boot. */
853void
854mca_init(void)
855{
856
857	_mca_init(1);
858}
859
860/* Must be executed on each CPU during resume. */
861void
862mca_resume(void)
863{
864
865	_mca_init(0);
866}
867
868/*
869 * The machine check registers for the BSP cannot be initialized until
870 * the local APIC is initialized.  This happens at SI_SUB_CPU,
871 * SI_ORDER_SECOND.
872 */
873static void
874mca_init_bsp(void *arg __unused)
875{
876
877	mca_init();
878}
879SYSINIT(mca_init_bsp, SI_SUB_CPU, SI_ORDER_ANY, mca_init_bsp, NULL);
880
881/* Called when a machine check exception fires. */
882int
883mca_intr(void)
884{
885	uint64_t mcg_status;
886	int recoverable;
887
888	if (!(cpu_feature & CPUID_MCA)) {
889		/*
890		 * Just print the values of the old Pentium registers
891		 * and panic.
892		 */
893		printf("MC Type: 0x%jx  Address: 0x%jx\n",
894		    (uintmax_t)rdmsr(MSR_P5_MC_TYPE),
895		    (uintmax_t)rdmsr(MSR_P5_MC_ADDR));
896		return (0);
897	}
898
899	/* Scan the banks and check for any non-recoverable errors. */
900	recoverable = mca_scan(MCE);
901	mcg_status = rdmsr(MSR_MCG_STATUS);
902	if (!(mcg_status & MCG_STATUS_RIPV))
903		recoverable = 0;
904
905	/* Clear MCIP. */
906	wrmsr(MSR_MCG_STATUS, mcg_status & ~MCG_STATUS_MCIP);
907	return (recoverable);
908}
909
910#ifdef DEV_APIC
911/* Called for a CMCI (correctable machine check interrupt). */
912void
913cmc_intr(void)
914{
915	struct mca_internal *mca;
916	int count;
917
918	/*
919	 * Serialize MCA bank scanning to prevent collisions from
920	 * sibling threads.
921	 */
922	count = mca_scan(CMCI);
923
924	/* If we found anything, log them to the console. */
925	if (count != 0) {
926		mtx_lock_spin(&mca_lock);
927		STAILQ_FOREACH(mca, &mca_records, link) {
928			if (!mca->logged) {
929				mca->logged = 1;
930				mtx_unlock_spin(&mca_lock);
931				mca_log(&mca->rec);
932				mtx_lock_spin(&mca_lock);
933			}
934		}
935		mtx_unlock_spin(&mca_lock);
936	}
937}
938#endif
939