1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35#include "opt_acpi.h"
36
37#include <sys/param.h>
38#include <sys/bus.h>
39#include <sys/kernel.h>
40#include <sys/malloc.h>
41#include <sys/memdesc.h>
42#include <sys/module.h>
43#include <sys/rman.h>
44#include <sys/taskqueue.h>
45#include <sys/tree.h>
46#include <sys/vmem.h>
47#include <machine/bus.h>
48#include <contrib/dev/acpica/include/acpi.h>
49#include <contrib/dev/acpica/include/accommon.h>
50#include <dev/acpica/acpivar.h>
51#include <dev/pci/pcireg.h>
52#include <dev/pci/pcivar.h>
53#include <vm/vm.h>
54#include <vm/vm_extern.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_page.h>
57#include <vm/vm_map.h>
58#include <x86/include/busdma_impl.h>
59#include <x86/iommu/intel_reg.h>
60#include <dev/iommu/busdma_iommu.h>
61#include <x86/iommu/intel_dmar.h>
62
63/*
64 * Fault interrupt handling for DMARs.  If advanced fault logging is
65 * not implemented by hardware, the code emulates it.  Fast interrupt
66 * handler flushes the fault registers into circular buffer at
67 * unit->fault_log, and schedules a task.
68 *
69 * The fast handler is used since faults usually come in bursts, and
70 * number of fault log registers is limited, e.g. down to one for 5400
71 * MCH.  We are trying to reduce the latency for clearing the fault
72 * register file.  The task is usually long-running, since printf() is
73 * slow, but this is not problematic because bursts are rare.
74 *
75 * For the same reason, each translation unit task is executed in its
76 * own thread.
77 *
78 * XXXKIB It seems there is no hardware available which implements
79 * advanced fault logging, so the code to handle AFL is not written.
80 */
81
82static int
83dmar_fault_next(struct dmar_unit *unit, int faultp)
84{
85
86	faultp += 2;
87	if (faultp == unit->fault_log_size)
88		faultp = 0;
89	return (faultp);
90}
91
92static void
93dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts)
94{
95	uint32_t clear;
96
97	clear = 0;
98	if ((fsts & DMAR_FSTS_ITE) != 0) {
99		printf("DMAR%d: Invalidation timed out\n", unit->iommu.unit);
100		clear |= DMAR_FSTS_ITE;
101	}
102	if ((fsts & DMAR_FSTS_ICE) != 0) {
103		printf("DMAR%d: Invalidation completion error\n",
104		    unit->iommu.unit);
105		clear |= DMAR_FSTS_ICE;
106	}
107	if ((fsts & DMAR_FSTS_IQE) != 0) {
108		printf("DMAR%d: Invalidation queue error\n",
109		    unit->iommu.unit);
110		clear |= DMAR_FSTS_IQE;
111	}
112	if ((fsts & DMAR_FSTS_APF) != 0) {
113		printf("DMAR%d: Advanced pending fault\n", unit->iommu.unit);
114		clear |= DMAR_FSTS_APF;
115	}
116	if ((fsts & DMAR_FSTS_AFO) != 0) {
117		printf("DMAR%d: Advanced fault overflow\n", unit->iommu.unit);
118		clear |= DMAR_FSTS_AFO;
119	}
120	if (clear != 0)
121		dmar_write4(unit, DMAR_FSTS_REG, clear);
122}
123
124int
125dmar_fault_intr(void *arg)
126{
127	struct dmar_unit *unit;
128	uint64_t fault_rec[2];
129	uint32_t fsts;
130	int fri, frir, faultp;
131	bool enqueue;
132
133	unit = arg;
134	enqueue = false;
135	fsts = dmar_read4(unit, DMAR_FSTS_REG);
136	dmar_fault_intr_clear(unit, fsts);
137
138	if ((fsts & DMAR_FSTS_PPF) == 0)
139		goto done;
140
141	fri = DMAR_FSTS_FRI(fsts);
142	for (;;) {
143		frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16;
144		fault_rec[1] = dmar_read8(unit, frir + 8);
145		if ((fault_rec[1] & DMAR_FRCD2_F) == 0)
146			break;
147		fault_rec[0] = dmar_read8(unit, frir);
148		dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
149		DMAR_FAULT_LOCK(unit);
150		faultp = unit->fault_log_head;
151		if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) {
152			/* XXXKIB log overflow */
153		} else {
154			unit->fault_log[faultp] = fault_rec[0];
155			unit->fault_log[faultp + 1] = fault_rec[1];
156			unit->fault_log_head = dmar_fault_next(unit, faultp);
157			enqueue = true;
158		}
159		DMAR_FAULT_UNLOCK(unit);
160		fri += 1;
161		if (fri >= DMAR_CAP_NFR(unit->hw_cap))
162			fri = 0;
163	}
164
165done:
166	/*
167	 * On SandyBridge, due to errata BJ124, IvyBridge errata
168	 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d
169	 * Interrupts May Occur When the PFO Bit is Set".  Handle the
170	 * cases by clearing overflow bit even if no fault is
171	 * reported.
172	 *
173	 * On IvyBridge, errata BV30 states that clearing clear
174	 * DMAR_FRCD2_F bit in the fault register causes spurious
175	 * interrupt.  Do nothing.
176	 *
177	 */
178	if ((fsts & DMAR_FSTS_PFO) != 0) {
179		printf("DMAR%d: Fault Overflow\n", unit->iommu.unit);
180		dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO);
181	}
182
183	if (enqueue) {
184		taskqueue_enqueue(unit->fault_taskqueue,
185		    &unit->fault_task);
186	}
187	return (FILTER_HANDLED);
188}
189
190static void
191dmar_fault_task(void *arg, int pending __unused)
192{
193	struct dmar_unit *unit;
194	struct dmar_ctx *ctx;
195	uint64_t fault_rec[2];
196	int sid, bus, slot, func, faultp;
197
198	unit = arg;
199	DMAR_FAULT_LOCK(unit);
200	for (;;) {
201		faultp = unit->fault_log_tail;
202		if (faultp == unit->fault_log_head)
203			break;
204
205		fault_rec[0] = unit->fault_log[faultp];
206		fault_rec[1] = unit->fault_log[faultp + 1];
207		unit->fault_log_tail = dmar_fault_next(unit, faultp);
208		DMAR_FAULT_UNLOCK(unit);
209
210		sid = DMAR_FRCD2_SID(fault_rec[1]);
211		printf("DMAR%d: ", unit->iommu.unit);
212		DMAR_LOCK(unit);
213		ctx = dmar_find_ctx_locked(unit, sid);
214		if (ctx == NULL) {
215			printf("<unknown dev>:");
216
217			/*
218			 * Note that the slot and function will not be correct
219			 * if ARI is in use, but without a ctx entry we have
220			 * no way of knowing whether ARI is in use or not.
221			 */
222			bus = PCI_RID2BUS(sid);
223			slot = PCI_RID2SLOT(sid);
224			func = PCI_RID2FUNC(sid);
225		} else {
226			ctx->context.flags |= IOMMU_CTX_FAULTED;
227			ctx->last_fault_rec[0] = fault_rec[0];
228			ctx->last_fault_rec[1] = fault_rec[1];
229			device_print_prettyname(ctx->context.tag->owner);
230			bus = pci_get_bus(ctx->context.tag->owner);
231			slot = pci_get_slot(ctx->context.tag->owner);
232			func = pci_get_function(ctx->context.tag->owner);
233		}
234		DMAR_UNLOCK(unit);
235		printf(
236		    "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x "
237		    "addr %jx\n",
238		    bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]),
239		    DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]),
240		    (uintmax_t)fault_rec[0]);
241		DMAR_FAULT_LOCK(unit);
242	}
243	DMAR_FAULT_UNLOCK(unit);
244}
245
246static void
247dmar_clear_faults(struct dmar_unit *unit)
248{
249	uint32_t frec, frir, fsts;
250	int i;
251
252	for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {
253		frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;
254		frec = dmar_read4(unit, frir + 12);
255		if ((frec & DMAR_FRCD2_F32) == 0)
256			continue;
257		dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
258	}
259	fsts = dmar_read4(unit, DMAR_FSTS_REG);
260	dmar_write4(unit, DMAR_FSTS_REG, fsts);
261}
262
263int
264dmar_init_fault_log(struct dmar_unit *unit)
265{
266
267	mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN);
268	unit->fault_log_size = 256; /* 128 fault log entries */
269	TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size);
270	if (unit->fault_log_size % 2 != 0)
271		panic("hw.dmar_fault_log_size must be even");
272	unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size,
273	    M_DEVBUF, M_WAITOK | M_ZERO);
274
275	TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit);
276	unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK,
277	    taskqueue_thread_enqueue, &unit->fault_taskqueue);
278	taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV,
279	    "dmar%d fault taskq", unit->iommu.unit);
280
281	DMAR_LOCK(unit);
282	dmar_disable_fault_intr(unit);
283	dmar_clear_faults(unit);
284	dmar_enable_fault_intr(unit);
285	DMAR_UNLOCK(unit);
286
287	return (0);
288}
289
290void
291dmar_fini_fault_log(struct dmar_unit *unit)
292{
293
294	if (unit->fault_taskqueue == NULL)
295		return;
296
297	DMAR_LOCK(unit);
298	dmar_disable_fault_intr(unit);
299	DMAR_UNLOCK(unit);
300
301	taskqueue_drain(unit->fault_taskqueue, &unit->fault_task);
302	taskqueue_free(unit->fault_taskqueue);
303	unit->fault_taskqueue = NULL;
304	mtx_destroy(&unit->fault_lock);
305
306	free(unit->fault_log, M_DEVBUF);
307	unit->fault_log = NULL;
308	unit->fault_log_head = unit->fault_log_tail = 0;
309}
310
311void
312dmar_enable_fault_intr(struct dmar_unit *unit)
313{
314	uint32_t fectl;
315
316	DMAR_ASSERT_LOCKED(unit);
317	fectl = dmar_read4(unit, DMAR_FECTL_REG);
318	fectl &= ~DMAR_FECTL_IM;
319	dmar_write4(unit, DMAR_FECTL_REG, fectl);
320}
321
322void
323dmar_disable_fault_intr(struct dmar_unit *unit)
324{
325	uint32_t fectl;
326
327	DMAR_ASSERT_LOCKED(unit);
328	fectl = dmar_read4(unit, DMAR_FECTL_REG);
329	dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);
330}
331