intel_fault.c revision 257902
1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_fault.c 257902 2013-11-09 22:00:44Z dim $"); 32257251Skib 33257251Skib#include "opt_acpi.h" 34257251Skib 35257251Skib#include <sys/param.h> 36257251Skib#include <sys/bus.h> 37257251Skib#include <sys/kernel.h> 38257251Skib#include <sys/malloc.h> 39257251Skib#include <sys/memdesc.h> 40257251Skib#include <sys/module.h> 41257251Skib#include <sys/rman.h> 42257251Skib#include <sys/taskqueue.h> 43257251Skib#include <sys/tree.h> 44257251Skib#include <machine/bus.h> 45257251Skib#include <contrib/dev/acpica/include/acpi.h> 46257251Skib#include <contrib/dev/acpica/include/accommon.h> 47257251Skib#include <dev/acpica/acpivar.h> 48257251Skib#include <vm/vm.h> 49257251Skib#include <vm/vm_extern.h> 50257251Skib#include <vm/vm_kern.h> 51257251Skib#include <vm/vm_page.h> 52257251Skib#include <vm/vm_map.h> 53257251Skib#include <x86/include/busdma_impl.h> 54257251Skib#include <x86/iommu/intel_reg.h> 55257251Skib#include <x86/iommu/busdma_dmar.h> 56257251Skib#include <x86/iommu/intel_dmar.h> 57257251Skib 58257251Skib/* 59257251Skib * Fault interrupt handling for DMARs. If advanced fault logging is 60257251Skib * not implemented by hardware, the code emulates it. Fast interrupt 61257251Skib * handler flushes the fault registers into circular buffer at 62257251Skib * unit->fault_log, and schedules a task. 63257251Skib * 64257251Skib * The fast handler is used since faults usually come in bursts, and 65257251Skib * number of fault log registers is limited, e.g. down to one for 5400 66257251Skib * MCH. We are trying to reduce the latency for clearing the fault 67257251Skib * register file. The task is usually long-running, since printf() is 68257251Skib * slow, but this is not problematic because bursts are rare. 69257251Skib * 70257251Skib * For the same reason, each translation unit task is executed in its 71257251Skib * own thread. 72257251Skib * 73257251Skib * XXXKIB It seems there is no hardware available which implements 74257251Skib * advanced fault logging, so the code to handle AFL is not written. 75257251Skib */ 76257251Skib 77257251Skibstatic int 78257251Skibdmar_fault_next(struct dmar_unit *unit, int faultp) 79257251Skib{ 80257251Skib 81257251Skib faultp += 2; 82257251Skib if (faultp == unit->fault_log_size) 83257251Skib faultp = 0; 84257251Skib return (faultp); 85257251Skib} 86257251Skib 87257251Skibstatic void 88257512Skibdmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) 89257251Skib{ 90257251Skib uint32_t clear; 91257251Skib 92257251Skib clear = 0; 93257251Skib if ((fsts & DMAR_FSTS_ITE) != 0) { 94257251Skib printf("DMAR%d: Invalidation timed out\n", unit->unit); 95257251Skib clear |= DMAR_FSTS_ITE; 96257251Skib } 97257251Skib if ((fsts & DMAR_FSTS_ICE) != 0) { 98257251Skib printf("DMAR%d: Invalidation completion error\n", 99257251Skib unit->unit); 100257251Skib clear |= DMAR_FSTS_ICE; 101257251Skib } 102257251Skib if ((fsts & DMAR_FSTS_IQE) != 0) { 103257251Skib printf("DMAR%d: Invalidation queue error\n", 104257251Skib unit->unit); 105257251Skib clear |= DMAR_FSTS_IQE; 106257251Skib } 107257251Skib if ((fsts & DMAR_FSTS_APF) != 0) { 108257251Skib printf("DMAR%d: Advanced pending fault\n", unit->unit); 109257251Skib clear |= DMAR_FSTS_APF; 110257251Skib } 111257251Skib if ((fsts & DMAR_FSTS_AFO) != 0) { 112257251Skib printf("DMAR%d: Advanced fault overflow\n", unit->unit); 113257251Skib clear |= DMAR_FSTS_AFO; 114257251Skib } 115257251Skib if (clear != 0) 116257251Skib dmar_write4(unit, DMAR_FSTS_REG, clear); 117257251Skib} 118257251Skib 119257251Skibint 120257512Skibdmar_fault_intr(void *arg) 121257251Skib{ 122257251Skib struct dmar_unit *unit; 123257251Skib uint64_t fault_rec[2]; 124257251Skib uint32_t fsts; 125257251Skib int fri, frir, faultp; 126257251Skib bool enqueue; 127257251Skib 128257251Skib unit = arg; 129257251Skib enqueue = false; 130257251Skib fsts = dmar_read4(unit, DMAR_FSTS_REG); 131257512Skib dmar_fault_intr_clear(unit, fsts); 132257251Skib 133257251Skib if ((fsts & DMAR_FSTS_PPF) == 0) 134257251Skib goto done; 135257251Skib 136257251Skib fri = DMAR_FSTS_FRI(fsts); 137257251Skib for (;;) { 138257251Skib frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; 139257251Skib fault_rec[1] = dmar_read8(unit, frir + 8); 140257251Skib if ((fault_rec[1] & DMAR_FRCD2_F) == 0) 141257251Skib break; 142257251Skib fault_rec[0] = dmar_read8(unit, frir); 143257251Skib dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 144257251Skib DMAR_FAULT_LOCK(unit); 145257251Skib faultp = unit->fault_log_head; 146257251Skib if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { 147257251Skib /* XXXKIB log overflow */ 148257251Skib } else { 149257251Skib unit->fault_log[faultp] = fault_rec[0]; 150257251Skib unit->fault_log[faultp + 1] = fault_rec[1]; 151257251Skib unit->fault_log_head = dmar_fault_next(unit, faultp); 152257251Skib enqueue = true; 153257251Skib } 154257251Skib DMAR_FAULT_UNLOCK(unit); 155257251Skib fri += 1; 156257251Skib if (fri >= DMAR_CAP_NFR(unit->hw_cap)) 157257251Skib fri = 0; 158257251Skib } 159257251Skib 160257251Skibdone: 161257251Skib /* 162257251Skib * On SandyBridge, due to errata BJ124, IvyBridge errata 163257251Skib * BV100, and Haswell errata HSD40, "Spurious Intel VT-d 164257251Skib * Interrupts May Occur When the PFO Bit is Set". Handle the 165257251Skib * cases by clearing overflow bit even if no fault is 166257251Skib * reported. 167257251Skib * 168257251Skib * On IvyBridge, errata BV30 states that clearing clear 169257251Skib * DMAR_FRCD2_F bit in the fault register causes spurious 170257251Skib * interrupt. Do nothing. 171257251Skib * 172257251Skib */ 173257251Skib if ((fsts & DMAR_FSTS_PFO) != 0) { 174257251Skib printf("DMAR%d: Fault Overflow\n", unit->unit); 175257251Skib dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); 176257251Skib } 177257251Skib 178257251Skib if (enqueue) { 179257251Skib taskqueue_enqueue_fast(unit->fault_taskqueue, 180257251Skib &unit->fault_task); 181257251Skib } 182257251Skib return (FILTER_HANDLED); 183257251Skib} 184257251Skib 185257251Skibstatic void 186257251Skibdmar_fault_task(void *arg, int pending __unused) 187257251Skib{ 188257251Skib struct dmar_unit *unit; 189257251Skib struct dmar_ctx *ctx; 190257251Skib uint64_t fault_rec[2]; 191257251Skib int sid, bus, slot, func, faultp; 192257251Skib 193257251Skib unit = arg; 194257251Skib DMAR_FAULT_LOCK(unit); 195257251Skib for (;;) { 196257251Skib faultp = unit->fault_log_tail; 197257251Skib if (faultp == unit->fault_log_head) 198257251Skib break; 199257251Skib 200257251Skib fault_rec[0] = unit->fault_log[faultp]; 201257251Skib fault_rec[1] = unit->fault_log[faultp + 1]; 202257251Skib unit->fault_log_tail = dmar_fault_next(unit, faultp); 203257251Skib DMAR_FAULT_UNLOCK(unit); 204257251Skib 205257251Skib sid = DMAR_FRCD2_SID(fault_rec[1]); 206257251Skib bus = (sid >> 8) & 0xf; 207257251Skib slot = (sid >> 3) & 0x1f; 208257251Skib func = sid & 0x7; 209257251Skib printf("DMAR%d: ", unit->unit); 210257251Skib DMAR_LOCK(unit); 211257251Skib ctx = dmar_find_ctx_locked(unit, bus, slot, func); 212257251Skib if (ctx == NULL) { 213257251Skib printf("<unknown dev>:"); 214257251Skib } else { 215257251Skib ctx->flags |= DMAR_CTX_FAULTED; 216257251Skib ctx->last_fault_rec[0] = fault_rec[0]; 217257251Skib ctx->last_fault_rec[1] = fault_rec[1]; 218257902Sdim device_print_prettyname(ctx->ctx_tag.owner); 219257251Skib } 220257251Skib DMAR_UNLOCK(unit); 221257251Skib printf( 222257251Skib "pci%d:%d:%d fault acc %x adt 0x%x reason 0x%x addr %jx\n", 223257251Skib bus, slot, func, DMAR_FRCD2_T(fault_rec[1]), 224257251Skib DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), 225257251Skib (uintmax_t)fault_rec[0]); 226257251Skib DMAR_FAULT_LOCK(unit); 227257251Skib } 228257251Skib DMAR_FAULT_UNLOCK(unit); 229257251Skib} 230257251Skib 231257251Skibstatic void 232257251Skibdmar_clear_faults(struct dmar_unit *unit) 233257251Skib{ 234257251Skib uint32_t frec, frir, fsts; 235257251Skib int i; 236257251Skib 237257251Skib for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { 238257251Skib frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; 239257251Skib frec = dmar_read4(unit, frir + 12); 240257251Skib if ((frec & DMAR_FRCD2_F32) == 0) 241257251Skib continue; 242257251Skib dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 243257251Skib } 244257251Skib fsts = dmar_read4(unit, DMAR_FSTS_REG); 245257251Skib dmar_write4(unit, DMAR_FSTS_REG, fsts); 246257251Skib} 247257251Skib 248257251Skibint 249257251Skibdmar_init_fault_log(struct dmar_unit *unit) 250257251Skib{ 251257251Skib 252257251Skib mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); 253257251Skib unit->fault_log_size = 256; /* 128 fault log entries */ 254257251Skib TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); 255257251Skib if (unit->fault_log_size % 2 != 0) 256257251Skib panic("hw.dmar_fault_log_size must be even"); 257257251Skib unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, 258257251Skib M_DEVBUF, M_WAITOK | M_ZERO); 259257251Skib 260257251Skib TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); 261257251Skib unit->fault_taskqueue = taskqueue_create_fast("dmar", M_WAITOK, 262257251Skib taskqueue_thread_enqueue, &unit->fault_taskqueue); 263257251Skib taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, 264257251Skib "dmar%d fault taskq", unit->unit); 265257251Skib 266257512Skib DMAR_LOCK(unit); 267257512Skib dmar_disable_fault_intr(unit); 268257251Skib dmar_clear_faults(unit); 269257512Skib dmar_enable_fault_intr(unit); 270257512Skib DMAR_UNLOCK(unit); 271257251Skib 272257251Skib return (0); 273257251Skib} 274257251Skib 275257251Skibvoid 276257251Skibdmar_fini_fault_log(struct dmar_unit *unit) 277257251Skib{ 278257251Skib 279257512Skib DMAR_LOCK(unit); 280257512Skib dmar_disable_fault_intr(unit); 281257512Skib DMAR_UNLOCK(unit); 282257251Skib 283257251Skib if (unit->fault_taskqueue == NULL) 284257251Skib return; 285257251Skib 286257251Skib taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); 287257251Skib taskqueue_free(unit->fault_taskqueue); 288257512Skib unit->fault_taskqueue = NULL; 289257251Skib mtx_destroy(&unit->fault_lock); 290257251Skib 291257251Skib free(unit->fault_log, M_DEVBUF); 292257251Skib unit->fault_log = NULL; 293257251Skib unit->fault_log_head = unit->fault_log_tail = 0; 294257251Skib} 295257512Skib 296257512Skibvoid 297257512Skibdmar_enable_fault_intr(struct dmar_unit *unit) 298257512Skib{ 299257512Skib uint32_t fectl; 300257512Skib 301257512Skib DMAR_ASSERT_LOCKED(unit); 302257512Skib fectl = dmar_read4(unit, DMAR_FECTL_REG); 303257512Skib fectl &= ~DMAR_FECTL_IM; 304257512Skib dmar_write4(unit, DMAR_FECTL_REG, fectl); 305257512Skib} 306257512Skib 307257512Skibvoid 308257512Skibdmar_disable_fault_intr(struct dmar_unit *unit) 309257512Skib{ 310257512Skib uint32_t fectl; 311257512Skib 312257512Skib DMAR_ASSERT_LOCKED(unit); 313257512Skib fectl = dmar_read4(unit, DMAR_FECTL_REG); 314257512Skib dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); 315257512Skib} 316