1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD: stable/11/sys/x86/iommu/intel_fault.c 309882 2016-12-12 09:43:48Z kib $"); 32257251Skib 33257251Skib#include "opt_acpi.h" 34257251Skib 35257251Skib#include <sys/param.h> 36257251Skib#include <sys/bus.h> 37257251Skib#include <sys/kernel.h> 38257251Skib#include <sys/malloc.h> 39257251Skib#include <sys/memdesc.h> 40257251Skib#include <sys/module.h> 41257251Skib#include <sys/rman.h> 42257251Skib#include <sys/taskqueue.h> 43257251Skib#include <sys/tree.h> 44280260Skib#include <sys/vmem.h> 45257251Skib#include <machine/bus.h> 46257251Skib#include <contrib/dev/acpica/include/acpi.h> 47257251Skib#include <contrib/dev/acpica/include/accommon.h> 48257251Skib#include <dev/acpica/acpivar.h> 49264008Srstone#include <dev/pci/pcireg.h> 50264008Srstone#include <dev/pci/pcivar.h> 51257251Skib#include <vm/vm.h> 52257251Skib#include <vm/vm_extern.h> 53257251Skib#include <vm/vm_kern.h> 54257251Skib#include <vm/vm_page.h> 55257251Skib#include <vm/vm_map.h> 56257251Skib#include <x86/include/busdma_impl.h> 57257251Skib#include <x86/iommu/intel_reg.h> 58257251Skib#include <x86/iommu/busdma_dmar.h> 59257251Skib#include <x86/iommu/intel_dmar.h> 60257251Skib 61257251Skib/* 62257251Skib * Fault interrupt handling for DMARs. If advanced fault logging is 63257251Skib * not implemented by hardware, the code emulates it. Fast interrupt 64257251Skib * handler flushes the fault registers into circular buffer at 65257251Skib * unit->fault_log, and schedules a task. 66257251Skib * 67257251Skib * The fast handler is used since faults usually come in bursts, and 68257251Skib * number of fault log registers is limited, e.g. down to one for 5400 69257251Skib * MCH. We are trying to reduce the latency for clearing the fault 70257251Skib * register file. The task is usually long-running, since printf() is 71257251Skib * slow, but this is not problematic because bursts are rare. 72257251Skib * 73257251Skib * For the same reason, each translation unit task is executed in its 74257251Skib * own thread. 75257251Skib * 76257251Skib * XXXKIB It seems there is no hardware available which implements 77257251Skib * advanced fault logging, so the code to handle AFL is not written. 78257251Skib */ 79257251Skib 80257251Skibstatic int 81257251Skibdmar_fault_next(struct dmar_unit *unit, int faultp) 82257251Skib{ 83257251Skib 84257251Skib faultp += 2; 85257251Skib if (faultp == unit->fault_log_size) 86257251Skib faultp = 0; 87257251Skib return (faultp); 88257251Skib} 89257251Skib 90257251Skibstatic void 91257512Skibdmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) 92257251Skib{ 93257251Skib uint32_t clear; 94257251Skib 95257251Skib clear = 0; 96257251Skib if ((fsts & DMAR_FSTS_ITE) != 0) { 97257251Skib printf("DMAR%d: Invalidation timed out\n", unit->unit); 98257251Skib clear |= DMAR_FSTS_ITE; 99257251Skib } 100257251Skib if ((fsts & DMAR_FSTS_ICE) != 0) { 101257251Skib printf("DMAR%d: Invalidation completion error\n", 102257251Skib unit->unit); 103257251Skib clear |= DMAR_FSTS_ICE; 104257251Skib } 105257251Skib if ((fsts & DMAR_FSTS_IQE) != 0) { 106257251Skib printf("DMAR%d: Invalidation queue error\n", 107257251Skib unit->unit); 108257251Skib clear |= DMAR_FSTS_IQE; 109257251Skib } 110257251Skib if ((fsts & DMAR_FSTS_APF) != 0) { 111257251Skib printf("DMAR%d: Advanced pending fault\n", unit->unit); 112257251Skib clear |= DMAR_FSTS_APF; 113257251Skib } 114257251Skib if ((fsts & DMAR_FSTS_AFO) != 0) { 115257251Skib printf("DMAR%d: Advanced fault overflow\n", unit->unit); 116257251Skib clear |= DMAR_FSTS_AFO; 117257251Skib } 118257251Skib if (clear != 0) 119257251Skib dmar_write4(unit, DMAR_FSTS_REG, clear); 120257251Skib} 121257251Skib 122257251Skibint 123257512Skibdmar_fault_intr(void *arg) 124257251Skib{ 125257251Skib struct dmar_unit *unit; 126257251Skib uint64_t fault_rec[2]; 127257251Skib uint32_t fsts; 128257251Skib int fri, frir, faultp; 129257251Skib bool enqueue; 130257251Skib 131257251Skib unit = arg; 132257251Skib enqueue = false; 133257251Skib fsts = dmar_read4(unit, DMAR_FSTS_REG); 134257512Skib dmar_fault_intr_clear(unit, fsts); 135257251Skib 136257251Skib if ((fsts & DMAR_FSTS_PPF) == 0) 137257251Skib goto done; 138257251Skib 139257251Skib fri = DMAR_FSTS_FRI(fsts); 140257251Skib for (;;) { 141257251Skib frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; 142257251Skib fault_rec[1] = dmar_read8(unit, frir + 8); 143257251Skib if ((fault_rec[1] & DMAR_FRCD2_F) == 0) 144257251Skib break; 145257251Skib fault_rec[0] = dmar_read8(unit, frir); 146257251Skib dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 147257251Skib DMAR_FAULT_LOCK(unit); 148257251Skib faultp = unit->fault_log_head; 149257251Skib if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { 150257251Skib /* XXXKIB log overflow */ 151257251Skib } else { 152257251Skib unit->fault_log[faultp] = fault_rec[0]; 153257251Skib unit->fault_log[faultp + 1] = fault_rec[1]; 154257251Skib unit->fault_log_head = dmar_fault_next(unit, faultp); 155257251Skib enqueue = true; 156257251Skib } 157257251Skib DMAR_FAULT_UNLOCK(unit); 158257251Skib fri += 1; 159257251Skib if (fri >= DMAR_CAP_NFR(unit->hw_cap)) 160257251Skib fri = 0; 161257251Skib } 162257251Skib 163257251Skibdone: 164257251Skib /* 165257251Skib * On SandyBridge, due to errata BJ124, IvyBridge errata 166257251Skib * BV100, and Haswell errata HSD40, "Spurious Intel VT-d 167257251Skib * Interrupts May Occur When the PFO Bit is Set". Handle the 168257251Skib * cases by clearing overflow bit even if no fault is 169257251Skib * reported. 170257251Skib * 171257251Skib * On IvyBridge, errata BV30 states that clearing clear 172257251Skib * DMAR_FRCD2_F bit in the fault register causes spurious 173257251Skib * interrupt. Do nothing. 174257251Skib * 175257251Skib */ 176257251Skib if ((fsts & DMAR_FSTS_PFO) != 0) { 177257251Skib printf("DMAR%d: Fault Overflow\n", unit->unit); 178257251Skib dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); 179257251Skib } 180257251Skib 181257251Skib if (enqueue) { 182296272Sjhb taskqueue_enqueue(unit->fault_taskqueue, 183257251Skib &unit->fault_task); 184257251Skib } 185257251Skib return (FILTER_HANDLED); 186257251Skib} 187257251Skib 188257251Skibstatic void 189257251Skibdmar_fault_task(void *arg, int pending __unused) 190257251Skib{ 191257251Skib struct dmar_unit *unit; 192257251Skib struct dmar_ctx *ctx; 193257251Skib uint64_t fault_rec[2]; 194257251Skib int sid, bus, slot, func, faultp; 195257251Skib 196257251Skib unit = arg; 197257251Skib DMAR_FAULT_LOCK(unit); 198257251Skib for (;;) { 199257251Skib faultp = unit->fault_log_tail; 200257251Skib if (faultp == unit->fault_log_head) 201257251Skib break; 202257251Skib 203257251Skib fault_rec[0] = unit->fault_log[faultp]; 204257251Skib fault_rec[1] = unit->fault_log[faultp + 1]; 205257251Skib unit->fault_log_tail = dmar_fault_next(unit, faultp); 206257251Skib DMAR_FAULT_UNLOCK(unit); 207257251Skib 208257251Skib sid = DMAR_FRCD2_SID(fault_rec[1]); 209257251Skib printf("DMAR%d: ", unit->unit); 210257251Skib DMAR_LOCK(unit); 211264008Srstone ctx = dmar_find_ctx_locked(unit, sid); 212257251Skib if (ctx == NULL) { 213257251Skib printf("<unknown dev>:"); 214264008Srstone 215264008Srstone /* 216264008Srstone * Note that the slot and function will not be correct 217264008Srstone * if ARI is in use, but without a ctx entry we have 218264008Srstone * no way of knowing whether ARI is in use or not. 219264008Srstone */ 220264008Srstone bus = PCI_RID2BUS(sid); 221264008Srstone slot = PCI_RID2SLOT(sid); 222264008Srstone func = PCI_RID2FUNC(sid); 223257251Skib } else { 224257251Skib ctx->flags |= DMAR_CTX_FAULTED; 225257251Skib ctx->last_fault_rec[0] = fault_rec[0]; 226257251Skib ctx->last_fault_rec[1] = fault_rec[1]; 227257902Sdim device_print_prettyname(ctx->ctx_tag.owner); 228264008Srstone bus = pci_get_bus(ctx->ctx_tag.owner); 229264008Srstone slot = pci_get_slot(ctx->ctx_tag.owner); 230264008Srstone func = pci_get_function(ctx->ctx_tag.owner); 231257251Skib } 232257251Skib DMAR_UNLOCK(unit); 233257251Skib printf( 234276948Skib "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x " 235276948Skib "addr %jx\n", 236276948Skib bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]), 237257251Skib DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), 238257251Skib (uintmax_t)fault_rec[0]); 239257251Skib DMAR_FAULT_LOCK(unit); 240257251Skib } 241257251Skib DMAR_FAULT_UNLOCK(unit); 242257251Skib} 243257251Skib 244257251Skibstatic void 245257251Skibdmar_clear_faults(struct dmar_unit *unit) 246257251Skib{ 247257251Skib uint32_t frec, frir, fsts; 248257251Skib int i; 249257251Skib 250257251Skib for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { 251257251Skib frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; 252257251Skib frec = dmar_read4(unit, frir + 12); 253257251Skib if ((frec & DMAR_FRCD2_F32) == 0) 254257251Skib continue; 255257251Skib dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 256257251Skib } 257257251Skib fsts = dmar_read4(unit, DMAR_FSTS_REG); 258257251Skib dmar_write4(unit, DMAR_FSTS_REG, fsts); 259257251Skib} 260257251Skib 261257251Skibint 262257251Skibdmar_init_fault_log(struct dmar_unit *unit) 263257251Skib{ 264257251Skib 265257251Skib mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); 266257251Skib unit->fault_log_size = 256; /* 128 fault log entries */ 267257251Skib TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); 268257251Skib if (unit->fault_log_size % 2 != 0) 269257251Skib panic("hw.dmar_fault_log_size must be even"); 270257251Skib unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, 271257251Skib M_DEVBUF, M_WAITOK | M_ZERO); 272257251Skib 273257251Skib TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); 274309882Skib unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK, 275257251Skib taskqueue_thread_enqueue, &unit->fault_taskqueue); 276257251Skib taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, 277257251Skib "dmar%d fault taskq", unit->unit); 278257251Skib 279257512Skib DMAR_LOCK(unit); 280257512Skib dmar_disable_fault_intr(unit); 281257251Skib dmar_clear_faults(unit); 282257512Skib dmar_enable_fault_intr(unit); 283257512Skib DMAR_UNLOCK(unit); 284257251Skib 285257251Skib return (0); 286257251Skib} 287257251Skib 288257251Skibvoid 289257251Skibdmar_fini_fault_log(struct dmar_unit *unit) 290257251Skib{ 291257251Skib 292257512Skib DMAR_LOCK(unit); 293257512Skib dmar_disable_fault_intr(unit); 294257512Skib DMAR_UNLOCK(unit); 295257251Skib 296257251Skib if (unit->fault_taskqueue == NULL) 297257251Skib return; 298257251Skib 299257251Skib taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); 300257251Skib taskqueue_free(unit->fault_taskqueue); 301257512Skib unit->fault_taskqueue = NULL; 302257251Skib mtx_destroy(&unit->fault_lock); 303257251Skib 304257251Skib free(unit->fault_log, M_DEVBUF); 305257251Skib unit->fault_log = NULL; 306257251Skib unit->fault_log_head = unit->fault_log_tail = 0; 307257251Skib} 308257512Skib 309257512Skibvoid 310257512Skibdmar_enable_fault_intr(struct dmar_unit *unit) 311257512Skib{ 312257512Skib uint32_t fectl; 313257512Skib 314257512Skib DMAR_ASSERT_LOCKED(unit); 315257512Skib fectl = dmar_read4(unit, DMAR_FECTL_REG); 316257512Skib fectl &= ~DMAR_FECTL_IM; 317257512Skib dmar_write4(unit, DMAR_FECTL_REG, fectl); 318257512Skib} 319257512Skib 320257512Skibvoid 321257512Skibdmar_disable_fault_intr(struct dmar_unit *unit) 322257512Skib{ 323257512Skib uint32_t fectl; 324257512Skib 325257512Skib DMAR_ASSERT_LOCKED(unit); 326257512Skib fectl = dmar_read4(unit, DMAR_FECTL_REG); 327257512Skib dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); 328257512Skib} 329