1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD$ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD$"); 31221828Sgrehan 32221828Sgrehan#include <sys/param.h> 33221828Sgrehan#include <sys/kernel.h> 34221828Sgrehan#include <sys/systm.h> 35221828Sgrehan#include <sys/malloc.h> 36221828Sgrehan 37221828Sgrehan#include <vm/vm.h> 38221828Sgrehan#include <vm/pmap.h> 39221828Sgrehan 40221828Sgrehan#include <dev/pci/pcireg.h> 41221828Sgrehan 42221828Sgrehan#include <machine/vmparam.h> 43254548Sneel#include <contrib/dev/acpica/include/acpi.h> 44221828Sgrehan 45221828Sgrehan#include "io/iommu.h" 46221828Sgrehan 47221828Sgrehan/* 48221828Sgrehan * Documented in the "Intel Virtualization Technology for Directed I/O", 49221828Sgrehan * Architecture Spec, September 2008. 50221828Sgrehan */ 51221828Sgrehan 52221828Sgrehan/* Section 10.4 "Register Descriptions" */ 53221828Sgrehanstruct vtdmap { 54221828Sgrehan volatile uint32_t version; 55221828Sgrehan volatile uint32_t res0; 56221828Sgrehan volatile uint64_t cap; 57221828Sgrehan volatile uint64_t ext_cap; 58221828Sgrehan volatile uint32_t gcr; 59221828Sgrehan volatile uint32_t gsr; 60221828Sgrehan volatile uint64_t rta; 61221828Sgrehan volatile uint64_t ccr; 62221828Sgrehan}; 63221828Sgrehan 64221828Sgrehan#define VTD_CAP_SAGAW(cap) (((cap) >> 8) & 0x1F) 65221828Sgrehan#define VTD_CAP_ND(cap) ((cap) & 0x7) 66221828Sgrehan#define VTD_CAP_CM(cap) (((cap) >> 7) & 0x1) 67221828Sgrehan#define VTD_CAP_SPS(cap) (((cap) >> 34) & 0xF) 68221828Sgrehan#define VTD_CAP_RWBF(cap) (((cap) >> 4) & 0x1) 69221828Sgrehan 70221828Sgrehan#define VTD_ECAP_DI(ecap) (((ecap) >> 2) & 0x1) 71221828Sgrehan#define VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1) 72221828Sgrehan#define VTD_ECAP_IRO(ecap) (((ecap) >> 8) & 0x3FF) 73221828Sgrehan 74221828Sgrehan#define VTD_GCR_WBF (1 << 27) 75221828Sgrehan#define VTD_GCR_SRTP (1 << 30) 76261455Seadler#define VTD_GCR_TE (1U << 31) 77221828Sgrehan 78221828Sgrehan#define VTD_GSR_WBFS (1 << 27) 79221828Sgrehan#define VTD_GSR_RTPS (1 << 30) 80261455Seadler#define VTD_GSR_TES (1U << 31) 81221828Sgrehan 82221828Sgrehan#define VTD_CCR_ICC (1UL << 63) /* invalidate context cache */ 83221828Sgrehan#define VTD_CCR_CIRG_GLOBAL (1UL << 61) /* global invalidation */ 84221828Sgrehan 85221828Sgrehan#define VTD_IIR_IVT (1UL << 63) /* invalidation IOTLB */ 86221828Sgrehan#define VTD_IIR_IIRG_GLOBAL (1ULL << 60) /* global IOTLB invalidation */ 87221828Sgrehan#define VTD_IIR_IIRG_DOMAIN (2ULL << 60) /* domain IOTLB invalidation */ 88221828Sgrehan#define VTD_IIR_IIRG_PAGE (3ULL << 60) /* page IOTLB invalidation */ 89221828Sgrehan#define VTD_IIR_DRAIN_READS (1ULL << 49) /* drain pending DMA reads */ 90221828Sgrehan#define VTD_IIR_DRAIN_WRITES (1ULL << 48) /* drain pending DMA writes */ 91221828Sgrehan#define VTD_IIR_DOMAIN_P 32 92221828Sgrehan 93221828Sgrehan#define VTD_ROOT_PRESENT 0x1 94221828Sgrehan#define VTD_CTX_PRESENT 0x1 95221828Sgrehan#define VTD_CTX_TT_ALL (1UL << 2) 96221828Sgrehan 97221828Sgrehan#define VTD_PTE_RD (1UL << 0) 98221828Sgrehan#define VTD_PTE_WR (1UL << 1) 99221828Sgrehan#define VTD_PTE_SUPERPAGE (1UL << 7) 100221828Sgrehan#define VTD_PTE_ADDR_M (0x000FFFFFFFFFF000UL) 101221828Sgrehan 102221828Sgrehanstruct domain { 103221828Sgrehan uint64_t *ptp; /* first level page table page */ 104221828Sgrehan int pt_levels; /* number of page table levels */ 105221828Sgrehan int addrwidth; /* 'AW' field in context entry */ 106221828Sgrehan int spsmask; /* supported super page sizes */ 107221828Sgrehan u_int id; /* domain id */ 108221828Sgrehan vm_paddr_t maxaddr; /* highest address to be mapped */ 109221828Sgrehan SLIST_ENTRY(domain) next; 110221828Sgrehan}; 111221828Sgrehan 112221828Sgrehanstatic SLIST_HEAD(, domain) domhead; 113221828Sgrehan 114221828Sgrehan#define DRHD_MAX_UNITS 8 115221828Sgrehanstatic int drhd_num; 116221828Sgrehanstatic struct vtdmap *vtdmaps[DRHD_MAX_UNITS]; 117221828Sgrehanstatic int max_domains; 118221828Sgrehantypedef int (*drhd_ident_func_t)(void); 119221828Sgrehan 120221828Sgrehanstatic uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 121221828Sgrehanstatic uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 122221828Sgrehan 123221828Sgrehanstatic MALLOC_DEFINE(M_VTD, "vtd", "vtd"); 124221828Sgrehan 125221828Sgrehanstatic int 126221828Sgrehanvtd_max_domains(struct vtdmap *vtdmap) 127221828Sgrehan{ 128221828Sgrehan int nd; 129221828Sgrehan 130221828Sgrehan nd = VTD_CAP_ND(vtdmap->cap); 131221828Sgrehan 132221828Sgrehan switch (nd) { 133221828Sgrehan case 0: 134221828Sgrehan return (16); 135221828Sgrehan case 1: 136221828Sgrehan return (64); 137221828Sgrehan case 2: 138221828Sgrehan return (256); 139221828Sgrehan case 3: 140221828Sgrehan return (1024); 141221828Sgrehan case 4: 142221828Sgrehan return (4 * 1024); 143221828Sgrehan case 5: 144221828Sgrehan return (16 * 1024); 145221828Sgrehan case 6: 146221828Sgrehan return (64 * 1024); 147221828Sgrehan default: 148221828Sgrehan panic("vtd_max_domains: invalid value of nd (0x%0x)", nd); 149221828Sgrehan } 150221828Sgrehan} 151221828Sgrehan 152221828Sgrehanstatic u_int 153221828Sgrehandomain_id(void) 154221828Sgrehan{ 155221828Sgrehan u_int id; 156221828Sgrehan struct domain *dom; 157221828Sgrehan 158221828Sgrehan /* Skip domain id 0 - it is reserved when Caching Mode field is set */ 159221828Sgrehan for (id = 1; id < max_domains; id++) { 160221828Sgrehan SLIST_FOREACH(dom, &domhead, next) { 161221828Sgrehan if (dom->id == id) 162221828Sgrehan break; 163221828Sgrehan } 164221828Sgrehan if (dom == NULL) 165221828Sgrehan break; /* found it */ 166221828Sgrehan } 167221828Sgrehan 168221828Sgrehan if (id >= max_domains) 169221828Sgrehan panic("domain ids exhausted"); 170221828Sgrehan 171221828Sgrehan return (id); 172221828Sgrehan} 173221828Sgrehan 174221828Sgrehanstatic void 175221828Sgrehanvtd_wbflush(struct vtdmap *vtdmap) 176221828Sgrehan{ 177221828Sgrehan 178221828Sgrehan if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0) 179221828Sgrehan pmap_invalidate_cache(); 180221828Sgrehan 181221828Sgrehan if (VTD_CAP_RWBF(vtdmap->cap)) { 182221828Sgrehan vtdmap->gcr = VTD_GCR_WBF; 183221828Sgrehan while ((vtdmap->gsr & VTD_GSR_WBFS) != 0) 184221828Sgrehan ; 185221828Sgrehan } 186221828Sgrehan} 187221828Sgrehan 188221828Sgrehanstatic void 189221828Sgrehanvtd_ctx_global_invalidate(struct vtdmap *vtdmap) 190221828Sgrehan{ 191221828Sgrehan 192221828Sgrehan vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL; 193221828Sgrehan while ((vtdmap->ccr & VTD_CCR_ICC) != 0) 194221828Sgrehan ; 195221828Sgrehan} 196221828Sgrehan 197221828Sgrehanstatic void 198221828Sgrehanvtd_iotlb_global_invalidate(struct vtdmap *vtdmap) 199221828Sgrehan{ 200221828Sgrehan int offset; 201221828Sgrehan volatile uint64_t *iotlb_reg, val; 202221828Sgrehan 203221828Sgrehan vtd_wbflush(vtdmap); 204221828Sgrehan 205221828Sgrehan offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16; 206221828Sgrehan iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8); 207221828Sgrehan 208221828Sgrehan *iotlb_reg = VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL | 209221828Sgrehan VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES; 210221828Sgrehan 211221828Sgrehan while (1) { 212221828Sgrehan val = *iotlb_reg; 213221828Sgrehan if ((val & VTD_IIR_IVT) == 0) 214221828Sgrehan break; 215221828Sgrehan } 216221828Sgrehan} 217221828Sgrehan 218221828Sgrehanstatic void 219221828Sgrehanvtd_translation_enable(struct vtdmap *vtdmap) 220221828Sgrehan{ 221221828Sgrehan 222221828Sgrehan vtdmap->gcr = VTD_GCR_TE; 223221828Sgrehan while ((vtdmap->gsr & VTD_GSR_TES) == 0) 224221828Sgrehan ; 225221828Sgrehan} 226221828Sgrehan 227221828Sgrehanstatic void 228221828Sgrehanvtd_translation_disable(struct vtdmap *vtdmap) 229221828Sgrehan{ 230221828Sgrehan 231221828Sgrehan vtdmap->gcr = 0; 232221828Sgrehan while ((vtdmap->gsr & VTD_GSR_TES) != 0) 233221828Sgrehan ; 234221828Sgrehan} 235221828Sgrehan 236221828Sgrehanstatic int 237221828Sgrehanvtd_init(void) 238221828Sgrehan{ 239254548Sneel int i, units, remaining; 240221828Sgrehan struct vtdmap *vtdmap; 241221828Sgrehan vm_paddr_t ctx_paddr; 242254548Sneel char *end, envname[32]; 243254548Sneel unsigned long mapaddr; 244254548Sneel ACPI_STATUS status; 245254548Sneel ACPI_TABLE_DMAR *dmar; 246254548Sneel ACPI_DMAR_HEADER *hdr; 247254548Sneel ACPI_DMAR_HARDWARE_UNIT *drhd; 248254548Sneel 249254548Sneel /* 250254548Sneel * Allow the user to override the ACPI DMAR table by specifying the 251254548Sneel * physical address of each remapping unit. 252254548Sneel * 253254548Sneel * The following example specifies two remapping units at 254254548Sneel * physical addresses 0xfed90000 and 0xfeda0000 respectively. 255254548Sneel * set vtd.regmap.0.addr=0xfed90000 256254548Sneel * set vtd.regmap.1.addr=0xfeda0000 257254548Sneel */ 258254548Sneel for (units = 0; units < DRHD_MAX_UNITS; units++) { 259254548Sneel snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units); 260254548Sneel if (getenv_ulong(envname, &mapaddr) == 0) 261221828Sgrehan break; 262254548Sneel vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr); 263221828Sgrehan } 264221828Sgrehan 265254548Sneel if (units > 0) 266254548Sneel goto skip_dmar; 267254548Sneel 268254548Sneel /* Search for DMAR table. */ 269254548Sneel status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar); 270254548Sneel if (ACPI_FAILURE(status)) 271254548Sneel return (ENXIO); 272254548Sneel 273254548Sneel end = (char *)dmar + dmar->Header.Length; 274254548Sneel remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR); 275254548Sneel while (remaining > sizeof(ACPI_DMAR_HEADER)) { 276254548Sneel hdr = (ACPI_DMAR_HEADER *)(end - remaining); 277254548Sneel if (hdr->Length > remaining) 278254548Sneel break; 279254548Sneel /* 280254548Sneel * From Intel VT-d arch spec, version 1.3: 281254548Sneel * BIOS implementations must report mapping structures 282254548Sneel * in numerical order, i.e. All remapping structures of 283254548Sneel * type 0 (DRHD) enumerated before remapping structures of 284254548Sneel * type 1 (RMRR) and so forth. 285254548Sneel */ 286254548Sneel if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT) 287254548Sneel break; 288254548Sneel 289254548Sneel drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr; 290254548Sneel vtdmaps[units++] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address); 291254548Sneel if (units >= DRHD_MAX_UNITS) 292254548Sneel break; 293254548Sneel remaining -= hdr->Length; 294254548Sneel } 295254548Sneel 296221828Sgrehan if (units <= 0) 297221828Sgrehan return (ENXIO); 298221828Sgrehan 299254548Sneelskip_dmar: 300221828Sgrehan drhd_num = units; 301221828Sgrehan vtdmap = vtdmaps[0]; 302221828Sgrehan 303221828Sgrehan if (VTD_CAP_CM(vtdmap->cap) != 0) 304221828Sgrehan panic("vtd_init: invalid caching mode"); 305221828Sgrehan 306221828Sgrehan max_domains = vtd_max_domains(vtdmap); 307221828Sgrehan 308221828Sgrehan /* 309221828Sgrehan * Set up the root-table to point to the context-entry tables 310221828Sgrehan */ 311221828Sgrehan for (i = 0; i < 256; i++) { 312221828Sgrehan ctx_paddr = vtophys(ctx_tables[i]); 313221828Sgrehan if (ctx_paddr & PAGE_MASK) 314221828Sgrehan panic("ctx table (0x%0lx) not page aligned", ctx_paddr); 315221828Sgrehan 316221828Sgrehan root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT; 317221828Sgrehan } 318221828Sgrehan 319221828Sgrehan return (0); 320221828Sgrehan} 321221828Sgrehan 322221828Sgrehanstatic void 323221828Sgrehanvtd_cleanup(void) 324221828Sgrehan{ 325221828Sgrehan} 326221828Sgrehan 327221828Sgrehanstatic void 328221828Sgrehanvtd_enable(void) 329221828Sgrehan{ 330221828Sgrehan int i; 331221828Sgrehan struct vtdmap *vtdmap; 332221828Sgrehan 333221828Sgrehan for (i = 0; i < drhd_num; i++) { 334221828Sgrehan vtdmap = vtdmaps[i]; 335221828Sgrehan vtd_wbflush(vtdmap); 336221828Sgrehan 337221828Sgrehan /* Update the root table address */ 338221828Sgrehan vtdmap->rta = vtophys(root_table); 339221828Sgrehan vtdmap->gcr = VTD_GCR_SRTP; 340221828Sgrehan while ((vtdmap->gsr & VTD_GSR_RTPS) == 0) 341221828Sgrehan ; 342221828Sgrehan 343221828Sgrehan vtd_ctx_global_invalidate(vtdmap); 344221828Sgrehan vtd_iotlb_global_invalidate(vtdmap); 345221828Sgrehan 346221828Sgrehan vtd_translation_enable(vtdmap); 347221828Sgrehan } 348221828Sgrehan} 349221828Sgrehan 350221828Sgrehanstatic void 351221828Sgrehanvtd_disable(void) 352221828Sgrehan{ 353221828Sgrehan int i; 354221828Sgrehan struct vtdmap *vtdmap; 355221828Sgrehan 356221828Sgrehan for (i = 0; i < drhd_num; i++) { 357221828Sgrehan vtdmap = vtdmaps[i]; 358221828Sgrehan vtd_translation_disable(vtdmap); 359221828Sgrehan } 360221828Sgrehan} 361221828Sgrehan 362221828Sgrehanstatic void 363221828Sgrehanvtd_add_device(void *arg, int bus, int slot, int func) 364221828Sgrehan{ 365221828Sgrehan int idx; 366221828Sgrehan uint64_t *ctxp; 367221828Sgrehan struct domain *dom = arg; 368221828Sgrehan vm_paddr_t pt_paddr; 369221828Sgrehan struct vtdmap *vtdmap; 370221828Sgrehan 371221828Sgrehan if (bus < 0 || bus > PCI_BUSMAX || 372221828Sgrehan slot < 0 || slot > PCI_SLOTMAX || 373221828Sgrehan func < 0 || func > PCI_FUNCMAX) 374221828Sgrehan panic("vtd_add_device: invalid bsf %d/%d/%d", bus, slot, func); 375221828Sgrehan 376221828Sgrehan vtdmap = vtdmaps[0]; 377221828Sgrehan ctxp = ctx_tables[bus]; 378221828Sgrehan pt_paddr = vtophys(dom->ptp); 379221828Sgrehan idx = (slot << 3 | func) * 2; 380221828Sgrehan 381221828Sgrehan if (ctxp[idx] & VTD_CTX_PRESENT) { 382221828Sgrehan panic("vtd_add_device: device %d/%d/%d is already owned by " 383221828Sgrehan "domain %d", bus, slot, func, 384221828Sgrehan (uint16_t)(ctxp[idx + 1] >> 8)); 385221828Sgrehan } 386221828Sgrehan 387221828Sgrehan /* 388221828Sgrehan * Order is important. The 'present' bit is set only after all fields 389221828Sgrehan * of the context pointer are initialized. 390221828Sgrehan */ 391221828Sgrehan ctxp[idx + 1] = dom->addrwidth | (dom->id << 8); 392221828Sgrehan 393221828Sgrehan if (VTD_ECAP_DI(vtdmap->ext_cap)) 394221828Sgrehan ctxp[idx] = VTD_CTX_TT_ALL; 395221828Sgrehan else 396221828Sgrehan ctxp[idx] = 0; 397221828Sgrehan 398221828Sgrehan ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT; 399221828Sgrehan 400221828Sgrehan /* 401221828Sgrehan * 'Not Present' entries are not cached in either the Context Cache 402221828Sgrehan * or in the IOTLB, so there is no need to invalidate either of them. 403221828Sgrehan */ 404221828Sgrehan} 405221828Sgrehan 406221828Sgrehanstatic void 407221828Sgrehanvtd_remove_device(void *arg, int bus, int slot, int func) 408221828Sgrehan{ 409221828Sgrehan int i, idx; 410221828Sgrehan uint64_t *ctxp; 411221828Sgrehan struct vtdmap *vtdmap; 412221828Sgrehan 413221828Sgrehan if (bus < 0 || bus > PCI_BUSMAX || 414221828Sgrehan slot < 0 || slot > PCI_SLOTMAX || 415221828Sgrehan func < 0 || func > PCI_FUNCMAX) 416221828Sgrehan panic("vtd_add_device: invalid bsf %d/%d/%d", bus, slot, func); 417221828Sgrehan 418221828Sgrehan ctxp = ctx_tables[bus]; 419221828Sgrehan idx = (slot << 3 | func) * 2; 420221828Sgrehan 421221828Sgrehan /* 422221828Sgrehan * Order is important. The 'present' bit is must be cleared first. 423221828Sgrehan */ 424221828Sgrehan ctxp[idx] = 0; 425221828Sgrehan ctxp[idx + 1] = 0; 426221828Sgrehan 427221828Sgrehan /* 428221828Sgrehan * Invalidate the Context Cache and the IOTLB. 429221828Sgrehan * 430221828Sgrehan * XXX use device-selective invalidation for Context Cache 431221828Sgrehan * XXX use domain-selective invalidation for IOTLB 432221828Sgrehan */ 433221828Sgrehan for (i = 0; i < drhd_num; i++) { 434221828Sgrehan vtdmap = vtdmaps[i]; 435221828Sgrehan vtd_ctx_global_invalidate(vtdmap); 436221828Sgrehan vtd_iotlb_global_invalidate(vtdmap); 437221828Sgrehan } 438221828Sgrehan} 439221828Sgrehan 440241362Sneel#define CREATE_MAPPING 0 441241362Sneel#define REMOVE_MAPPING 1 442241362Sneel 443221828Sgrehanstatic uint64_t 444241362Sneelvtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len, 445241362Sneel int remove) 446221828Sgrehan{ 447221828Sgrehan struct domain *dom; 448221828Sgrehan int i, spshift, ptpshift, ptpindex, nlevels; 449221828Sgrehan uint64_t spsize, *ptp; 450221828Sgrehan 451221828Sgrehan dom = arg; 452221828Sgrehan ptpindex = 0; 453221828Sgrehan ptpshift = 0; 454221828Sgrehan 455270159Sgrehan KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__, 456270159Sgrehan gpa, len)); 457270159Sgrehan KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond " 458270159Sgrehan "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr)); 459270159Sgrehan 460221828Sgrehan if (gpa & PAGE_MASK) 461221828Sgrehan panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa); 462221828Sgrehan 463221828Sgrehan if (hpa & PAGE_MASK) 464221828Sgrehan panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa); 465221828Sgrehan 466221828Sgrehan if (len & PAGE_MASK) 467221828Sgrehan panic("vtd_create_mapping: unaligned len 0x%0lx", len); 468221828Sgrehan 469221828Sgrehan /* 470221828Sgrehan * Compute the size of the mapping that we can accomodate. 471221828Sgrehan * 472221828Sgrehan * This is based on three factors: 473221828Sgrehan * - supported super page size 474221828Sgrehan * - alignment of the region starting at 'gpa' and 'hpa' 475221828Sgrehan * - length of the region 'len' 476221828Sgrehan */ 477221828Sgrehan spshift = 48; 478221828Sgrehan for (i = 3; i >= 0; i--) { 479221828Sgrehan spsize = 1UL << spshift; 480221828Sgrehan if ((dom->spsmask & (1 << i)) != 0 && 481221828Sgrehan (gpa & (spsize - 1)) == 0 && 482221828Sgrehan (hpa & (spsize - 1)) == 0 && 483221828Sgrehan (len >= spsize)) { 484221828Sgrehan break; 485221828Sgrehan } 486221828Sgrehan spshift -= 9; 487221828Sgrehan } 488221828Sgrehan 489221828Sgrehan ptp = dom->ptp; 490221828Sgrehan nlevels = dom->pt_levels; 491221828Sgrehan while (--nlevels >= 0) { 492221828Sgrehan ptpshift = 12 + nlevels * 9; 493221828Sgrehan ptpindex = (gpa >> ptpshift) & 0x1FF; 494221828Sgrehan 495221828Sgrehan /* We have reached the leaf mapping */ 496221828Sgrehan if (spshift >= ptpshift) { 497221828Sgrehan break; 498221828Sgrehan } 499221828Sgrehan 500221828Sgrehan /* 501221828Sgrehan * We are working on a non-leaf page table page. 502221828Sgrehan * 503221828Sgrehan * Create a downstream page table page if necessary and point 504221828Sgrehan * to it from the current page table. 505221828Sgrehan */ 506221828Sgrehan if (ptp[ptpindex] == 0) { 507221828Sgrehan void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO); 508221828Sgrehan ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR; 509221828Sgrehan } 510221828Sgrehan 511221828Sgrehan ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M); 512221828Sgrehan } 513221828Sgrehan 514221828Sgrehan if ((gpa & ((1UL << ptpshift) - 1)) != 0) 515221828Sgrehan panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift); 516221828Sgrehan 517221828Sgrehan /* 518241362Sneel * Update the 'gpa' -> 'hpa' mapping 519221828Sgrehan */ 520241362Sneel if (remove) { 521241362Sneel ptp[ptpindex] = 0; 522241362Sneel } else { 523241362Sneel ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR; 524221828Sgrehan 525241362Sneel if (nlevels > 0) 526241362Sneel ptp[ptpindex] |= VTD_PTE_SUPERPAGE; 527241362Sneel } 528221828Sgrehan 529221828Sgrehan return (1UL << ptpshift); 530221828Sgrehan} 531221828Sgrehan 532241362Sneelstatic uint64_t 533241362Sneelvtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 534241362Sneel{ 535241362Sneel 536241362Sneel return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING)); 537241362Sneel} 538241362Sneel 539241362Sneelstatic uint64_t 540241362Sneelvtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 541241362Sneel{ 542241362Sneel 543241362Sneel return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING)); 544241362Sneel} 545241362Sneel 546241362Sneelstatic void 547241362Sneelvtd_invalidate_tlb(void *dom) 548241362Sneel{ 549241362Sneel int i; 550241362Sneel struct vtdmap *vtdmap; 551241362Sneel 552241362Sneel /* 553241362Sneel * Invalidate the IOTLB. 554241362Sneel * XXX use domain-selective invalidation for IOTLB 555241362Sneel */ 556241362Sneel for (i = 0; i < drhd_num; i++) { 557241362Sneel vtdmap = vtdmaps[i]; 558241362Sneel vtd_iotlb_global_invalidate(vtdmap); 559241362Sneel } 560241362Sneel} 561241362Sneel 562221828Sgrehanstatic void * 563221828Sgrehanvtd_create_domain(vm_paddr_t maxaddr) 564221828Sgrehan{ 565221828Sgrehan struct domain *dom; 566221828Sgrehan vm_paddr_t addr; 567221828Sgrehan int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth; 568221828Sgrehan struct vtdmap *vtdmap; 569221828Sgrehan 570221828Sgrehan if (drhd_num <= 0) 571221828Sgrehan panic("vtd_create_domain: no dma remapping hardware available"); 572221828Sgrehan 573221828Sgrehan vtdmap = vtdmaps[0]; 574221828Sgrehan 575221828Sgrehan /* 576221828Sgrehan * Calculate AGAW. 577221828Sgrehan * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec. 578221828Sgrehan */ 579221828Sgrehan addr = 0; 580221828Sgrehan for (gaw = 0; addr < maxaddr; gaw++) 581221828Sgrehan addr = 1ULL << gaw; 582221828Sgrehan 583221828Sgrehan res = (gaw - 12) % 9; 584221828Sgrehan if (res == 0) 585221828Sgrehan agaw = gaw; 586221828Sgrehan else 587221828Sgrehan agaw = gaw + 9 - res; 588221828Sgrehan 589221828Sgrehan if (agaw > 64) 590221828Sgrehan agaw = 64; 591221828Sgrehan 592221828Sgrehan /* 593221828Sgrehan * Select the smallest Supported AGAW and the corresponding number 594221828Sgrehan * of page table levels. 595221828Sgrehan */ 596221828Sgrehan pt_levels = 2; 597221828Sgrehan sagaw = 30; 598221828Sgrehan addrwidth = 0; 599221828Sgrehan tmp = VTD_CAP_SAGAW(vtdmap->cap); 600221828Sgrehan for (i = 0; i < 5; i++) { 601221828Sgrehan if ((tmp & (1 << i)) != 0 && sagaw >= agaw) 602221828Sgrehan break; 603221828Sgrehan pt_levels++; 604221828Sgrehan addrwidth++; 605221828Sgrehan sagaw += 9; 606221828Sgrehan if (sagaw > 64) 607221828Sgrehan sagaw = 64; 608221828Sgrehan } 609221828Sgrehan 610221828Sgrehan if (i >= 5) { 611221828Sgrehan panic("vtd_create_domain: SAGAW 0x%lx does not support AGAW %d", 612221828Sgrehan VTD_CAP_SAGAW(vtdmap->cap), agaw); 613221828Sgrehan } 614221828Sgrehan 615221828Sgrehan dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK); 616221828Sgrehan dom->pt_levels = pt_levels; 617221828Sgrehan dom->addrwidth = addrwidth; 618221828Sgrehan dom->id = domain_id(); 619221828Sgrehan dom->maxaddr = maxaddr; 620221828Sgrehan dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK); 621221828Sgrehan if ((uintptr_t)dom->ptp & PAGE_MASK) 622221828Sgrehan panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp); 623221828Sgrehan 624254549Sneel#ifdef notyet 625254549Sneel /* 626254549Sneel * XXX superpage mappings for the iommu do not work correctly. 627254549Sneel * 628254549Sneel * By default all physical memory is mapped into the host_domain. 629254549Sneel * When a VM is allocated wired memory the pages belonging to it 630254549Sneel * are removed from the host_domain and added to the vm's domain. 631254549Sneel * 632254549Sneel * If the page being removed was mapped using a superpage mapping 633254549Sneel * in the host_domain then we need to demote the mapping before 634254549Sneel * removing the page. 635254549Sneel * 636254549Sneel * There is not any code to deal with the demotion at the moment 637254549Sneel * so we disable superpage mappings altogether. 638254549Sneel */ 639254549Sneel dom->spsmask = VTD_CAP_SPS(vtdmap->cap); 640254549Sneel#endif 641254549Sneel 642221828Sgrehan SLIST_INSERT_HEAD(&domhead, dom, next); 643221828Sgrehan 644221828Sgrehan return (dom); 645221828Sgrehan} 646221828Sgrehan 647221828Sgrehanstatic void 648221828Sgrehanvtd_free_ptp(uint64_t *ptp, int level) 649221828Sgrehan{ 650221828Sgrehan int i; 651221828Sgrehan uint64_t *nlp; 652221828Sgrehan 653221828Sgrehan if (level > 1) { 654221828Sgrehan for (i = 0; i < 512; i++) { 655221828Sgrehan if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0) 656221828Sgrehan continue; 657221828Sgrehan if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0) 658221828Sgrehan continue; 659221828Sgrehan nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M); 660221828Sgrehan vtd_free_ptp(nlp, level - 1); 661221828Sgrehan } 662221828Sgrehan } 663221828Sgrehan 664221828Sgrehan bzero(ptp, PAGE_SIZE); 665221828Sgrehan free(ptp, M_VTD); 666221828Sgrehan} 667221828Sgrehan 668221828Sgrehanstatic void 669221828Sgrehanvtd_destroy_domain(void *arg) 670221828Sgrehan{ 671221828Sgrehan struct domain *dom; 672221828Sgrehan 673221828Sgrehan dom = arg; 674221828Sgrehan 675221828Sgrehan SLIST_REMOVE(&domhead, dom, domain, next); 676221828Sgrehan vtd_free_ptp(dom->ptp, dom->pt_levels); 677221828Sgrehan free(dom, M_VTD); 678221828Sgrehan} 679221828Sgrehan 680221828Sgrehanstruct iommu_ops iommu_ops_intel = { 681221828Sgrehan vtd_init, 682221828Sgrehan vtd_cleanup, 683221828Sgrehan vtd_enable, 684221828Sgrehan vtd_disable, 685221828Sgrehan vtd_create_domain, 686221828Sgrehan vtd_destroy_domain, 687221828Sgrehan vtd_create_mapping, 688241362Sneel vtd_remove_mapping, 689221828Sgrehan vtd_add_device, 690221828Sgrehan vtd_remove_device, 691241362Sneel vtd_invalidate_tlb, 692221828Sgrehan}; 693