1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD$"); 32257251Skib 33257251Skib#include <sys/param.h> 34257251Skib#include <sys/bus.h> 35257251Skib#include <sys/kernel.h> 36257251Skib#include <sys/lock.h> 37257251Skib#include <sys/malloc.h> 38257251Skib#include <sys/memdesc.h> 39257251Skib#include <sys/mutex.h> 40257251Skib#include <sys/proc.h> 41257251Skib#include <sys/queue.h> 42257251Skib#include <sys/rman.h> 43257251Skib#include <sys/rwlock.h> 44257251Skib#include <sys/sched.h> 45257251Skib#include <sys/sf_buf.h> 46257251Skib#include <sys/sysctl.h> 47257251Skib#include <sys/systm.h> 48257251Skib#include <sys/taskqueue.h> 49257251Skib#include <sys/tree.h> 50279470Srstone#include <dev/pci/pcivar.h> 51257251Skib#include <vm/vm.h> 52257251Skib#include <vm/vm_extern.h> 53257251Skib#include <vm/vm_kern.h> 54257251Skib#include <vm/vm_object.h> 55257251Skib#include <vm/vm_page.h> 56257251Skib#include <vm/vm_map.h> 57257251Skib#include <vm/vm_pageout.h> 58257251Skib#include <machine/bus.h> 59257251Skib#include <machine/cpu.h> 60257251Skib#include <x86/include/busdma_impl.h> 61257251Skib#include <x86/iommu/intel_reg.h> 62257251Skib#include <x86/iommu/busdma_dmar.h> 63257251Skib#include <x86/iommu/intel_dmar.h> 64257251Skib 65257251Skibu_int 66257251Skibdmar_nd2mask(u_int nd) 67257251Skib{ 68257251Skib static const u_int masks[] = { 69257251Skib 0x000f, /* nd == 0 */ 70257251Skib 0x002f, /* nd == 1 */ 71257251Skib 0x00ff, /* nd == 2 */ 72257251Skib 0x02ff, /* nd == 3 */ 73257251Skib 0x0fff, /* nd == 4 */ 74257251Skib 0x2fff, /* nd == 5 */ 75257251Skib 0xffff, /* nd == 6 */ 76257251Skib 0x0000, /* nd == 7 reserved */ 77257251Skib }; 78257251Skib 79257251Skib KASSERT(nd <= 6, ("number of domains %d", nd)); 80257251Skib return (masks[nd]); 81257251Skib} 82257251Skib 83257251Skibstatic const struct sagaw_bits_tag { 84257251Skib int agaw; 85257251Skib int cap; 86257251Skib int awlvl; 87257251Skib int pglvl; 88257251Skib} sagaw_bits[] = { 89257251Skib {.agaw = 30, .cap = DMAR_CAP_SAGAW_2LVL, .awlvl = DMAR_CTX2_AW_2LVL, 90257251Skib .pglvl = 2}, 91257251Skib {.agaw = 39, .cap = DMAR_CAP_SAGAW_3LVL, .awlvl = DMAR_CTX2_AW_3LVL, 92257251Skib .pglvl = 3}, 93257251Skib {.agaw = 48, .cap = DMAR_CAP_SAGAW_4LVL, .awlvl = DMAR_CTX2_AW_4LVL, 94257251Skib .pglvl = 4}, 95257251Skib {.agaw = 57, .cap = DMAR_CAP_SAGAW_5LVL, .awlvl = DMAR_CTX2_AW_5LVL, 96257251Skib .pglvl = 5}, 97257251Skib {.agaw = 64, .cap = DMAR_CAP_SAGAW_6LVL, .awlvl = DMAR_CTX2_AW_6LVL, 98257251Skib .pglvl = 6} 99257251Skib}; 100257251Skib#define SIZEOF_SAGAW_BITS (sizeof(sagaw_bits) / sizeof(sagaw_bits[0])) 101257251Skib 102257251Skibbool 103257251Skibdmar_pglvl_supported(struct dmar_unit *unit, int pglvl) 104257251Skib{ 105257251Skib int i; 106257251Skib 107257251Skib for (i = 0; i < SIZEOF_SAGAW_BITS; i++) { 108257251Skib if (sagaw_bits[i].pglvl != pglvl) 109257251Skib continue; 110257251Skib if ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) 111257251Skib return (true); 112257251Skib } 113257251Skib return (false); 114257251Skib} 115257251Skib 116257251Skibint 117257251Skibctx_set_agaw(struct dmar_ctx *ctx, int mgaw) 118257251Skib{ 119257251Skib int sagaw, i; 120257251Skib 121257251Skib ctx->mgaw = mgaw; 122257251Skib sagaw = DMAR_CAP_SAGAW(ctx->dmar->hw_cap); 123257251Skib for (i = 0; i < SIZEOF_SAGAW_BITS; i++) { 124257251Skib if (sagaw_bits[i].agaw >= mgaw) { 125257251Skib ctx->agaw = sagaw_bits[i].agaw; 126257251Skib ctx->pglvl = sagaw_bits[i].pglvl; 127257251Skib ctx->awlvl = sagaw_bits[i].awlvl; 128257251Skib return (0); 129257251Skib } 130257251Skib } 131257251Skib device_printf(ctx->dmar->dev, 132257251Skib "context request mgaw %d for pci%d:%d:%d:%d, " 133279470Srstone "no agaw found, sagaw %x\n", mgaw, ctx->dmar->segment, 134279470Srstone pci_get_bus(ctx->ctx_tag.owner), 135279470Srstone pci_get_slot(ctx->ctx_tag.owner), 136279470Srstone pci_get_function(ctx->ctx_tag.owner), sagaw); 137257251Skib return (EINVAL); 138257251Skib} 139257251Skib 140257251Skib/* 141257251Skib * Find a best fit mgaw for the given maxaddr: 142257251Skib * - if allow_less is false, must find sagaw which maps all requested 143257251Skib * addresses (used by identity mappings); 144257251Skib * - if allow_less is true, and no supported sagaw can map all requested 145257251Skib * address space, accept the biggest sagaw, whatever is it. 146257251Skib */ 147257251Skibint 148257251Skibdmar_maxaddr2mgaw(struct dmar_unit *unit, dmar_gaddr_t maxaddr, bool allow_less) 149257251Skib{ 150257251Skib int i; 151257251Skib 152257251Skib for (i = 0; i < SIZEOF_SAGAW_BITS; i++) { 153257251Skib if ((1ULL << sagaw_bits[i].agaw) >= maxaddr && 154257251Skib (DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) 155257251Skib break; 156257251Skib } 157257251Skib if (allow_less && i == SIZEOF_SAGAW_BITS) { 158257251Skib do { 159257251Skib i--; 160257251Skib } while ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) 161257251Skib == 0); 162257251Skib } 163257251Skib if (i < SIZEOF_SAGAW_BITS) 164257251Skib return (sagaw_bits[i].agaw); 165257251Skib KASSERT(0, ("no mgaw for maxaddr %jx allow_less %d", 166257251Skib (uintmax_t) maxaddr, allow_less)); 167257251Skib return (-1); 168257251Skib} 169257251Skib 170257251Skib/* 171257251Skib * Calculate the total amount of page table pages needed to map the 172257251Skib * whole bus address space on the context with the selected agaw. 173257251Skib */ 174257251Skibvm_pindex_t 175257251Skibpglvl_max_pages(int pglvl) 176257251Skib{ 177257251Skib vm_pindex_t res; 178257251Skib int i; 179257251Skib 180257251Skib for (res = 0, i = pglvl; i > 0; i--) { 181257251Skib res *= DMAR_NPTEPG; 182257251Skib res++; 183257251Skib } 184257251Skib return (res); 185257251Skib} 186257251Skib 187257251Skib/* 188257251Skib * Return true if the page table level lvl supports the superpage for 189257251Skib * the context ctx. 190257251Skib */ 191257251Skibint 192257251Skibctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl) 193257251Skib{ 194257251Skib int alvl, cap_sps; 195257251Skib static const int sagaw_sp[] = { 196257251Skib DMAR_CAP_SPS_2M, 197257251Skib DMAR_CAP_SPS_1G, 198257251Skib DMAR_CAP_SPS_512G, 199257251Skib DMAR_CAP_SPS_1T 200257251Skib }; 201257251Skib 202257251Skib alvl = ctx->pglvl - lvl - 1; 203257251Skib cap_sps = DMAR_CAP_SPS(ctx->dmar->hw_cap); 204257251Skib return (alvl < sizeof(sagaw_sp) / sizeof(sagaw_sp[0]) && 205257251Skib (sagaw_sp[alvl] & cap_sps) != 0); 206257251Skib} 207257251Skib 208257251Skibdmar_gaddr_t 209257251Skibpglvl_page_size(int total_pglvl, int lvl) 210257251Skib{ 211257251Skib int rlvl; 212257251Skib static const dmar_gaddr_t pg_sz[] = { 213257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE, 214257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE << DMAR_NPTEPGSHIFT, 215257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE << (2 * DMAR_NPTEPGSHIFT), 216257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE << (3 * DMAR_NPTEPGSHIFT), 217257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE << (4 * DMAR_NPTEPGSHIFT), 218257251Skib (dmar_gaddr_t)DMAR_PAGE_SIZE << (5 * DMAR_NPTEPGSHIFT) 219257251Skib }; 220257251Skib 221257251Skib KASSERT(lvl >= 0 && lvl < total_pglvl, 222257251Skib ("total %d lvl %d", total_pglvl, lvl)); 223257251Skib rlvl = total_pglvl - lvl - 1; 224257251Skib KASSERT(rlvl < sizeof(pg_sz) / sizeof(pg_sz[0]), 225257251Skib ("sizeof pg_sz lvl %d", lvl)); 226257251Skib return (pg_sz[rlvl]); 227257251Skib} 228257251Skib 229257251Skibdmar_gaddr_t 230257251Skibctx_page_size(struct dmar_ctx *ctx, int lvl) 231257251Skib{ 232257251Skib 233257251Skib return (pglvl_page_size(ctx->pglvl, lvl)); 234257251Skib} 235257251Skib 236259512Skibint 237259512Skibcalc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, 238259512Skib dmar_gaddr_t *isizep) 239259512Skib{ 240259512Skib dmar_gaddr_t isize; 241259512Skib int am; 242259512Skib 243259512Skib for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) { 244259512Skib isize = 1ULL << (am + DMAR_PAGE_SHIFT); 245259512Skib if ((base & (isize - 1)) == 0 && size >= isize) 246259512Skib break; 247259512Skib if (am == 0) 248259512Skib break; 249259512Skib } 250259512Skib *isizep = isize; 251259512Skib return (am); 252259512Skib} 253259512Skib 254257251Skibdmar_haddr_t dmar_high; 255257251Skibint haw; 256257251Skibint dmar_tbl_pagecnt; 257257251Skib 258257251Skibvm_page_t 259257251Skibdmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags) 260257251Skib{ 261257251Skib vm_page_t m; 262257251Skib int zeroed; 263257251Skib 264257251Skib zeroed = (flags & DMAR_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0; 265257251Skib for (;;) { 266257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 267257251Skib VM_OBJECT_WLOCK(obj); 268257251Skib m = vm_page_lookup(obj, idx); 269257251Skib if ((flags & DMAR_PGF_NOALLOC) != 0 || m != NULL) { 270257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 271257251Skib VM_OBJECT_WUNLOCK(obj); 272257251Skib break; 273257251Skib } 274257251Skib m = vm_page_alloc_contig(obj, idx, VM_ALLOC_NOBUSY | 275257251Skib VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP | zeroed, 1, 0, 276257251Skib dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 277257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 278257251Skib VM_OBJECT_WUNLOCK(obj); 279257251Skib if (m != NULL) { 280257251Skib if (zeroed && (m->flags & PG_ZERO) == 0) 281257251Skib pmap_zero_page(m); 282257251Skib atomic_add_int(&dmar_tbl_pagecnt, 1); 283257251Skib break; 284257251Skib } 285257251Skib if ((flags & DMAR_PGF_WAITOK) == 0) 286257251Skib break; 287257251Skib if ((flags & DMAR_PGF_OBJL) != 0) 288257251Skib VM_OBJECT_WUNLOCK(obj); 289257251Skib VM_WAIT; 290257251Skib if ((flags & DMAR_PGF_OBJL) != 0) 291257251Skib VM_OBJECT_WLOCK(obj); 292257251Skib } 293257251Skib return (m); 294257251Skib} 295257251Skib 296257251Skibvoid 297257251Skibdmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags) 298257251Skib{ 299257251Skib vm_page_t m; 300257251Skib 301257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 302257251Skib VM_OBJECT_WLOCK(obj); 303257251Skib m = vm_page_lookup(obj, idx); 304257251Skib if (m != NULL) { 305257251Skib vm_page_free(m); 306257251Skib atomic_subtract_int(&dmar_tbl_pagecnt, 1); 307257251Skib } 308257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 309257251Skib VM_OBJECT_WUNLOCK(obj); 310257251Skib} 311257251Skib 312257251Skibvoid * 313257251Skibdmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, 314257251Skib struct sf_buf **sf) 315257251Skib{ 316257251Skib vm_page_t m; 317257251Skib bool allocated; 318257251Skib 319257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 320257251Skib VM_OBJECT_WLOCK(obj); 321257251Skib m = vm_page_lookup(obj, idx); 322257251Skib if (m == NULL && (flags & DMAR_PGF_ALLOC) != 0) { 323257251Skib m = dmar_pgalloc(obj, idx, flags | DMAR_PGF_OBJL); 324257251Skib allocated = true; 325257251Skib } else 326257251Skib allocated = false; 327257251Skib if (m == NULL) { 328257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 329257251Skib VM_OBJECT_WUNLOCK(obj); 330257251Skib return (NULL); 331257251Skib } 332257251Skib /* Sleepable allocations cannot fail. */ 333257251Skib if ((flags & DMAR_PGF_WAITOK) != 0) 334257251Skib VM_OBJECT_WUNLOCK(obj); 335257251Skib sched_pin(); 336257251Skib *sf = sf_buf_alloc(m, SFB_CPUPRIVATE | ((flags & DMAR_PGF_WAITOK) 337257251Skib == 0 ? SFB_NOWAIT : 0)); 338257251Skib if (*sf == NULL) { 339257251Skib sched_unpin(); 340257251Skib if (allocated) { 341257251Skib VM_OBJECT_ASSERT_WLOCKED(obj); 342257251Skib dmar_pgfree(obj, m->pindex, flags | DMAR_PGF_OBJL); 343257251Skib } 344257251Skib if ((flags & DMAR_PGF_OBJL) == 0) 345257251Skib VM_OBJECT_WUNLOCK(obj); 346257251Skib return (NULL); 347257251Skib } 348257251Skib if ((flags & (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) == 349257251Skib (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) 350257251Skib VM_OBJECT_WLOCK(obj); 351257251Skib else if ((flags & (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) == 0) 352257251Skib VM_OBJECT_WUNLOCK(obj); 353257251Skib return ((void *)sf_buf_kva(*sf)); 354257251Skib} 355257251Skib 356257251Skibvoid 357277315Skibdmar_unmap_pgtbl(struct sf_buf *sf) 358257251Skib{ 359257251Skib 360257251Skib sf_buf_free(sf); 361257251Skib sched_unpin(); 362277315Skib} 363257251Skib 364277315Skibstatic void 365277315Skibdmar_flush_transl_to_ram(struct dmar_unit *unit, void *dst, size_t sz) 366277315Skib{ 367277315Skib 368277315Skib if (DMAR_IS_COHERENT(unit)) 369277315Skib return; 370257251Skib /* 371257251Skib * If DMAR does not snoop paging structures accesses, flush 372257251Skib * CPU cache to memory. 373257251Skib */ 374277315Skib pmap_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz, 375277315Skib TRUE); 376257251Skib} 377257251Skib 378277315Skibvoid 379277315Skibdmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst) 380277315Skib{ 381277315Skib 382277315Skib dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); 383277315Skib} 384277315Skib 385277315Skibvoid 386277315Skibdmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst) 387277315Skib{ 388277315Skib 389277315Skib dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); 390277315Skib} 391277315Skib 392277315Skibvoid 393277315Skibdmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst) 394277315Skib{ 395277315Skib 396277315Skib dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); 397277315Skib} 398277315Skib 399257251Skib/* 400257251Skib * Load the root entry pointer into the hardware, busily waiting for 401257251Skib * the completion. 402257251Skib */ 403257251Skibint 404257251Skibdmar_load_root_entry_ptr(struct dmar_unit *unit) 405257251Skib{ 406257251Skib vm_page_t root_entry; 407257251Skib 408257251Skib /* 409257251Skib * Access to the GCMD register must be serialized while the 410257251Skib * command is submitted. 411257251Skib */ 412257251Skib DMAR_ASSERT_LOCKED(unit); 413257251Skib 414278946Skib VM_OBJECT_RLOCK(unit->ctx_obj); 415257251Skib root_entry = vm_page_lookup(unit->ctx_obj, 0); 416278946Skib VM_OBJECT_RUNLOCK(unit->ctx_obj); 417257251Skib dmar_write8(unit, DMAR_RTADDR_REG, VM_PAGE_TO_PHYS(root_entry)); 418257251Skib dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SRTP); 419257251Skib /* XXXKIB should have a timeout */ 420257251Skib while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_RTPS) == 0) 421257251Skib cpu_spinwait(); 422257251Skib return (0); 423257251Skib} 424257251Skib 425257251Skib/* 426257251Skib * Globally invalidate the context entries cache, busily waiting for 427257251Skib * the completion. 428257251Skib */ 429257251Skibint 430257251Skibdmar_inv_ctx_glob(struct dmar_unit *unit) 431257251Skib{ 432257251Skib 433257251Skib /* 434257251Skib * Access to the CCMD register must be serialized while the 435257251Skib * command is submitted. 436257251Skib */ 437257251Skib DMAR_ASSERT_LOCKED(unit); 438259512Skib KASSERT(!unit->qi_enabled, ("QI enabled")); 439257251Skib 440257251Skib /* 441257251Skib * The DMAR_CCMD_ICC bit in the upper dword should be written 442257251Skib * after the low dword write is completed. Amd64 443257251Skib * dmar_write8() does not have this issue, i386 dmar_write8() 444257251Skib * writes the upper dword last. 445257251Skib */ 446257251Skib dmar_write8(unit, DMAR_CCMD_REG, DMAR_CCMD_ICC | DMAR_CCMD_CIRG_GLOB); 447257251Skib /* XXXKIB should have a timeout */ 448257251Skib while ((dmar_read4(unit, DMAR_CCMD_REG + 4) & DMAR_CCMD_ICC32) != 0) 449257251Skib cpu_spinwait(); 450257251Skib return (0); 451257251Skib} 452257251Skib 453257251Skib/* 454257251Skib * Globally invalidate the IOTLB, busily waiting for the completion. 455257251Skib */ 456257251Skibint 457257251Skibdmar_inv_iotlb_glob(struct dmar_unit *unit) 458257251Skib{ 459257251Skib int reg; 460257251Skib 461257251Skib DMAR_ASSERT_LOCKED(unit); 462259512Skib KASSERT(!unit->qi_enabled, ("QI enabled")); 463257251Skib 464257251Skib reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap); 465257251Skib /* See a comment about DMAR_CCMD_ICC in dmar_inv_ctx_glob. */ 466257251Skib dmar_write8(unit, reg + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT | 467257251Skib DMAR_IOTLB_IIRG_GLB | DMAR_IOTLB_DR | DMAR_IOTLB_DW); 468257251Skib /* XXXKIB should have a timeout */ 469257251Skib while ((dmar_read4(unit, reg + DMAR_IOTLB_REG_OFF + 4) & 470257251Skib DMAR_IOTLB_IVT32) != 0) 471257251Skib cpu_spinwait(); 472257251Skib return (0); 473257251Skib} 474257251Skib 475257251Skib/* 476257251Skib * Flush the chipset write buffers. See 11.1 "Write Buffer Flushing" 477257251Skib * in the architecture specification. 478257251Skib */ 479257251Skibint 480257251Skibdmar_flush_write_bufs(struct dmar_unit *unit) 481257251Skib{ 482257251Skib 483257251Skib DMAR_ASSERT_LOCKED(unit); 484257251Skib 485257251Skib /* 486257251Skib * DMAR_GCMD_WBF is only valid when CAP_RWBF is reported. 487257251Skib */ 488257251Skib KASSERT((unit->hw_cap & DMAR_CAP_RWBF) != 0, 489257251Skib ("dmar%d: no RWBF", unit->unit)); 490257251Skib 491257251Skib dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_WBF); 492257251Skib /* XXXKIB should have a timeout */ 493257251Skib while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_WBFS) == 0) 494257251Skib cpu_spinwait(); 495257251Skib return (0); 496257251Skib} 497257251Skib 498257251Skibint 499257251Skibdmar_enable_translation(struct dmar_unit *unit) 500257251Skib{ 501257251Skib 502257251Skib DMAR_ASSERT_LOCKED(unit); 503257251Skib unit->hw_gcmd |= DMAR_GCMD_TE; 504257251Skib dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 505257251Skib /* XXXKIB should have a timeout */ 506257251Skib while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) == 0) 507257251Skib cpu_spinwait(); 508257251Skib return (0); 509257251Skib} 510257251Skib 511257251Skibint 512257251Skibdmar_disable_translation(struct dmar_unit *unit) 513257251Skib{ 514257251Skib 515257251Skib DMAR_ASSERT_LOCKED(unit); 516257251Skib unit->hw_gcmd &= ~DMAR_GCMD_TE; 517257251Skib dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 518257251Skib /* XXXKIB should have a timeout */ 519257251Skib while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) != 0) 520257251Skib cpu_spinwait(); 521257251Skib return (0); 522257251Skib} 523257251Skib 524257251Skib#define BARRIER_F \ 525257251Skib u_int f_done, f_inproc, f_wakeup; \ 526257251Skib \ 527257251Skib f_done = 1 << (barrier_id * 3); \ 528257251Skib f_inproc = 1 << (barrier_id * 3 + 1); \ 529257251Skib f_wakeup = 1 << (barrier_id * 3 + 2) 530257251Skib 531257251Skibbool 532257251Skibdmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id) 533257251Skib{ 534257251Skib BARRIER_F; 535257251Skib 536257251Skib DMAR_LOCK(dmar); 537257251Skib if ((dmar->barrier_flags & f_done) != 0) { 538257251Skib DMAR_UNLOCK(dmar); 539257251Skib return (false); 540257251Skib } 541257251Skib 542257251Skib if ((dmar->barrier_flags & f_inproc) != 0) { 543257251Skib while ((dmar->barrier_flags & f_inproc) != 0) { 544257251Skib dmar->barrier_flags |= f_wakeup; 545257251Skib msleep(&dmar->barrier_flags, &dmar->lock, 0, 546257251Skib "dmarb", 0); 547257251Skib } 548257251Skib KASSERT((dmar->barrier_flags & f_done) != 0, 549257251Skib ("dmar%d barrier %d missing done", dmar->unit, barrier_id)); 550257251Skib DMAR_UNLOCK(dmar); 551257251Skib return (false); 552257251Skib } 553257251Skib 554257251Skib dmar->barrier_flags |= f_inproc; 555257251Skib DMAR_UNLOCK(dmar); 556257251Skib return (true); 557257251Skib} 558257251Skib 559257251Skibvoid 560257251Skibdmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id) 561257251Skib{ 562257251Skib BARRIER_F; 563257251Skib 564257251Skib DMAR_ASSERT_LOCKED(dmar); 565257251Skib KASSERT((dmar->barrier_flags & (f_done | f_inproc)) == f_inproc, 566257251Skib ("dmar%d barrier %d missed entry", dmar->unit, barrier_id)); 567257251Skib dmar->barrier_flags |= f_done; 568257251Skib if ((dmar->barrier_flags & f_wakeup) != 0) 569257251Skib wakeup(&dmar->barrier_flags); 570257251Skib dmar->barrier_flags &= ~(f_inproc | f_wakeup); 571257251Skib DMAR_UNLOCK(dmar); 572257251Skib} 573257251Skib 574257251Skibint dmar_match_verbose; 575257251Skib 576257251Skibstatic SYSCTL_NODE(_hw, OID_AUTO, dmar, CTLFLAG_RD, NULL, 577257251Skib ""); 578257251SkibSYSCTL_INT(_hw_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD | CTLFLAG_TUN, 579257251Skib &dmar_tbl_pagecnt, 0, 580257251Skib "Count of pages used for DMAR pagetables"); 581257251SkibSYSCTL_INT(_hw_dmar, OID_AUTO, match_verbose, CTLFLAG_RW | CTLFLAG_TUN, 582257251Skib &dmar_match_verbose, 0, 583257251Skib "Verbose matching of the PCI devices to DMAR paths"); 584257251Skib#ifdef INVARIANTS 585257251Skibint dmar_check_free; 586257251SkibSYSCTL_INT(_hw_dmar, OID_AUTO, check_free, CTLFLAG_RW | CTLFLAG_TUN, 587257251Skib &dmar_check_free, 0, 588257251Skib "Check the GPA RBtree for free_down and free_after validity"); 589257251Skib#endif 590257251Skib 591