1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD$"); 32257251Skib 33257251Skib#include <sys/param.h> 34257251Skib#include <sys/systm.h> 35257251Skib#include <sys/malloc.h> 36257251Skib#include <sys/bus.h> 37257251Skib#include <sys/interrupt.h> 38257251Skib#include <sys/kernel.h> 39257251Skib#include <sys/ktr.h> 40257251Skib#include <sys/limits.h> 41257251Skib#include <sys/lock.h> 42257251Skib#include <sys/memdesc.h> 43257251Skib#include <sys/mutex.h> 44257251Skib#include <sys/proc.h> 45257251Skib#include <sys/rwlock.h> 46257251Skib#include <sys/rman.h> 47257251Skib#include <sys/sysctl.h> 48257251Skib#include <sys/taskqueue.h> 49257251Skib#include <sys/tree.h> 50257251Skib#include <sys/uio.h> 51257251Skib#include <vm/vm.h> 52257251Skib#include <vm/vm_extern.h> 53257251Skib#include <vm/vm_kern.h> 54257251Skib#include <vm/vm_object.h> 55257251Skib#include <vm/vm_page.h> 56257251Skib#include <vm/vm_pager.h> 57257251Skib#include <vm/vm_map.h> 58257251Skib#include <machine/atomic.h> 59257251Skib#include <machine/bus.h> 60257251Skib#include <machine/md_var.h> 61257251Skib#include <machine/specialreg.h> 62257251Skib#include <x86/include/busdma_impl.h> 63257251Skib#include <x86/iommu/intel_reg.h> 64257251Skib#include <x86/iommu/busdma_dmar.h> 65257251Skib#include <x86/iommu/intel_dmar.h> 66279470Srstone#include <dev/pci/pcireg.h> 67257251Skib#include <dev/pci/pcivar.h> 68257251Skib 69257251Skibstatic MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 70257251Skib 71257251Skibstatic void dmar_ctx_unload_task(void *arg, int pending); 72257251Skib 73257251Skibstatic void 74257251Skibdmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 75257251Skib{ 76257251Skib struct sf_buf *sf; 77257251Skib dmar_root_entry_t *re; 78257251Skib vm_page_t ctxm; 79257251Skib 80257251Skib /* 81257251Skib * Allocated context page must be linked. 82257251Skib */ 83257251Skib ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC); 84257251Skib if (ctxm != NULL) 85257251Skib return; 86257251Skib 87257251Skib /* 88257251Skib * Page not present, allocate and link. Note that other 89257251Skib * thread might execute this sequence in parallel. This 90257251Skib * should be safe, because the context entries written by both 91257251Skib * threads are equal. 92257251Skib */ 93257251Skib TD_PREP_PINNED_ASSERT; 94257251Skib ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO | 95257251Skib DMAR_PGF_WAITOK); 96257251Skib re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf); 97257251Skib re += bus; 98257251Skib dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 99257251Skib VM_PAGE_TO_PHYS(ctxm))); 100277315Skib dmar_flush_root_to_ram(dmar, re); 101277315Skib dmar_unmap_pgtbl(sf); 102257251Skib TD_PINNED_ASSERT; 103257251Skib} 104257251Skib 105257251Skibstatic dmar_ctx_entry_t * 106257251Skibdmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 107257251Skib{ 108257251Skib dmar_ctx_entry_t *ctxp; 109257251Skib 110279470Srstone ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->rid), 111257251Skib DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); 112279470Srstone ctxp += ctx->rid & 0xff; 113257251Skib return (ctxp); 114257251Skib} 115257251Skib 116257251Skibstatic void 117279470Srstonectx_tag_init(struct dmar_ctx *ctx, device_t dev) 118257251Skib{ 119257251Skib bus_addr_t maxaddr; 120257251Skib 121257251Skib maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR); 122257251Skib ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ 123257251Skib ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; 124257251Skib ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY; 125257251Skib ctx->ctx_tag.common.lowaddr = maxaddr; 126257251Skib ctx->ctx_tag.common.highaddr = maxaddr; 127257251Skib ctx->ctx_tag.common.maxsize = maxaddr; 128257251Skib ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; 129257251Skib ctx->ctx_tag.common.maxsegsz = maxaddr; 130257251Skib ctx->ctx_tag.ctx = ctx; 131279470Srstone ctx->ctx_tag.owner = dev; 132257251Skib /* XXXKIB initialize tag further */ 133257251Skib} 134257251Skib 135257251Skibstatic void 136257251Skibctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp) 137257251Skib{ 138257251Skib struct dmar_unit *unit; 139257251Skib vm_page_t ctx_root; 140257251Skib 141257251Skib unit = ctx->dmar; 142257251Skib KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0, 143257251Skib ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 144279470Srstone unit->unit, pci_get_bus(ctx->ctx_tag.owner), 145279470Srstone pci_get_slot(ctx->ctx_tag.owner), 146279470Srstone pci_get_function(ctx->ctx_tag.owner), 147279470Srstone ctxp->ctx1, 148257251Skib ctxp->ctx2)); 149257251Skib ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain); 150257251Skib ctxp->ctx2 |= ctx->awlvl; 151257251Skib if ((ctx->flags & DMAR_CTX_IDMAP) != 0 && 152257251Skib (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 153257251Skib KASSERT(ctx->pgtbl_obj == NULL, 154257251Skib ("ctx %p non-null pgtbl_obj", ctx)); 155257251Skib dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 156257251Skib } else { 157257251Skib ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC); 158257251Skib dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 159257251Skib (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 160257251Skib DMAR_CTX1_P); 161257251Skib } 162277315Skib dmar_flush_ctx_to_ram(unit, ctxp); 163257251Skib} 164257251Skib 165257251Skibstatic int 166257251Skibctx_init_rmrr(struct dmar_ctx *ctx, device_t dev) 167257251Skib{ 168257251Skib struct dmar_map_entries_tailq rmrr_entries; 169257251Skib struct dmar_map_entry *entry, *entry1; 170257251Skib vm_page_t *ma; 171257251Skib dmar_gaddr_t start, end; 172257251Skib vm_pindex_t size, i; 173257251Skib int error, error1; 174257251Skib 175257251Skib error = 0; 176257251Skib TAILQ_INIT(&rmrr_entries); 177257251Skib dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries); 178257251Skib TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 179257251Skib /* 180257251Skib * VT-d specification requires that the start of an 181257251Skib * RMRR entry is 4k-aligned. Buggy BIOSes put 182257251Skib * anything into the start and end fields. Truncate 183257251Skib * and round as neccesary. 184257251Skib * 185257251Skib * We also allow the overlapping RMRR entries, see 186257251Skib * dmar_gas_alloc_region(). 187257251Skib */ 188257251Skib start = entry->start; 189257251Skib end = entry->end; 190257251Skib entry->start = trunc_page(start); 191257251Skib entry->end = round_page(end); 192263746Skib if (entry->start == entry->end) { 193263746Skib /* Workaround for some AMI (?) BIOSes */ 194263746Skib if (bootverbose) { 195263746Skib device_printf(dev, "BIOS bug: dmar%d RMRR " 196263746Skib "region (%jx, %jx) corrected\n", 197263746Skib ctx->dmar->unit, start, end); 198263746Skib } 199263746Skib entry->end += DMAR_PAGE_SIZE * 0x20; 200263746Skib } 201257251Skib size = OFF_TO_IDX(entry->end - entry->start); 202257251Skib ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 203257251Skib for (i = 0; i < size; i++) { 204257251Skib ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 205257251Skib VM_MEMATTR_DEFAULT); 206257251Skib } 207257251Skib error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ | 208257251Skib DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma); 209257251Skib /* 210257251Skib * Non-failed RMRR entries are owned by context rb 211257251Skib * tree. Get rid of the failed entry, but do not stop 212257251Skib * the loop. Rest of the parsed RMRR entries are 213257251Skib * loaded and removed on the context destruction. 214257251Skib */ 215257251Skib if (error1 == 0 && entry->end != entry->start) { 216257251Skib DMAR_LOCK(ctx->dmar); 217257251Skib ctx->flags |= DMAR_CTX_RMRR; 218257251Skib DMAR_UNLOCK(ctx->dmar); 219257251Skib } else { 220257251Skib if (error1 != 0) { 221257251Skib device_printf(dev, 222257251Skib "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 223257251Skib ctx->dmar->unit, start, end, error1); 224257251Skib error = error1; 225257251Skib } 226257251Skib TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 227257251Skib dmar_gas_free_entry(ctx, entry); 228257251Skib } 229257251Skib for (i = 0; i < size; i++) 230257251Skib vm_page_putfake(ma[i]); 231257251Skib free(ma, M_TEMP); 232257251Skib } 233257251Skib return (error); 234257251Skib} 235257251Skib 236257251Skibstatic struct dmar_ctx * 237279470Srstonedmar_get_ctx_alloc(struct dmar_unit *dmar, uint16_t rid) 238257251Skib{ 239257251Skib struct dmar_ctx *ctx; 240257251Skib 241257251Skib ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 242257251Skib RB_INIT(&ctx->rb_root); 243257251Skib TAILQ_INIT(&ctx->unload_entries); 244257251Skib TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx); 245257251Skib mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF); 246257251Skib ctx->dmar = dmar; 247279470Srstone ctx->rid = rid; 248257251Skib return (ctx); 249257251Skib} 250257251Skib 251257251Skibstatic void 252257251Skibdmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited) 253257251Skib{ 254257251Skib 255257251Skib if (gas_inited) { 256257251Skib DMAR_CTX_LOCK(ctx); 257257251Skib dmar_gas_fini_ctx(ctx); 258257251Skib DMAR_CTX_UNLOCK(ctx); 259257251Skib } 260257251Skib if (pgtbl_inited) { 261257251Skib if (ctx->pgtbl_obj != NULL) 262257251Skib DMAR_CTX_PGLOCK(ctx); 263257251Skib ctx_free_pgtbl(ctx); 264257251Skib } 265257251Skib mtx_destroy(&ctx->lock); 266257251Skib free(ctx, M_DMAR_CTX); 267257251Skib} 268257251Skib 269257251Skibstruct dmar_ctx * 270279470Srstonedmar_get_ctx(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, 271279470Srstone bool rmrr_init) 272257251Skib{ 273257251Skib struct dmar_ctx *ctx, *ctx1; 274257251Skib dmar_ctx_entry_t *ctxp; 275257251Skib struct sf_buf *sf; 276279470Srstone int bus, slot, func, error, mgaw; 277257251Skib bool enable; 278257251Skib 279279470Srstone bus = pci_get_bus(dev); 280279470Srstone slot = pci_get_slot(dev); 281279470Srstone func = pci_get_function(dev); 282257251Skib enable = false; 283257251Skib TD_PREP_PINNED_ASSERT; 284257251Skib DMAR_LOCK(dmar); 285279470Srstone ctx = dmar_find_ctx_locked(dmar, rid); 286257251Skib error = 0; 287257251Skib if (ctx == NULL) { 288257251Skib /* 289257251Skib * Perform the allocations which require sleep or have 290257251Skib * higher chance to succeed if the sleep is allowed. 291257251Skib */ 292257251Skib DMAR_UNLOCK(dmar); 293279484Skib dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 294279470Srstone ctx1 = dmar_get_ctx_alloc(dmar, rid); 295257251Skib 296257251Skib if (id_mapped) { 297257251Skib /* 298257251Skib * For now, use the maximal usable physical 299257251Skib * address of the installed memory to 300257251Skib * calculate the mgaw. It is useful for the 301257251Skib * identity mapping, and less so for the 302257251Skib * virtualized bus address space. 303257251Skib */ 304257251Skib ctx1->end = ptoa(Maxmem); 305257251Skib mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false); 306257251Skib error = ctx_set_agaw(ctx1, mgaw); 307257251Skib if (error != 0) { 308257251Skib dmar_ctx_dtr(ctx1, false, false); 309257251Skib TD_PINNED_ASSERT; 310257251Skib return (NULL); 311257251Skib } 312257251Skib } else { 313257251Skib ctx1->end = BUS_SPACE_MAXADDR; 314257251Skib mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true); 315257251Skib error = ctx_set_agaw(ctx1, mgaw); 316257251Skib if (error != 0) { 317257251Skib dmar_ctx_dtr(ctx1, false, false); 318257251Skib TD_PINNED_ASSERT; 319257251Skib return (NULL); 320257251Skib } 321257251Skib /* Use all supported address space for remapping. */ 322257251Skib ctx1->end = 1ULL << (ctx1->agaw - 1); 323257251Skib } 324257251Skib 325257251Skib 326257251Skib dmar_gas_init_ctx(ctx1); 327257251Skib if (id_mapped) { 328257251Skib if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 329257251Skib ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1, 330257251Skib ctx1->end); 331257251Skib } 332257251Skib ctx1->flags |= DMAR_CTX_IDMAP; 333257251Skib } else { 334257251Skib error = ctx_alloc_pgtbl(ctx1); 335257251Skib if (error != 0) { 336257251Skib dmar_ctx_dtr(ctx1, true, false); 337257251Skib TD_PINNED_ASSERT; 338257251Skib return (NULL); 339257251Skib } 340257251Skib /* Disable local apic region access */ 341257251Skib error = dmar_gas_reserve_region(ctx1, 0xfee00000, 342257251Skib 0xfeefffff + 1); 343257251Skib if (error != 0) { 344257251Skib dmar_ctx_dtr(ctx1, true, true); 345257251Skib TD_PINNED_ASSERT; 346257251Skib return (NULL); 347257251Skib } 348257251Skib error = ctx_init_rmrr(ctx1, dev); 349257251Skib if (error != 0) { 350257251Skib dmar_ctx_dtr(ctx1, true, true); 351257251Skib TD_PINNED_ASSERT; 352257251Skib return (NULL); 353257251Skib } 354257251Skib } 355257251Skib ctxp = dmar_map_ctx_entry(ctx1, &sf); 356257251Skib DMAR_LOCK(dmar); 357257251Skib 358257251Skib /* 359257251Skib * Recheck the contexts, other thread might have 360257251Skib * already allocated needed one. 361257251Skib */ 362279470Srstone ctx = dmar_find_ctx_locked(dmar, rid); 363257251Skib if (ctx == NULL) { 364257251Skib ctx = ctx1; 365263747Skib ctx->ctx_tag.owner = dev; 366257251Skib ctx->domain = alloc_unrl(dmar->domids); 367257251Skib if (ctx->domain == -1) { 368257251Skib DMAR_UNLOCK(dmar); 369277315Skib dmar_unmap_pgtbl(sf); 370257251Skib dmar_ctx_dtr(ctx, true, true); 371257251Skib TD_PINNED_ASSERT; 372257251Skib return (NULL); 373257251Skib } 374279470Srstone ctx_tag_init(ctx, dev); 375257251Skib 376257251Skib /* 377257251Skib * This is the first activated context for the 378257251Skib * DMAR unit. Enable the translation after 379257251Skib * everything is set up. 380257251Skib */ 381257251Skib if (LIST_EMPTY(&dmar->contexts)) 382257251Skib enable = true; 383257251Skib LIST_INSERT_HEAD(&dmar->contexts, ctx, link); 384257251Skib ctx_id_entry_init(ctx, ctxp); 385257251Skib device_printf(dev, 386279485Skib "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " 387263747Skib "agaw %d %s-mapped\n", 388257251Skib dmar->unit, dmar->segment, bus, slot, 389279485Skib func, rid, ctx->domain, ctx->mgaw, ctx->agaw, 390263747Skib id_mapped ? "id" : "re"); 391257251Skib } else { 392257251Skib dmar_ctx_dtr(ctx1, true, true); 393257251Skib } 394277315Skib dmar_unmap_pgtbl(sf); 395257251Skib } 396257251Skib ctx->refs++; 397257251Skib if ((ctx->flags & DMAR_CTX_RMRR) != 0) 398257251Skib ctx->refs++; /* XXXKIB */ 399257251Skib 400257251Skib /* 401257251Skib * If dmar declares Caching Mode as Set, follow 11.5 "Caching 402257251Skib * Mode Consideration" and do the (global) invalidation of the 403257251Skib * negative TLB entries. 404257251Skib */ 405257251Skib if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) { 406259512Skib if (dmar->qi_enabled) { 407259512Skib dmar_qi_invalidate_ctx_glob_locked(dmar); 408259512Skib if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) 409259512Skib dmar_qi_invalidate_iotlb_glob_locked(dmar); 410259512Skib } else { 411259512Skib error = dmar_inv_ctx_glob(dmar); 412259512Skib if (error == 0 && 413259512Skib (dmar->hw_ecap & DMAR_ECAP_DI) != 0) 414259512Skib error = dmar_inv_iotlb_glob(dmar); 415259512Skib if (error != 0) { 416259512Skib dmar_free_ctx_locked(dmar, ctx); 417259512Skib TD_PINNED_ASSERT; 418259512Skib return (NULL); 419259512Skib } 420257251Skib } 421257251Skib } 422259512Skib 423259512Skib /* 424259512Skib * The dmar lock was potentially dropped between check for the 425259512Skib * empty context list and now. Recheck the state of GCMD_TE 426259512Skib * to avoid unneeded command. 427259512Skib */ 428259512Skib if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 429257251Skib error = dmar_enable_translation(dmar); 430257251Skib if (error != 0) { 431257251Skib dmar_free_ctx_locked(dmar, ctx); 432257251Skib TD_PINNED_ASSERT; 433257251Skib return (NULL); 434257251Skib } 435257251Skib } 436257251Skib DMAR_UNLOCK(dmar); 437257251Skib TD_PINNED_ASSERT; 438257251Skib return (ctx); 439257251Skib} 440257251Skib 441257251Skibvoid 442257251Skibdmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 443257251Skib{ 444257251Skib struct sf_buf *sf; 445257251Skib dmar_ctx_entry_t *ctxp; 446257251Skib 447257251Skib DMAR_ASSERT_LOCKED(dmar); 448257251Skib KASSERT(ctx->refs >= 1, 449257251Skib ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 450257251Skib 451257251Skib /* 452257251Skib * If our reference is not last, only the dereference should 453257251Skib * be performed. 454257251Skib */ 455257251Skib if (ctx->refs > 1) { 456257251Skib ctx->refs--; 457257251Skib DMAR_UNLOCK(dmar); 458257251Skib return; 459257251Skib } 460257251Skib 461257251Skib KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 462257251Skib ("lost ref on RMRR ctx %p", ctx)); 463257251Skib KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 464257251Skib ("lost ref on disabled ctx %p", ctx)); 465257251Skib 466257251Skib /* 467257251Skib * Otherwise, the context entry must be cleared before the 468257251Skib * page table is destroyed. The mapping of the context 469257251Skib * entries page could require sleep, unlock the dmar. 470257251Skib */ 471257251Skib DMAR_UNLOCK(dmar); 472257251Skib TD_PREP_PINNED_ASSERT; 473257251Skib ctxp = dmar_map_ctx_entry(ctx, &sf); 474257251Skib DMAR_LOCK(dmar); 475257251Skib KASSERT(ctx->refs >= 1, 476257251Skib ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 477257251Skib 478257251Skib /* 479257251Skib * Other thread might have referenced the context, in which 480257251Skib * case again only the dereference should be performed. 481257251Skib */ 482257251Skib if (ctx->refs > 1) { 483257251Skib ctx->refs--; 484257251Skib DMAR_UNLOCK(dmar); 485277315Skib dmar_unmap_pgtbl(sf); 486257251Skib TD_PINNED_ASSERT; 487257251Skib return; 488257251Skib } 489257251Skib 490257251Skib KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 491257251Skib ("lost ref on RMRR ctx %p", ctx)); 492257251Skib KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 493257251Skib ("lost ref on disabled ctx %p", ctx)); 494257251Skib 495257251Skib /* 496257251Skib * Clear the context pointer and flush the caches. 497257251Skib * XXXKIB: cannot do this if any RMRR entries are still present. 498257251Skib */ 499257251Skib dmar_pte_clear(&ctxp->ctx1); 500257251Skib ctxp->ctx2 = 0; 501277315Skib dmar_flush_ctx_to_ram(dmar, ctxp); 502257251Skib dmar_inv_ctx_glob(dmar); 503259512Skib if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 504259512Skib if (dmar->qi_enabled) 505259512Skib dmar_qi_invalidate_iotlb_glob_locked(dmar); 506259512Skib else 507259512Skib dmar_inv_iotlb_glob(dmar); 508259512Skib } 509257251Skib LIST_REMOVE(ctx, link); 510257251Skib DMAR_UNLOCK(dmar); 511257251Skib 512257251Skib /* 513257251Skib * The rest of the destruction is invisible for other users of 514257251Skib * the dmar unit. 515257251Skib */ 516257251Skib taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task); 517257251Skib KASSERT(TAILQ_EMPTY(&ctx->unload_entries), 518257251Skib ("unfinished unloads %p", ctx)); 519277315Skib dmar_unmap_pgtbl(sf); 520257251Skib free_unr(dmar->domids, ctx->domain); 521257251Skib dmar_ctx_dtr(ctx, true, true); 522257251Skib TD_PINNED_ASSERT; 523257251Skib} 524257251Skib 525257251Skibvoid 526257251Skibdmar_free_ctx(struct dmar_ctx *ctx) 527257251Skib{ 528257251Skib struct dmar_unit *dmar; 529257251Skib 530257251Skib dmar = ctx->dmar; 531257251Skib DMAR_LOCK(dmar); 532257251Skib dmar_free_ctx_locked(dmar, ctx); 533257251Skib} 534257251Skib 535257251Skibstruct dmar_ctx * 536279470Srstonedmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 537257251Skib{ 538257251Skib struct dmar_ctx *ctx; 539257251Skib 540257251Skib DMAR_ASSERT_LOCKED(dmar); 541257251Skib 542257251Skib LIST_FOREACH(ctx, &dmar->contexts, link) { 543279470Srstone if (ctx->rid == rid) 544257251Skib return (ctx); 545257251Skib } 546257251Skib return (NULL); 547257251Skib} 548257251Skib 549257251Skibvoid 550259512Skibdmar_ctx_free_entry(struct dmar_map_entry *entry, bool free) 551259512Skib{ 552259512Skib struct dmar_ctx *ctx; 553259512Skib 554259512Skib ctx = entry->ctx; 555259512Skib DMAR_CTX_LOCK(ctx); 556259512Skib if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) 557259512Skib dmar_gas_free_region(ctx, entry); 558259512Skib else 559259512Skib dmar_gas_free_space(ctx, entry); 560259512Skib DMAR_CTX_UNLOCK(ctx); 561259512Skib if (free) 562259512Skib dmar_gas_free_entry(ctx, entry); 563259512Skib else 564259512Skib entry->flags = 0; 565259512Skib} 566259512Skib 567259512Skibvoid 568259512Skibdmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free) 569259512Skib{ 570259512Skib struct dmar_unit *unit; 571259512Skib 572259512Skib unit = entry->ctx->dmar; 573259512Skib if (unit->qi_enabled) { 574259512Skib DMAR_LOCK(unit); 575259512Skib dmar_qi_invalidate_locked(entry->ctx, entry->start, 576259512Skib entry->end - entry->start, &entry->gseq); 577259512Skib if (!free) 578259512Skib entry->flags |= DMAR_MAP_ENTRY_QI_NF; 579259512Skib TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 580259512Skib DMAR_UNLOCK(unit); 581259512Skib } else { 582259512Skib ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end - 583259512Skib entry->start); 584259512Skib dmar_ctx_free_entry(entry, free); 585259512Skib } 586259512Skib} 587259512Skib 588259512Skibvoid 589257251Skibdmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries, 590257251Skib bool cansleep) 591257251Skib{ 592259512Skib struct dmar_unit *unit; 593259512Skib struct dmar_map_entry *entry, *entry1; 594259512Skib struct dmar_qi_genseq gseq; 595257251Skib int error; 596257251Skib 597259512Skib unit = ctx->dmar; 598259512Skib 599259512Skib TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 600257251Skib KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, 601257251Skib ("not mapped entry %p %p", ctx, entry)); 602257251Skib error = ctx_unmap_buf(ctx, entry->start, entry->end - 603257251Skib entry->start, cansleep ? DMAR_PGF_WAITOK : 0); 604257251Skib KASSERT(error == 0, ("unmap %p error %d", ctx, error)); 605259512Skib if (!unit->qi_enabled) { 606259512Skib ctx_flush_iotlb_sync(ctx, entry->start, 607259512Skib entry->end - entry->start); 608259512Skib TAILQ_REMOVE(entries, entry, dmamap_link); 609259512Skib dmar_ctx_free_entry(entry, true); 610259512Skib } 611257251Skib } 612259512Skib if (TAILQ_EMPTY(entries)) 613259512Skib return; 614259512Skib 615259512Skib KASSERT(unit->qi_enabled, ("loaded entry left")); 616259512Skib DMAR_LOCK(unit); 617259512Skib TAILQ_FOREACH(entry, entries, dmamap_link) { 618259512Skib entry->gseq.gen = 0; 619259512Skib entry->gseq.seq = 0; 620259512Skib dmar_qi_invalidate_locked(ctx, entry->start, entry->end - 621259512Skib entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ? 622259512Skib &gseq : NULL); 623259512Skib } 624259512Skib TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 625259512Skib entry->gseq = gseq; 626259512Skib TAILQ_REMOVE(entries, entry, dmamap_link); 627259512Skib TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 628259512Skib } 629259512Skib DMAR_UNLOCK(unit); 630257251Skib} 631257251Skib 632257251Skibstatic void 633257251Skibdmar_ctx_unload_task(void *arg, int pending) 634257251Skib{ 635257251Skib struct dmar_ctx *ctx; 636257251Skib struct dmar_map_entries_tailq entries; 637257251Skib 638257251Skib ctx = arg; 639257251Skib TAILQ_INIT(&entries); 640257251Skib 641257251Skib for (;;) { 642257251Skib DMAR_CTX_LOCK(ctx); 643257251Skib TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry, 644257251Skib dmamap_link); 645257251Skib DMAR_CTX_UNLOCK(ctx); 646257251Skib if (TAILQ_EMPTY(&entries)) 647257251Skib break; 648257251Skib dmar_ctx_unload(ctx, &entries, true); 649257251Skib } 650257251Skib} 651