1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD$"); 32257251Skib 33257251Skib#include <sys/param.h> 34257251Skib#include <sys/systm.h> 35257251Skib#include <sys/malloc.h> 36257251Skib#include <sys/bus.h> 37257251Skib#include <sys/interrupt.h> 38257251Skib#include <sys/kernel.h> 39257251Skib#include <sys/ktr.h> 40257251Skib#include <sys/limits.h> 41257251Skib#include <sys/lock.h> 42257251Skib#include <sys/memdesc.h> 43257251Skib#include <sys/mutex.h> 44257251Skib#include <sys/proc.h> 45257251Skib#include <sys/rwlock.h> 46257251Skib#include <sys/rman.h> 47257251Skib#include <sys/sysctl.h> 48257251Skib#include <sys/taskqueue.h> 49257251Skib#include <sys/tree.h> 50257251Skib#include <sys/uio.h> 51257251Skib#include <vm/vm.h> 52257251Skib#include <vm/vm_extern.h> 53257251Skib#include <vm/vm_kern.h> 54257251Skib#include <vm/vm_object.h> 55257251Skib#include <vm/vm_page.h> 56257251Skib#include <vm/vm_pager.h> 57257251Skib#include <vm/vm_map.h> 58257251Skib#include <machine/atomic.h> 59257251Skib#include <machine/bus.h> 60257251Skib#include <machine/md_var.h> 61257251Skib#include <machine/specialreg.h> 62257251Skib#include <x86/include/busdma_impl.h> 63257251Skib#include <x86/iommu/intel_reg.h> 64257251Skib#include <x86/iommu/busdma_dmar.h> 65257251Skib#include <x86/iommu/intel_dmar.h> 66257251Skib#include <dev/pci/pcivar.h> 67257251Skib 68257251Skibstatic MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 69257251Skib 70257251Skibstatic void dmar_ctx_unload_task(void *arg, int pending); 71257251Skib 72257251Skibstatic void 73257251Skibdmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 74257251Skib{ 75257251Skib struct sf_buf *sf; 76257251Skib dmar_root_entry_t *re; 77257251Skib vm_page_t ctxm; 78257251Skib 79257251Skib /* 80257251Skib * Allocated context page must be linked. 81257251Skib */ 82257251Skib ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC); 83257251Skib if (ctxm != NULL) 84257251Skib return; 85257251Skib 86257251Skib /* 87257251Skib * Page not present, allocate and link. Note that other 88257251Skib * thread might execute this sequence in parallel. This 89257251Skib * should be safe, because the context entries written by both 90257251Skib * threads are equal. 91257251Skib */ 92257251Skib TD_PREP_PINNED_ASSERT; 93257251Skib ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO | 94257251Skib DMAR_PGF_WAITOK); 95257251Skib re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf); 96257251Skib re += bus; 97257251Skib dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 98257251Skib VM_PAGE_TO_PHYS(ctxm))); 99257251Skib dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 100257251Skib TD_PINNED_ASSERT; 101257251Skib} 102257251Skib 103257251Skibstatic dmar_ctx_entry_t * 104257251Skibdmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 105257251Skib{ 106257251Skib dmar_ctx_entry_t *ctxp; 107257251Skib 108257251Skib ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + ctx->bus, 109257251Skib DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); 110257251Skib ctxp += ((ctx->slot & 0x1f) << 3) + (ctx->func & 0x7); 111257251Skib return (ctxp); 112257251Skib} 113257251Skib 114257251Skibstatic void 115257251Skibctx_tag_init(struct dmar_ctx *ctx) 116257251Skib{ 117257251Skib bus_addr_t maxaddr; 118257251Skib 119257251Skib maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR); 120257251Skib ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ 121257251Skib ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; 122257251Skib ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY; 123257251Skib ctx->ctx_tag.common.lowaddr = maxaddr; 124257251Skib ctx->ctx_tag.common.highaddr = maxaddr; 125257251Skib ctx->ctx_tag.common.maxsize = maxaddr; 126257251Skib ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; 127257251Skib ctx->ctx_tag.common.maxsegsz = maxaddr; 128257251Skib ctx->ctx_tag.ctx = ctx; 129257251Skib /* XXXKIB initialize tag further */ 130257251Skib} 131257251Skib 132257251Skibstatic void 133257251Skibctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp) 134257251Skib{ 135257251Skib struct dmar_unit *unit; 136257251Skib vm_page_t ctx_root; 137257251Skib 138257251Skib unit = ctx->dmar; 139257251Skib KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0, 140257251Skib ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 141257251Skib unit->unit, ctx->bus, ctx->slot, ctx->func, ctxp->ctx1, 142257251Skib ctxp->ctx2)); 143257251Skib ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain); 144257251Skib ctxp->ctx2 |= ctx->awlvl; 145257251Skib if ((ctx->flags & DMAR_CTX_IDMAP) != 0 && 146257251Skib (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 147257251Skib KASSERT(ctx->pgtbl_obj == NULL, 148257251Skib ("ctx %p non-null pgtbl_obj", ctx)); 149257251Skib dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 150257251Skib } else { 151257251Skib ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC); 152257251Skib dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 153257251Skib (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 154257251Skib DMAR_CTX1_P); 155257251Skib } 156257251Skib} 157257251Skib 158257251Skibstatic int 159257251Skibctx_init_rmrr(struct dmar_ctx *ctx, device_t dev) 160257251Skib{ 161257251Skib struct dmar_map_entries_tailq rmrr_entries; 162257251Skib struct dmar_map_entry *entry, *entry1; 163257251Skib vm_page_t *ma; 164257251Skib dmar_gaddr_t start, end; 165257251Skib vm_pindex_t size, i; 166257251Skib int error, error1; 167257251Skib 168257251Skib error = 0; 169257251Skib TAILQ_INIT(&rmrr_entries); 170257251Skib dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries); 171257251Skib TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 172257251Skib /* 173257251Skib * VT-d specification requires that the start of an 174257251Skib * RMRR entry is 4k-aligned. Buggy BIOSes put 175257251Skib * anything into the start and end fields. Truncate 176257251Skib * and round as neccesary. 177257251Skib * 178257251Skib * We also allow the overlapping RMRR entries, see 179257251Skib * dmar_gas_alloc_region(). 180257251Skib */ 181257251Skib start = entry->start; 182257251Skib end = entry->end; 183257251Skib entry->start = trunc_page(start); 184257251Skib entry->end = round_page(end); 185263746Skib if (entry->start == entry->end) { 186263746Skib /* Workaround for some AMI (?) BIOSes */ 187263746Skib if (bootverbose) { 188263746Skib device_printf(dev, "BIOS bug: dmar%d RMRR " 189263746Skib "region (%jx, %jx) corrected\n", 190263746Skib ctx->dmar->unit, start, end); 191263746Skib } 192263746Skib entry->end += DMAR_PAGE_SIZE * 0x20; 193263746Skib } 194257251Skib size = OFF_TO_IDX(entry->end - entry->start); 195257251Skib ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 196257251Skib for (i = 0; i < size; i++) { 197257251Skib ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 198257251Skib VM_MEMATTR_DEFAULT); 199257251Skib } 200257251Skib error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ | 201257251Skib DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma); 202257251Skib /* 203257251Skib * Non-failed RMRR entries are owned by context rb 204257251Skib * tree. Get rid of the failed entry, but do not stop 205257251Skib * the loop. Rest of the parsed RMRR entries are 206257251Skib * loaded and removed on the context destruction. 207257251Skib */ 208257251Skib if (error1 == 0 && entry->end != entry->start) { 209257251Skib DMAR_LOCK(ctx->dmar); 210257251Skib ctx->flags |= DMAR_CTX_RMRR; 211257251Skib DMAR_UNLOCK(ctx->dmar); 212257251Skib } else { 213257251Skib if (error1 != 0) { 214257251Skib device_printf(dev, 215257251Skib "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 216257251Skib ctx->dmar->unit, start, end, error1); 217257251Skib error = error1; 218257251Skib } 219257251Skib TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 220257251Skib dmar_gas_free_entry(ctx, entry); 221257251Skib } 222257251Skib for (i = 0; i < size; i++) 223257251Skib vm_page_putfake(ma[i]); 224257251Skib free(ma, M_TEMP); 225257251Skib } 226257251Skib return (error); 227257251Skib} 228257251Skib 229257251Skibstatic struct dmar_ctx * 230257251Skibdmar_get_ctx_alloc(struct dmar_unit *dmar, int bus, int slot, int func) 231257251Skib{ 232257251Skib struct dmar_ctx *ctx; 233257251Skib 234257251Skib ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 235257251Skib RB_INIT(&ctx->rb_root); 236257251Skib TAILQ_INIT(&ctx->unload_entries); 237257251Skib TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx); 238257251Skib mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF); 239257251Skib ctx->dmar = dmar; 240257251Skib ctx->bus = bus; 241257251Skib ctx->slot = slot; 242257251Skib ctx->func = func; 243257251Skib return (ctx); 244257251Skib} 245257251Skib 246257251Skibstatic void 247257251Skibdmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited) 248257251Skib{ 249257251Skib 250257251Skib if (gas_inited) { 251257251Skib DMAR_CTX_LOCK(ctx); 252257251Skib dmar_gas_fini_ctx(ctx); 253257251Skib DMAR_CTX_UNLOCK(ctx); 254257251Skib } 255257251Skib if (pgtbl_inited) { 256257251Skib if (ctx->pgtbl_obj != NULL) 257257251Skib DMAR_CTX_PGLOCK(ctx); 258257251Skib ctx_free_pgtbl(ctx); 259257251Skib } 260257251Skib mtx_destroy(&ctx->lock); 261257251Skib free(ctx, M_DMAR_CTX); 262257251Skib} 263257251Skib 264257251Skibstruct dmar_ctx * 265263747Skibdmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func, 266263747Skib bool id_mapped, bool rmrr_init) 267257251Skib{ 268257251Skib struct dmar_ctx *ctx, *ctx1; 269257251Skib dmar_ctx_entry_t *ctxp; 270257251Skib struct sf_buf *sf; 271263747Skib int error, mgaw; 272257251Skib bool enable; 273257251Skib 274257251Skib enable = false; 275257251Skib TD_PREP_PINNED_ASSERT; 276257251Skib DMAR_LOCK(dmar); 277257251Skib ctx = dmar_find_ctx_locked(dmar, bus, slot, func); 278257251Skib error = 0; 279257251Skib if (ctx == NULL) { 280257251Skib /* 281257251Skib * Perform the allocations which require sleep or have 282257251Skib * higher chance to succeed if the sleep is allowed. 283257251Skib */ 284257251Skib DMAR_UNLOCK(dmar); 285257251Skib dmar_ensure_ctx_page(dmar, bus); 286257251Skib ctx1 = dmar_get_ctx_alloc(dmar, bus, slot, func); 287257251Skib 288257251Skib if (id_mapped) { 289257251Skib /* 290257251Skib * For now, use the maximal usable physical 291257251Skib * address of the installed memory to 292257251Skib * calculate the mgaw. It is useful for the 293257251Skib * identity mapping, and less so for the 294257251Skib * virtualized bus address space. 295257251Skib */ 296257251Skib ctx1->end = ptoa(Maxmem); 297257251Skib mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false); 298257251Skib error = ctx_set_agaw(ctx1, mgaw); 299257251Skib if (error != 0) { 300257251Skib dmar_ctx_dtr(ctx1, false, false); 301257251Skib TD_PINNED_ASSERT; 302257251Skib return (NULL); 303257251Skib } 304257251Skib } else { 305257251Skib ctx1->end = BUS_SPACE_MAXADDR; 306257251Skib mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true); 307257251Skib error = ctx_set_agaw(ctx1, mgaw); 308257251Skib if (error != 0) { 309257251Skib dmar_ctx_dtr(ctx1, false, false); 310257251Skib TD_PINNED_ASSERT; 311257251Skib return (NULL); 312257251Skib } 313257251Skib /* Use all supported address space for remapping. */ 314257251Skib ctx1->end = 1ULL << (ctx1->agaw - 1); 315257251Skib } 316257251Skib 317257251Skib 318257251Skib dmar_gas_init_ctx(ctx1); 319257251Skib if (id_mapped) { 320257251Skib if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 321257251Skib ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1, 322257251Skib ctx1->end); 323257251Skib } 324257251Skib ctx1->flags |= DMAR_CTX_IDMAP; 325257251Skib } else { 326257251Skib error = ctx_alloc_pgtbl(ctx1); 327257251Skib if (error != 0) { 328257251Skib dmar_ctx_dtr(ctx1, true, false); 329257251Skib TD_PINNED_ASSERT; 330257251Skib return (NULL); 331257251Skib } 332257251Skib /* Disable local apic region access */ 333257251Skib error = dmar_gas_reserve_region(ctx1, 0xfee00000, 334257251Skib 0xfeefffff + 1); 335257251Skib if (error != 0) { 336257251Skib dmar_ctx_dtr(ctx1, true, true); 337257251Skib TD_PINNED_ASSERT; 338257251Skib return (NULL); 339257251Skib } 340257251Skib error = ctx_init_rmrr(ctx1, dev); 341257251Skib if (error != 0) { 342257251Skib dmar_ctx_dtr(ctx1, true, true); 343257251Skib TD_PINNED_ASSERT; 344257251Skib return (NULL); 345257251Skib } 346257251Skib } 347257251Skib ctxp = dmar_map_ctx_entry(ctx1, &sf); 348257251Skib DMAR_LOCK(dmar); 349257251Skib 350257251Skib /* 351257251Skib * Recheck the contexts, other thread might have 352257251Skib * already allocated needed one. 353257251Skib */ 354257251Skib ctx = dmar_find_ctx_locked(dmar, bus, slot, func); 355257251Skib if (ctx == NULL) { 356257251Skib ctx = ctx1; 357263747Skib ctx->ctx_tag.owner = dev; 358257251Skib ctx->domain = alloc_unrl(dmar->domids); 359257251Skib if (ctx->domain == -1) { 360257251Skib DMAR_UNLOCK(dmar); 361257251Skib dmar_unmap_pgtbl(sf, true); 362257251Skib dmar_ctx_dtr(ctx, true, true); 363257251Skib TD_PINNED_ASSERT; 364257251Skib return (NULL); 365257251Skib } 366257251Skib ctx_tag_init(ctx); 367257251Skib 368257251Skib /* 369257251Skib * This is the first activated context for the 370257251Skib * DMAR unit. Enable the translation after 371257251Skib * everything is set up. 372257251Skib */ 373257251Skib if (LIST_EMPTY(&dmar->contexts)) 374257251Skib enable = true; 375257251Skib LIST_INSERT_HEAD(&dmar->contexts, ctx, link); 376257251Skib ctx_id_entry_init(ctx, ctxp); 377257251Skib device_printf(dev, 378263747Skib "dmar%d pci%d:%d:%d:%d domain %d mgaw %d " 379263747Skib "agaw %d %s-mapped\n", 380257251Skib dmar->unit, dmar->segment, bus, slot, 381263747Skib func, ctx->domain, ctx->mgaw, ctx->agaw, 382263747Skib id_mapped ? "id" : "re"); 383257251Skib } else { 384257251Skib dmar_ctx_dtr(ctx1, true, true); 385257251Skib } 386257251Skib dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 387257251Skib } 388257251Skib ctx->refs++; 389257251Skib if ((ctx->flags & DMAR_CTX_RMRR) != 0) 390257251Skib ctx->refs++; /* XXXKIB */ 391257251Skib 392257251Skib /* 393257251Skib * If dmar declares Caching Mode as Set, follow 11.5 "Caching 394257251Skib * Mode Consideration" and do the (global) invalidation of the 395257251Skib * negative TLB entries. 396257251Skib */ 397257251Skib if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) { 398259512Skib if (dmar->qi_enabled) { 399259512Skib dmar_qi_invalidate_ctx_glob_locked(dmar); 400259512Skib if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) 401259512Skib dmar_qi_invalidate_iotlb_glob_locked(dmar); 402259512Skib } else { 403259512Skib error = dmar_inv_ctx_glob(dmar); 404259512Skib if (error == 0 && 405259512Skib (dmar->hw_ecap & DMAR_ECAP_DI) != 0) 406259512Skib error = dmar_inv_iotlb_glob(dmar); 407259512Skib if (error != 0) { 408259512Skib dmar_free_ctx_locked(dmar, ctx); 409259512Skib TD_PINNED_ASSERT; 410259512Skib return (NULL); 411259512Skib } 412257251Skib } 413257251Skib } 414259512Skib 415259512Skib /* 416259512Skib * The dmar lock was potentially dropped between check for the 417259512Skib * empty context list and now. Recheck the state of GCMD_TE 418259512Skib * to avoid unneeded command. 419259512Skib */ 420259512Skib if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 421257251Skib error = dmar_enable_translation(dmar); 422257251Skib if (error != 0) { 423257251Skib dmar_free_ctx_locked(dmar, ctx); 424257251Skib TD_PINNED_ASSERT; 425257251Skib return (NULL); 426257251Skib } 427257251Skib } 428257251Skib DMAR_UNLOCK(dmar); 429257251Skib TD_PINNED_ASSERT; 430257251Skib return (ctx); 431257251Skib} 432257251Skib 433257251Skibvoid 434257251Skibdmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 435257251Skib{ 436257251Skib struct sf_buf *sf; 437257251Skib dmar_ctx_entry_t *ctxp; 438257251Skib 439257251Skib DMAR_ASSERT_LOCKED(dmar); 440257251Skib KASSERT(ctx->refs >= 1, 441257251Skib ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 442257251Skib 443257251Skib /* 444257251Skib * If our reference is not last, only the dereference should 445257251Skib * be performed. 446257251Skib */ 447257251Skib if (ctx->refs > 1) { 448257251Skib ctx->refs--; 449257251Skib DMAR_UNLOCK(dmar); 450257251Skib return; 451257251Skib } 452257251Skib 453257251Skib KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 454257251Skib ("lost ref on RMRR ctx %p", ctx)); 455257251Skib KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 456257251Skib ("lost ref on disabled ctx %p", ctx)); 457257251Skib 458257251Skib /* 459257251Skib * Otherwise, the context entry must be cleared before the 460257251Skib * page table is destroyed. The mapping of the context 461257251Skib * entries page could require sleep, unlock the dmar. 462257251Skib */ 463257251Skib DMAR_UNLOCK(dmar); 464257251Skib TD_PREP_PINNED_ASSERT; 465257251Skib ctxp = dmar_map_ctx_entry(ctx, &sf); 466257251Skib DMAR_LOCK(dmar); 467257251Skib KASSERT(ctx->refs >= 1, 468257251Skib ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 469257251Skib 470257251Skib /* 471257251Skib * Other thread might have referenced the context, in which 472257251Skib * case again only the dereference should be performed. 473257251Skib */ 474257251Skib if (ctx->refs > 1) { 475257251Skib ctx->refs--; 476257251Skib DMAR_UNLOCK(dmar); 477257251Skib dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 478257251Skib TD_PINNED_ASSERT; 479257251Skib return; 480257251Skib } 481257251Skib 482257251Skib KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 483257251Skib ("lost ref on RMRR ctx %p", ctx)); 484257251Skib KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 485257251Skib ("lost ref on disabled ctx %p", ctx)); 486257251Skib 487257251Skib /* 488257251Skib * Clear the context pointer and flush the caches. 489257251Skib * XXXKIB: cannot do this if any RMRR entries are still present. 490257251Skib */ 491257251Skib dmar_pte_clear(&ctxp->ctx1); 492257251Skib ctxp->ctx2 = 0; 493257251Skib dmar_inv_ctx_glob(dmar); 494259512Skib if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 495259512Skib if (dmar->qi_enabled) 496259512Skib dmar_qi_invalidate_iotlb_glob_locked(dmar); 497259512Skib else 498259512Skib dmar_inv_iotlb_glob(dmar); 499259512Skib } 500257251Skib LIST_REMOVE(ctx, link); 501257251Skib DMAR_UNLOCK(dmar); 502257251Skib 503257251Skib /* 504257251Skib * The rest of the destruction is invisible for other users of 505257251Skib * the dmar unit. 506257251Skib */ 507257251Skib taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task); 508257251Skib KASSERT(TAILQ_EMPTY(&ctx->unload_entries), 509257251Skib ("unfinished unloads %p", ctx)); 510257251Skib dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 511257251Skib free_unr(dmar->domids, ctx->domain); 512257251Skib dmar_ctx_dtr(ctx, true, true); 513257251Skib TD_PINNED_ASSERT; 514257251Skib} 515257251Skib 516257251Skibvoid 517257251Skibdmar_free_ctx(struct dmar_ctx *ctx) 518257251Skib{ 519257251Skib struct dmar_unit *dmar; 520257251Skib 521257251Skib dmar = ctx->dmar; 522257251Skib DMAR_LOCK(dmar); 523257251Skib dmar_free_ctx_locked(dmar, ctx); 524257251Skib} 525257251Skib 526257251Skibstruct dmar_ctx * 527257251Skibdmar_find_ctx_locked(struct dmar_unit *dmar, int bus, int slot, int func) 528257251Skib{ 529257251Skib struct dmar_ctx *ctx; 530257251Skib 531257251Skib DMAR_ASSERT_LOCKED(dmar); 532257251Skib 533257251Skib LIST_FOREACH(ctx, &dmar->contexts, link) { 534257251Skib if (ctx->bus == bus && ctx->slot == slot && ctx->func == func) 535257251Skib return (ctx); 536257251Skib } 537257251Skib return (NULL); 538257251Skib} 539257251Skib 540257251Skibvoid 541259512Skibdmar_ctx_free_entry(struct dmar_map_entry *entry, bool free) 542259512Skib{ 543259512Skib struct dmar_ctx *ctx; 544259512Skib 545259512Skib ctx = entry->ctx; 546259512Skib DMAR_CTX_LOCK(ctx); 547259512Skib if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) 548259512Skib dmar_gas_free_region(ctx, entry); 549259512Skib else 550259512Skib dmar_gas_free_space(ctx, entry); 551259512Skib DMAR_CTX_UNLOCK(ctx); 552259512Skib if (free) 553259512Skib dmar_gas_free_entry(ctx, entry); 554259512Skib else 555259512Skib entry->flags = 0; 556259512Skib} 557259512Skib 558259512Skibvoid 559259512Skibdmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free) 560259512Skib{ 561259512Skib struct dmar_unit *unit; 562259512Skib 563259512Skib unit = entry->ctx->dmar; 564259512Skib if (unit->qi_enabled) { 565259512Skib DMAR_LOCK(unit); 566259512Skib dmar_qi_invalidate_locked(entry->ctx, entry->start, 567259512Skib entry->end - entry->start, &entry->gseq); 568259512Skib if (!free) 569259512Skib entry->flags |= DMAR_MAP_ENTRY_QI_NF; 570259512Skib TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 571259512Skib DMAR_UNLOCK(unit); 572259512Skib } else { 573259512Skib ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end - 574259512Skib entry->start); 575259512Skib dmar_ctx_free_entry(entry, free); 576259512Skib } 577259512Skib} 578259512Skib 579259512Skibvoid 580257251Skibdmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries, 581257251Skib bool cansleep) 582257251Skib{ 583259512Skib struct dmar_unit *unit; 584259512Skib struct dmar_map_entry *entry, *entry1; 585259512Skib struct dmar_qi_genseq gseq; 586257251Skib int error; 587257251Skib 588259512Skib unit = ctx->dmar; 589259512Skib 590259512Skib TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 591257251Skib KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, 592257251Skib ("not mapped entry %p %p", ctx, entry)); 593257251Skib error = ctx_unmap_buf(ctx, entry->start, entry->end - 594257251Skib entry->start, cansleep ? DMAR_PGF_WAITOK : 0); 595257251Skib KASSERT(error == 0, ("unmap %p error %d", ctx, error)); 596259512Skib if (!unit->qi_enabled) { 597259512Skib ctx_flush_iotlb_sync(ctx, entry->start, 598259512Skib entry->end - entry->start); 599259512Skib TAILQ_REMOVE(entries, entry, dmamap_link); 600259512Skib dmar_ctx_free_entry(entry, true); 601259512Skib } 602257251Skib } 603259512Skib if (TAILQ_EMPTY(entries)) 604259512Skib return; 605259512Skib 606259512Skib KASSERT(unit->qi_enabled, ("loaded entry left")); 607259512Skib DMAR_LOCK(unit); 608259512Skib TAILQ_FOREACH(entry, entries, dmamap_link) { 609259512Skib entry->gseq.gen = 0; 610259512Skib entry->gseq.seq = 0; 611259512Skib dmar_qi_invalidate_locked(ctx, entry->start, entry->end - 612259512Skib entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ? 613259512Skib &gseq : NULL); 614259512Skib } 615259512Skib TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 616259512Skib entry->gseq = gseq; 617259512Skib TAILQ_REMOVE(entries, entry, dmamap_link); 618259512Skib TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 619259512Skib } 620259512Skib DMAR_UNLOCK(unit); 621257251Skib} 622257251Skib 623257251Skibstatic void 624257251Skibdmar_ctx_unload_task(void *arg, int pending) 625257251Skib{ 626257251Skib struct dmar_ctx *ctx; 627257251Skib struct dmar_map_entries_tailq entries; 628257251Skib 629257251Skib ctx = arg; 630257251Skib TAILQ_INIT(&entries); 631257251Skib 632257251Skib for (;;) { 633257251Skib DMAR_CTX_LOCK(ctx); 634257251Skib TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry, 635257251Skib dmamap_link); 636257251Skib DMAR_CTX_UNLOCK(ctx); 637257251Skib if (TAILQ_EMPTY(&entries)) 638257251Skib break; 639257251Skib dmar_ctx_unload(ctx, &entries, true); 640257251Skib } 641257251Skib} 642