1257251Skib/*- 2257251Skib * Copyright (c) 2013 The FreeBSD Foundation 3257251Skib * All rights reserved. 4257251Skib * 5257251Skib * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6257251Skib * under sponsorship from the FreeBSD Foundation. 7257251Skib * 8257251Skib * Redistribution and use in source and binary forms, with or without 9257251Skib * modification, are permitted provided that the following conditions 10257251Skib * are met: 11257251Skib * 1. Redistributions of source code must retain the above copyright 12257251Skib * notice, this list of conditions and the following disclaimer. 13257251Skib * 2. Redistributions in binary form must reproduce the above copyright 14257251Skib * notice, this list of conditions and the following disclaimer in the 15257251Skib * documentation and/or other materials provided with the distribution. 16257251Skib * 17257251Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18257251Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19257251Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20257251Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21257251Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22257251Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23257251Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24257251Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25257251Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26257251Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27257251Skib * SUCH DAMAGE. 28257251Skib */ 29257251Skib 30257251Skib#include <sys/cdefs.h> 31257251Skib__FBSDID("$FreeBSD: stable/11/sys/x86/iommu/intel_gas.c 329942 2018-02-25 00:32:42Z markj $"); 32257251Skib 33257251Skib#define RB_AUGMENT(entry) dmar_gas_augment_entry(entry) 34257251Skib 35257251Skib#include <sys/param.h> 36257251Skib#include <sys/systm.h> 37257251Skib#include <sys/malloc.h> 38257251Skib#include <sys/bus.h> 39257251Skib#include <sys/interrupt.h> 40257251Skib#include <sys/kernel.h> 41257251Skib#include <sys/ktr.h> 42257251Skib#include <sys/lock.h> 43257251Skib#include <sys/proc.h> 44257251Skib#include <sys/rwlock.h> 45257251Skib#include <sys/memdesc.h> 46257251Skib#include <sys/mutex.h> 47257251Skib#include <sys/sysctl.h> 48257251Skib#include <sys/rman.h> 49257251Skib#include <sys/taskqueue.h> 50257251Skib#include <sys/tree.h> 51257251Skib#include <sys/uio.h> 52280260Skib#include <sys/vmem.h> 53257251Skib#include <dev/pci/pcivar.h> 54257251Skib#include <vm/vm.h> 55257251Skib#include <vm/vm_extern.h> 56257251Skib#include <vm/vm_kern.h> 57257251Skib#include <vm/vm_object.h> 58257251Skib#include <vm/vm_page.h> 59257251Skib#include <vm/vm_map.h> 60257251Skib#include <vm/uma.h> 61257251Skib#include <machine/atomic.h> 62257251Skib#include <machine/bus.h> 63257251Skib#include <machine/md_var.h> 64257251Skib#include <machine/specialreg.h> 65257251Skib#include <x86/include/busdma_impl.h> 66257251Skib#include <x86/iommu/intel_reg.h> 67257251Skib#include <x86/iommu/busdma_dmar.h> 68257251Skib#include <x86/iommu/intel_dmar.h> 69257251Skib 70257251Skib/* 71257251Skib * Guest Address Space management. 72257251Skib */ 73257251Skib 74257251Skibstatic uma_zone_t dmar_map_entry_zone; 75257251Skib 76257251Skibstatic void 77257251Skibintel_gas_init(void) 78257251Skib{ 79257251Skib 80257251Skib dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", 81257251Skib sizeof(struct dmar_map_entry), NULL, NULL, 82329942Smarkj NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); 83257251Skib} 84257251SkibSYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); 85257251Skib 86257251Skibstruct dmar_map_entry * 87284869Skibdmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) 88257251Skib{ 89257251Skib struct dmar_map_entry *res; 90257251Skib 91257251Skib KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, 92257251Skib ("unsupported flags %x", flags)); 93257251Skib 94257251Skib res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != 95257251Skib 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); 96257512Skib if (res != NULL) { 97284869Skib res->domain = domain; 98284869Skib atomic_add_int(&domain->entries_cnt, 1); 99257512Skib } 100257251Skib return (res); 101257251Skib} 102257251Skib 103257251Skibvoid 104284869Skibdmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) 105257251Skib{ 106257251Skib 107284869Skib KASSERT(domain == entry->domain, 108284869Skib ("mismatched free domain %p entry %p entry->domain %p", domain, 109284869Skib entry, entry->domain)); 110284869Skib atomic_subtract_int(&domain->entries_cnt, 1); 111257251Skib uma_zfree(dmar_map_entry_zone, entry); 112257251Skib} 113257251Skib 114257251Skibstatic int 115257251Skibdmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) 116257251Skib{ 117257251Skib 118257251Skib /* Last entry have zero size, so <= */ 119257251Skib KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", 120257251Skib a, (uintmax_t)a->start, (uintmax_t)a->end)); 121257251Skib KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", 122257251Skib b, (uintmax_t)b->start, (uintmax_t)b->end)); 123257251Skib KASSERT(a->end <= b->start || b->end <= a->start || 124257251Skib a->end == a->start || b->end == b->start, 125257251Skib ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", 126257251Skib a, (uintmax_t)a->start, (uintmax_t)a->end, 127257251Skib b, (uintmax_t)b->start, (uintmax_t)b->end)); 128257251Skib 129257251Skib if (a->end < b->end) 130257251Skib return (-1); 131257251Skib else if (b->end < a->end) 132257251Skib return (1); 133257251Skib return (0); 134257251Skib} 135257251Skib 136257251Skibstatic void 137257251Skibdmar_gas_augment_entry(struct dmar_map_entry *entry) 138257251Skib{ 139257251Skib struct dmar_map_entry *l, *r; 140257251Skib 141257251Skib for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) { 142257251Skib l = RB_LEFT(entry, rb_entry); 143257251Skib r = RB_RIGHT(entry, rb_entry); 144257251Skib if (l == NULL && r == NULL) { 145257251Skib entry->free_down = entry->free_after; 146257251Skib } else if (l == NULL && r != NULL) { 147257251Skib entry->free_down = MAX(entry->free_after, r->free_down); 148257251Skib } else if (/*l != NULL && */ r == NULL) { 149257251Skib entry->free_down = MAX(entry->free_after, l->free_down); 150257251Skib } else /* if (l != NULL && r != NULL) */ { 151257251Skib entry->free_down = MAX(entry->free_after, l->free_down); 152257251Skib entry->free_down = MAX(entry->free_down, r->free_down); 153257251Skib } 154257251Skib } 155257251Skib} 156257251Skib 157257251SkibRB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, 158257251Skib dmar_gas_cmp_entries); 159257251Skib 160257251Skibstatic void 161284869Skibdmar_gas_fix_free(struct dmar_domain *domain, struct dmar_map_entry *entry) 162257251Skib{ 163257251Skib struct dmar_map_entry *next; 164257251Skib 165284869Skib next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 166284869Skib entry->free_after = (next != NULL ? next->start : domain->end) - 167257251Skib entry->end; 168257251Skib dmar_gas_augment_entry(entry); 169257251Skib} 170257251Skib 171257251Skib#ifdef INVARIANTS 172257251Skibstatic void 173284869Skibdmar_gas_check_free(struct dmar_domain *domain) 174257251Skib{ 175257251Skib struct dmar_map_entry *entry, *next, *l, *r; 176257251Skib dmar_gaddr_t v; 177257251Skib 178284869Skib RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { 179284869Skib KASSERT(domain == entry->domain, 180284869Skib ("mismatched free domain %p entry %p entry->domain %p", 181284869Skib domain, entry, entry->domain)); 182284869Skib next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 183257251Skib if (next == NULL) { 184284869Skib MPASS(entry->free_after == domain->end - entry->end); 185257251Skib } else { 186257251Skib MPASS(entry->free_after = next->start - entry->end); 187257251Skib MPASS(entry->end <= next->start); 188257251Skib } 189257251Skib l = RB_LEFT(entry, rb_entry); 190257251Skib r = RB_RIGHT(entry, rb_entry); 191257251Skib if (l == NULL && r == NULL) { 192257251Skib MPASS(entry->free_down == entry->free_after); 193257251Skib } else if (l == NULL && r != NULL) { 194257251Skib MPASS(entry->free_down = MAX(entry->free_after, 195257251Skib r->free_down)); 196257251Skib } else if (r == NULL) { 197257251Skib MPASS(entry->free_down = MAX(entry->free_after, 198257251Skib l->free_down)); 199257251Skib } else { 200257251Skib v = MAX(entry->free_after, l->free_down); 201317248Skib v = MAX(v, r->free_down); 202257251Skib MPASS(entry->free_down == v); 203257251Skib } 204257251Skib } 205257251Skib} 206257251Skib#endif 207257251Skib 208257251Skibstatic bool 209284869Skibdmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) 210257251Skib{ 211257251Skib struct dmar_map_entry *prev, *found; 212257251Skib 213284869Skib found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); 214284869Skib dmar_gas_fix_free(domain, entry); 215284869Skib prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 216257251Skib if (prev != NULL) 217284869Skib dmar_gas_fix_free(domain, prev); 218257251Skib return (found == NULL); 219257251Skib} 220257251Skib 221257251Skibstatic void 222284869Skibdmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) 223257251Skib{ 224257251Skib struct dmar_map_entry *prev; 225257251Skib 226284869Skib prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 227284869Skib RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 228257251Skib if (prev != NULL) 229284869Skib dmar_gas_fix_free(domain, prev); 230257251Skib} 231257251Skib 232257251Skibvoid 233284869Skibdmar_gas_init_domain(struct dmar_domain *domain) 234257251Skib{ 235257251Skib struct dmar_map_entry *begin, *end; 236257251Skib 237284869Skib begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 238284869Skib end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 239257251Skib 240284869Skib DMAR_DOMAIN_LOCK(domain); 241284869Skib KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); 242284869Skib KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); 243257251Skib 244257251Skib begin->start = 0; 245257251Skib begin->end = DMAR_PAGE_SIZE; 246284869Skib begin->free_after = domain->end - begin->end; 247257251Skib begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 248284869Skib dmar_gas_rb_insert(domain, begin); 249257251Skib 250284869Skib end->start = domain->end; 251284869Skib end->end = domain->end; 252257251Skib end->free_after = 0; 253257251Skib end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 254284869Skib dmar_gas_rb_insert(domain, end); 255257251Skib 256284869Skib domain->first_place = begin; 257284869Skib domain->last_place = end; 258284869Skib domain->flags |= DMAR_DOMAIN_GAS_INITED; 259284869Skib DMAR_DOMAIN_UNLOCK(domain); 260257251Skib} 261257251Skib 262257251Skibvoid 263284869Skibdmar_gas_fini_domain(struct dmar_domain *domain) 264257251Skib{ 265257251Skib struct dmar_map_entry *entry, *entry1; 266257251Skib 267284869Skib DMAR_DOMAIN_ASSERT_LOCKED(domain); 268284869Skib KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); 269257251Skib 270284869Skib entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); 271284869Skib KASSERT(entry->start == 0, ("start entry start %p", domain)); 272284869Skib KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); 273257251Skib KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 274284869Skib ("start entry flags %p", domain)); 275284869Skib RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 276284869Skib dmar_gas_free_entry(domain, entry); 277257251Skib 278284869Skib entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); 279284869Skib KASSERT(entry->start == domain->end, ("end entry start %p", domain)); 280284869Skib KASSERT(entry->end == domain->end, ("end entry end %p", domain)); 281284869Skib KASSERT(entry->free_after == 0, ("end entry free_after %p", domain)); 282257251Skib KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 283284869Skib ("end entry flags %p", domain)); 284284869Skib RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 285284869Skib dmar_gas_free_entry(domain, entry); 286257251Skib 287284869Skib RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, 288284869Skib entry1) { 289257251Skib KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, 290284869Skib ("non-RMRR entry left %p", domain)); 291284869Skib RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 292284869Skib dmar_gas_free_entry(domain, entry); 293257251Skib } 294257251Skib} 295257251Skib 296257251Skibstruct dmar_gas_match_args { 297284869Skib struct dmar_domain *domain; 298257251Skib dmar_gaddr_t size; 299281254Skib int offset; 300257251Skib const struct bus_dma_tag_common *common; 301257251Skib u_int gas_flags; 302257251Skib struct dmar_map_entry *entry; 303257251Skib}; 304257251Skib 305257251Skibstatic bool 306257251Skibdmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev, 307257251Skib dmar_gaddr_t end) 308257251Skib{ 309257251Skib dmar_gaddr_t bs, start; 310257251Skib 311257251Skib if (a->entry->start + a->size > end) 312257251Skib return (false); 313257251Skib 314257251Skib /* DMAR_PAGE_SIZE to create gap after new entry. */ 315257251Skib if (a->entry->start < prev->end + DMAR_PAGE_SIZE || 316281254Skib a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > 317281254Skib prev->end + prev->free_after) 318257251Skib return (false); 319257251Skib 320257251Skib /* No boundary crossing. */ 321281254Skib if (dmar_test_boundary(a->entry->start + a->offset, a->size, 322281254Skib a->common->boundary)) 323257251Skib return (true); 324257251Skib 325257251Skib /* 326281254Skib * The start + offset to start + offset + size region crosses 327281254Skib * the boundary. Check if there is enough space after the 328281254Skib * next boundary after the prev->end. 329257251Skib */ 330298433Spfg bs = rounddown2(a->entry->start + a->offset + a->common->boundary, 331298433Spfg a->common->boundary); 332257251Skib start = roundup2(bs, a->common->alignment); 333257251Skib /* DMAR_PAGE_SIZE to create gap after new entry. */ 334281254Skib if (start + a->offset + a->size + DMAR_PAGE_SIZE <= 335281254Skib prev->end + prev->free_after && 336281254Skib start + a->offset + a->size <= end && 337281254Skib dmar_test_boundary(start + a->offset, a->size, 338280196Skib a->common->boundary)) { 339257251Skib a->entry->start = start; 340257251Skib return (true); 341257251Skib } 342257251Skib 343257251Skib /* 344280196Skib * Not enough space to align at the requested boundary, or 345280196Skib * boundary is smaller than the size, but allowed to split. 346257251Skib * We already checked that start + size does not overlap end. 347257251Skib * 348257251Skib * XXXKIB. It is possible that bs is exactly at the start of 349257251Skib * the next entry, then we do not have gap. Ignore for now. 350257251Skib */ 351257251Skib if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { 352257251Skib a->size = bs - a->entry->start; 353257251Skib return (true); 354257251Skib } 355257251Skib 356257251Skib return (false); 357257251Skib} 358257251Skib 359257251Skibstatic void 360257251Skibdmar_gas_match_insert(struct dmar_gas_match_args *a, 361257251Skib struct dmar_map_entry *prev) 362257251Skib{ 363257251Skib struct dmar_map_entry *next; 364257251Skib bool found; 365257251Skib 366257251Skib /* 367257251Skib * The prev->end is always aligned on the page size, which 368257251Skib * causes page alignment for the entry->start too. The size 369257251Skib * is checked to be multiple of the page size. 370257251Skib * 371257251Skib * The page sized gap is created between consequent 372257251Skib * allocations to ensure that out-of-bounds accesses fault. 373257251Skib */ 374257251Skib a->entry->end = a->entry->start + a->size; 375257251Skib 376284869Skib next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, prev); 377257251Skib KASSERT(next->start >= a->entry->end && 378280195Skib next->start - a->entry->start >= a->size && 379280195Skib prev->end <= a->entry->end, 380257251Skib ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) " 381284869Skib "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->domain, 382257251Skib (uintmax_t)prev->start, (uintmax_t)prev->end, 383257251Skib (uintmax_t)prev->free_after, 384257251Skib (uintmax_t)next->start, (uintmax_t)next->end, 385257251Skib (uintmax_t)a->entry->start, (uintmax_t)a->entry->end)); 386257251Skib 387257251Skib prev->free_after = a->entry->start - prev->end; 388257251Skib a->entry->free_after = next->start - a->entry->end; 389257251Skib 390284869Skib found = dmar_gas_rb_insert(a->domain, a->entry); 391257251Skib KASSERT(found, ("found dup %p start %jx size %jx", 392284869Skib a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); 393257251Skib a->entry->flags = DMAR_MAP_ENTRY_MAP; 394257251Skib 395284869Skib KASSERT(RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, 396257251Skib a->entry) == prev, 397257251Skib ("entry %p prev %p inserted prev %p", a->entry, prev, 398284869Skib RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 399284869Skib KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 400257251Skib a->entry) == next, 401257251Skib ("entry %p next %p inserted next %p", a->entry, next, 402284869Skib RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 403257251Skib} 404257251Skib 405257251Skibstatic int 406257251Skibdmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev) 407257251Skib{ 408257251Skib struct dmar_map_entry *l; 409257251Skib int ret; 410257251Skib 411257251Skib if (prev->end < a->common->lowaddr) { 412257251Skib a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE, 413257251Skib a->common->alignment); 414257251Skib if (dmar_gas_match_one(a, prev, a->common->lowaddr)) { 415257251Skib dmar_gas_match_insert(a, prev); 416257251Skib return (0); 417257251Skib } 418257251Skib } 419281254Skib if (prev->free_down < a->size + a->offset + DMAR_PAGE_SIZE) 420257251Skib return (ENOMEM); 421257251Skib l = RB_LEFT(prev, rb_entry); 422257251Skib if (l != NULL) { 423257251Skib ret = dmar_gas_lowermatch(a, l); 424257251Skib if (ret == 0) 425257251Skib return (0); 426257251Skib } 427257251Skib l = RB_RIGHT(prev, rb_entry); 428257251Skib if (l != NULL) 429257251Skib return (dmar_gas_lowermatch(a, l)); 430257251Skib return (ENOMEM); 431257251Skib} 432257251Skib 433257251Skibstatic int 434257251Skibdmar_gas_uppermatch(struct dmar_gas_match_args *a) 435257251Skib{ 436257251Skib struct dmar_map_entry *next, *prev, find_entry; 437257251Skib 438257251Skib find_entry.start = a->common->highaddr; 439284869Skib next = RB_NFIND(dmar_gas_entries_tree, &a->domain->rb_root, 440284869Skib &find_entry); 441257251Skib if (next == NULL) 442257251Skib return (ENOMEM); 443284869Skib prev = RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, next); 444284869Skib KASSERT(prev != NULL, ("no prev %p %jx", a->domain, 445257251Skib (uintmax_t)find_entry.start)); 446257251Skib for (;;) { 447257251Skib a->entry->start = prev->start + DMAR_PAGE_SIZE; 448257251Skib if (a->entry->start < a->common->highaddr) 449257251Skib a->entry->start = a->common->highaddr; 450257251Skib a->entry->start = roundup2(a->entry->start, 451257251Skib a->common->alignment); 452284869Skib if (dmar_gas_match_one(a, prev, a->domain->end)) { 453257251Skib dmar_gas_match_insert(a, prev); 454257251Skib return (0); 455257251Skib } 456257251Skib 457257251Skib /* 458257251Skib * XXXKIB. This falls back to linear iteration over 459257251Skib * the free space in the high region. But high 460257251Skib * regions are almost unused, the code should be 461257251Skib * enough to cover the case, although in the 462257251Skib * non-optimal way. 463257251Skib */ 464257251Skib prev = next; 465284869Skib next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 466284869Skib prev); 467284869Skib KASSERT(next != NULL, ("no next %p %jx", a->domain, 468257251Skib (uintmax_t)find_entry.start)); 469284869Skib if (next->end >= a->domain->end) 470257251Skib return (ENOMEM); 471257251Skib } 472257251Skib} 473257251Skib 474257251Skibstatic int 475284869Skibdmar_gas_find_space(struct dmar_domain *domain, 476257251Skib const struct bus_dma_tag_common *common, dmar_gaddr_t size, 477281254Skib int offset, u_int flags, struct dmar_map_entry *entry) 478257251Skib{ 479257251Skib struct dmar_gas_match_args a; 480257251Skib int error; 481257251Skib 482284869Skib DMAR_DOMAIN_ASSERT_LOCKED(domain); 483284869Skib KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); 484257251Skib KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); 485257251Skib 486284869Skib a.domain = domain; 487257251Skib a.size = size; 488281254Skib a.offset = offset; 489257251Skib a.common = common; 490257251Skib a.gas_flags = flags; 491257251Skib a.entry = entry; 492257251Skib 493257251Skib /* Handle lower region. */ 494257251Skib if (common->lowaddr > 0) { 495284869Skib error = dmar_gas_lowermatch(&a, RB_ROOT(&domain->rb_root)); 496257251Skib if (error == 0) 497257251Skib return (0); 498257251Skib KASSERT(error == ENOMEM, 499257251Skib ("error %d from dmar_gas_lowermatch", error)); 500257251Skib } 501257251Skib /* Handle upper region. */ 502284869Skib if (common->highaddr >= domain->end) 503257251Skib return (ENOMEM); 504257251Skib error = dmar_gas_uppermatch(&a); 505257251Skib KASSERT(error == ENOMEM, 506257251Skib ("error %d from dmar_gas_uppermatch", error)); 507257251Skib return (error); 508257251Skib} 509257251Skib 510257251Skibstatic int 511284869Skibdmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 512257251Skib u_int flags) 513257251Skib{ 514257251Skib struct dmar_map_entry *next, *prev; 515257251Skib bool found; 516257251Skib 517284869Skib DMAR_DOMAIN_ASSERT_LOCKED(domain); 518257251Skib 519257251Skib if ((entry->start & DMAR_PAGE_MASK) != 0 || 520257251Skib (entry->end & DMAR_PAGE_MASK) != 0) 521257251Skib return (EINVAL); 522257251Skib if (entry->start >= entry->end) 523257251Skib return (EINVAL); 524284869Skib if (entry->end >= domain->end) 525257251Skib return (EINVAL); 526257251Skib 527284869Skib next = RB_NFIND(dmar_gas_entries_tree, &domain->rb_root, entry); 528284869Skib KASSERT(next != NULL, ("next must be non-null %p %jx", domain, 529257251Skib (uintmax_t)entry->start)); 530284869Skib prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, next); 531257251Skib /* prev could be NULL */ 532257251Skib 533257251Skib /* 534257251Skib * Adapt to broken BIOSes which specify overlapping RMRR 535257251Skib * entries. 536257251Skib * 537257251Skib * XXXKIB: this does not handle a case when prev or next 538257251Skib * entries are completely covered by the current one, which 539257251Skib * extends both ways. 540257251Skib */ 541257251Skib if (prev != NULL && prev->end > entry->start && 542257251Skib (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 543257251Skib if ((prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) 544257251Skib return (EBUSY); 545257251Skib entry->start = prev->end; 546257251Skib } 547257251Skib if (next != NULL && next->start < entry->end && 548257251Skib (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 549257251Skib if ((next->flags & DMAR_MAP_ENTRY_RMRR) == 0) 550257251Skib return (EBUSY); 551257251Skib entry->end = next->start; 552257251Skib } 553257251Skib if (entry->end == entry->start) 554257251Skib return (0); 555257251Skib 556257251Skib if (prev != NULL && prev->end > entry->start) { 557257251Skib /* This assumes that prev is the placeholder entry. */ 558284869Skib dmar_gas_rb_remove(domain, prev); 559257251Skib prev = NULL; 560257251Skib } 561257251Skib if (next != NULL && next->start < entry->end) { 562284869Skib dmar_gas_rb_remove(domain, next); 563257251Skib next = NULL; 564257251Skib } 565257251Skib 566284869Skib found = dmar_gas_rb_insert(domain, entry); 567257251Skib KASSERT(found, ("found RMRR dup %p start %jx end %jx", 568284869Skib domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); 569257251Skib entry->flags = DMAR_MAP_ENTRY_RMRR; 570257251Skib 571257251Skib#ifdef INVARIANTS 572257251Skib struct dmar_map_entry *ip, *in; 573284869Skib ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 574284869Skib in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 575257251Skib KASSERT(prev == NULL || ip == prev, 576257251Skib ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", 577257251Skib entry, entry->start, entry->end, prev, 578257251Skib prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, 579257251Skib ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); 580257251Skib KASSERT(next == NULL || in == next, 581257251Skib ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", 582257251Skib entry, entry->start, entry->end, next, 583257251Skib next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, 584257251Skib in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); 585257251Skib#endif 586257251Skib 587257251Skib return (0); 588257251Skib} 589257251Skib 590257251Skibvoid 591284869Skibdmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) 592257251Skib{ 593257251Skib 594284869Skib DMAR_DOMAIN_ASSERT_LOCKED(domain); 595257251Skib KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 596257251Skib DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, 597284869Skib ("permanent entry %p %p", domain, entry)); 598257251Skib 599284869Skib dmar_gas_rb_remove(domain, entry); 600257251Skib entry->flags &= ~DMAR_MAP_ENTRY_MAP; 601257251Skib#ifdef INVARIANTS 602257251Skib if (dmar_check_free) 603284869Skib dmar_gas_check_free(domain); 604257251Skib#endif 605257251Skib} 606257251Skib 607257512Skibvoid 608284869Skibdmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) 609257251Skib{ 610257251Skib struct dmar_map_entry *next, *prev; 611257251Skib 612284869Skib DMAR_DOMAIN_ASSERT_LOCKED(domain); 613257251Skib KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 614257251Skib DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, 615284869Skib ("non-RMRR entry %p %p", domain, entry)); 616257251Skib 617284869Skib prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 618284869Skib next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 619284869Skib dmar_gas_rb_remove(domain, entry); 620257251Skib entry->flags &= ~DMAR_MAP_ENTRY_RMRR; 621257251Skib 622257251Skib if (prev == NULL) 623284869Skib dmar_gas_rb_insert(domain, domain->first_place); 624257251Skib if (next == NULL) 625284869Skib dmar_gas_rb_insert(domain, domain->last_place); 626257251Skib} 627257251Skib 628257251Skibint 629284869Skibdmar_gas_map(struct dmar_domain *domain, 630284869Skib const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, 631284869Skib u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) 632257251Skib{ 633257251Skib struct dmar_map_entry *entry; 634257251Skib int error; 635257251Skib 636257251Skib KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, 637257251Skib ("invalid flags 0x%x", flags)); 638257251Skib 639284869Skib entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? 640257251Skib DMAR_PGF_WAITOK : 0); 641257251Skib if (entry == NULL) 642257251Skib return (ENOMEM); 643284869Skib DMAR_DOMAIN_LOCK(domain); 644284869Skib error = dmar_gas_find_space(domain, common, size, offset, flags, 645284869Skib entry); 646257251Skib if (error == ENOMEM) { 647284869Skib DMAR_DOMAIN_UNLOCK(domain); 648284869Skib dmar_gas_free_entry(domain, entry); 649257251Skib return (error); 650257251Skib } 651257251Skib#ifdef INVARIANTS 652257251Skib if (dmar_check_free) 653284869Skib dmar_gas_check_free(domain); 654257251Skib#endif 655257251Skib KASSERT(error == 0, 656257251Skib ("unexpected error %d from dmar_gas_find_entry", error)); 657284869Skib KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", 658284869Skib (uintmax_t)entry->end, (uintmax_t)domain->end)); 659257251Skib entry->flags |= eflags; 660284869Skib DMAR_DOMAIN_UNLOCK(domain); 661257251Skib 662284869Skib error = domain_map_buf(domain, entry->start, entry->end - entry->start, 663284869Skib ma, 664257251Skib ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 665257251Skib ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 666257251Skib ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 667257251Skib ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 668257251Skib (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 669257251Skib if (error == ENOMEM) { 670284869Skib dmar_domain_unload_entry(entry, true); 671257251Skib return (error); 672257251Skib } 673257251Skib KASSERT(error == 0, 674284869Skib ("unexpected error %d from domain_map_buf", error)); 675257251Skib 676257251Skib *res = entry; 677257251Skib return (0); 678257251Skib} 679257251Skib 680257251Skibint 681284869Skibdmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 682257251Skib u_int eflags, u_int flags, vm_page_t *ma) 683257251Skib{ 684257251Skib dmar_gaddr_t start; 685257251Skib int error; 686257251Skib 687284869Skib KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, 688257251Skib entry, entry->flags)); 689257251Skib KASSERT((flags & ~(DMAR_GM_CANWAIT)) == 0, 690257251Skib ("invalid flags 0x%x", flags)); 691257251Skib 692257251Skib start = entry->start; 693284869Skib DMAR_DOMAIN_LOCK(domain); 694284869Skib error = dmar_gas_alloc_region(domain, entry, flags); 695257251Skib if (error != 0) { 696284869Skib DMAR_DOMAIN_UNLOCK(domain); 697257251Skib return (error); 698257251Skib } 699257251Skib entry->flags |= eflags; 700284869Skib DMAR_DOMAIN_UNLOCK(domain); 701257251Skib if (entry->end == entry->start) 702257251Skib return (0); 703257251Skib 704284869Skib error = domain_map_buf(domain, entry->start, entry->end - entry->start, 705257251Skib ma + OFF_TO_IDX(start - entry->start), 706257251Skib ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 707257251Skib ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 708257251Skib ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 709257251Skib ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 710257251Skib (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 711257251Skib if (error == ENOMEM) { 712284869Skib dmar_domain_unload_entry(entry, false); 713257251Skib return (error); 714257251Skib } 715257251Skib KASSERT(error == 0, 716284869Skib ("unexpected error %d from domain_map_buf", error)); 717257251Skib 718257251Skib return (0); 719257251Skib} 720257251Skib 721257251Skibint 722284869Skibdmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, 723257251Skib dmar_gaddr_t end) 724257251Skib{ 725257251Skib struct dmar_map_entry *entry; 726257251Skib int error; 727257251Skib 728284869Skib entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 729257251Skib entry->start = start; 730257251Skib entry->end = end; 731284869Skib DMAR_DOMAIN_LOCK(domain); 732284869Skib error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); 733257251Skib if (error == 0) 734257251Skib entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; 735284869Skib DMAR_DOMAIN_UNLOCK(domain); 736257251Skib if (error != 0) 737284869Skib dmar_gas_free_entry(domain, entry); 738257251Skib return (error); 739257251Skib} 740