1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34 35#define RB_AUGMENT(entry) dmar_gas_augment_entry(entry) 36 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/malloc.h> 40#include <sys/bus.h> 41#include <sys/interrupt.h> 42#include <sys/kernel.h> 43#include <sys/ktr.h> 44#include <sys/lock.h> 45#include <sys/proc.h> 46#include <sys/rwlock.h> 47#include <sys/memdesc.h> 48#include <sys/mutex.h> 49#include <sys/sysctl.h> 50#include <sys/rman.h> 51#include <sys/taskqueue.h> 52#include <sys/tree.h> 53#include <sys/uio.h> 54#include <sys/vmem.h> 55#include <dev/pci/pcivar.h> 56#include <vm/vm.h> 57#include <vm/vm_extern.h> 58#include <vm/vm_kern.h> 59#include <vm/vm_object.h> 60#include <vm/vm_page.h> 61#include <vm/vm_map.h> 62#include <vm/uma.h> 63#include <machine/atomic.h> 64#include <machine/bus.h> 65#include <machine/md_var.h> 66#include <machine/specialreg.h> 67#include <x86/include/busdma_impl.h> 68#include <x86/iommu/intel_reg.h> 69#include <x86/iommu/busdma_dmar.h> 70#include <dev/pci/pcireg.h> 71#include <x86/iommu/intel_dmar.h> 72 73/* 74 * Guest Address Space management. 75 */ 76 77static uma_zone_t dmar_map_entry_zone; 78 79static void 80intel_gas_init(void) 81{ 82 83 dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", 84 sizeof(struct dmar_map_entry), NULL, NULL, 85 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); 86} 87SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); 88 89struct dmar_map_entry * 90dmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) 91{ 92 struct dmar_map_entry *res; 93 94 KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, 95 ("unsupported flags %x", flags)); 96 97 res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != 98 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); 99 if (res != NULL) { 100 res->domain = domain; 101 atomic_add_int(&domain->entries_cnt, 1); 102 } 103 return (res); 104} 105 106void 107dmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) 108{ 109 110 KASSERT(domain == entry->domain, 111 ("mismatched free domain %p entry %p entry->domain %p", domain, 112 entry, entry->domain)); 113 atomic_subtract_int(&domain->entries_cnt, 1); 114 uma_zfree(dmar_map_entry_zone, entry); 115} 116 117static int 118dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) 119{ 120 121 /* Last entry have zero size, so <= */ 122 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", 123 a, (uintmax_t)a->start, (uintmax_t)a->end)); 124 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", 125 b, (uintmax_t)b->start, (uintmax_t)b->end)); 126 KASSERT(a->end <= b->start || b->end <= a->start || 127 a->end == a->start || b->end == b->start, 128 ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", 129 a, (uintmax_t)a->start, (uintmax_t)a->end, 130 b, (uintmax_t)b->start, (uintmax_t)b->end)); 131 132 if (a->end < b->end) 133 return (-1); 134 else if (b->end < a->end) 135 return (1); 136 return (0); 137} 138 139static void 140dmar_gas_augment_entry(struct dmar_map_entry *entry) 141{ 142 struct dmar_map_entry *l, *r; 143 144 for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) { 145 l = RB_LEFT(entry, rb_entry); 146 r = RB_RIGHT(entry, rb_entry); 147 if (l == NULL && r == NULL) { 148 entry->free_down = entry->free_after; 149 } else if (l == NULL && r != NULL) { 150 entry->free_down = MAX(entry->free_after, r->free_down); 151 } else if (/*l != NULL && */ r == NULL) { 152 entry->free_down = MAX(entry->free_after, l->free_down); 153 } else /* if (l != NULL && r != NULL) */ { 154 entry->free_down = MAX(entry->free_after, l->free_down); 155 entry->free_down = MAX(entry->free_down, r->free_down); 156 } 157 } 158} 159 160RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, 161 dmar_gas_cmp_entries); 162 163static void 164dmar_gas_fix_free(struct dmar_domain *domain, struct dmar_map_entry *entry) 165{ 166 struct dmar_map_entry *next; 167 168 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 169 entry->free_after = (next != NULL ? next->start : domain->end) - 170 entry->end; 171 dmar_gas_augment_entry(entry); 172} 173 174#ifdef INVARIANTS 175static void 176dmar_gas_check_free(struct dmar_domain *domain) 177{ 178 struct dmar_map_entry *entry, *next, *l, *r; 179 dmar_gaddr_t v; 180 181 RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { 182 KASSERT(domain == entry->domain, 183 ("mismatched free domain %p entry %p entry->domain %p", 184 domain, entry, entry->domain)); 185 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 186 if (next == NULL) { 187 MPASS(entry->free_after == domain->end - entry->end); 188 } else { 189 MPASS(entry->free_after = next->start - entry->end); 190 MPASS(entry->end <= next->start); 191 } 192 l = RB_LEFT(entry, rb_entry); 193 r = RB_RIGHT(entry, rb_entry); 194 if (l == NULL && r == NULL) { 195 MPASS(entry->free_down == entry->free_after); 196 } else if (l == NULL && r != NULL) { 197 MPASS(entry->free_down = MAX(entry->free_after, 198 r->free_down)); 199 } else if (r == NULL) { 200 MPASS(entry->free_down = MAX(entry->free_after, 201 l->free_down)); 202 } else { 203 v = MAX(entry->free_after, l->free_down); 204 v = MAX(v, r->free_down); 205 MPASS(entry->free_down == v); 206 } 207 } 208} 209#endif 210 211static bool 212dmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) 213{ 214 struct dmar_map_entry *prev, *found; 215 216 found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); 217 dmar_gas_fix_free(domain, entry); 218 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 219 if (prev != NULL) 220 dmar_gas_fix_free(domain, prev); 221 return (found == NULL); 222} 223 224static void 225dmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) 226{ 227 struct dmar_map_entry *prev; 228 229 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 230 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 231 if (prev != NULL) 232 dmar_gas_fix_free(domain, prev); 233} 234 235void 236dmar_gas_init_domain(struct dmar_domain *domain) 237{ 238 struct dmar_map_entry *begin, *end; 239 240 begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 241 end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 242 243 DMAR_DOMAIN_LOCK(domain); 244 KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); 245 KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); 246 247 begin->start = 0; 248 begin->end = DMAR_PAGE_SIZE; 249 begin->free_after = domain->end - begin->end; 250 begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 251 dmar_gas_rb_insert(domain, begin); 252 253 end->start = domain->end; 254 end->end = domain->end; 255 end->free_after = 0; 256 end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; 257 dmar_gas_rb_insert(domain, end); 258 259 domain->first_place = begin; 260 domain->last_place = end; 261 domain->flags |= DMAR_DOMAIN_GAS_INITED; 262 DMAR_DOMAIN_UNLOCK(domain); 263} 264 265void 266dmar_gas_fini_domain(struct dmar_domain *domain) 267{ 268 struct dmar_map_entry *entry, *entry1; 269 270 DMAR_DOMAIN_ASSERT_LOCKED(domain); 271 KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); 272 273 entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); 274 KASSERT(entry->start == 0, ("start entry start %p", domain)); 275 KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); 276 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 277 ("start entry flags %p", domain)); 278 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 279 dmar_gas_free_entry(domain, entry); 280 281 entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); 282 KASSERT(entry->start == domain->end, ("end entry start %p", domain)); 283 KASSERT(entry->end == domain->end, ("end entry end %p", domain)); 284 KASSERT(entry->free_after == 0, ("end entry free_after %p", domain)); 285 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, 286 ("end entry flags %p", domain)); 287 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 288 dmar_gas_free_entry(domain, entry); 289 290 RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, 291 entry1) { 292 KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, 293 ("non-RMRR entry left %p", domain)); 294 RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); 295 dmar_gas_free_entry(domain, entry); 296 } 297} 298 299struct dmar_gas_match_args { 300 struct dmar_domain *domain; 301 dmar_gaddr_t size; 302 int offset; 303 const struct bus_dma_tag_common *common; 304 u_int gas_flags; 305 struct dmar_map_entry *entry; 306}; 307 308static bool 309dmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev, 310 dmar_gaddr_t end) 311{ 312 dmar_gaddr_t bs, start; 313 314 if (a->entry->start + a->size > end) 315 return (false); 316 317 /* DMAR_PAGE_SIZE to create gap after new entry. */ 318 if (a->entry->start < prev->end + DMAR_PAGE_SIZE || 319 a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > 320 prev->end + prev->free_after) 321 return (false); 322 323 /* No boundary crossing. */ 324 if (dmar_test_boundary(a->entry->start + a->offset, a->size, 325 a->common->boundary)) 326 return (true); 327 328 /* 329 * The start + offset to start + offset + size region crosses 330 * the boundary. Check if there is enough space after the 331 * next boundary after the prev->end. 332 */ 333 bs = rounddown2(a->entry->start + a->offset + a->common->boundary, 334 a->common->boundary); 335 start = roundup2(bs, a->common->alignment); 336 /* DMAR_PAGE_SIZE to create gap after new entry. */ 337 if (start + a->offset + a->size + DMAR_PAGE_SIZE <= 338 prev->end + prev->free_after && 339 start + a->offset + a->size <= end && 340 dmar_test_boundary(start + a->offset, a->size, 341 a->common->boundary)) { 342 a->entry->start = start; 343 return (true); 344 } 345 346 /* 347 * Not enough space to align at the requested boundary, or 348 * boundary is smaller than the size, but allowed to split. 349 * We already checked that start + size does not overlap end. 350 * 351 * XXXKIB. It is possible that bs is exactly at the start of 352 * the next entry, then we do not have gap. Ignore for now. 353 */ 354 if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { 355 a->size = bs - a->entry->start; 356 return (true); 357 } 358 359 return (false); 360} 361 362static void 363dmar_gas_match_insert(struct dmar_gas_match_args *a, 364 struct dmar_map_entry *prev) 365{ 366 struct dmar_map_entry *next; 367 bool found; 368 369 /* 370 * The prev->end is always aligned on the page size, which 371 * causes page alignment for the entry->start too. The size 372 * is checked to be multiple of the page size. 373 * 374 * The page sized gap is created between consequent 375 * allocations to ensure that out-of-bounds accesses fault. 376 */ 377 a->entry->end = a->entry->start + a->size; 378 379 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, prev); 380 KASSERT(next->start >= a->entry->end && 381 next->start - a->entry->start >= a->size && 382 prev->end <= a->entry->end, 383 ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) " 384 "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->domain, 385 (uintmax_t)prev->start, (uintmax_t)prev->end, 386 (uintmax_t)prev->free_after, 387 (uintmax_t)next->start, (uintmax_t)next->end, 388 (uintmax_t)a->entry->start, (uintmax_t)a->entry->end)); 389 390 prev->free_after = a->entry->start - prev->end; 391 a->entry->free_after = next->start - a->entry->end; 392 393 found = dmar_gas_rb_insert(a->domain, a->entry); 394 KASSERT(found, ("found dup %p start %jx size %jx", 395 a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); 396 a->entry->flags = DMAR_MAP_ENTRY_MAP; 397 398 KASSERT(RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, 399 a->entry) == prev, 400 ("entry %p prev %p inserted prev %p", a->entry, prev, 401 RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 402 KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 403 a->entry) == next, 404 ("entry %p next %p inserted next %p", a->entry, next, 405 RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, a->entry))); 406} 407 408static int 409dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev) 410{ 411 struct dmar_map_entry *l; 412 int ret; 413 414 if (prev->end < a->common->lowaddr) { 415 a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE, 416 a->common->alignment); 417 if (dmar_gas_match_one(a, prev, a->common->lowaddr)) { 418 dmar_gas_match_insert(a, prev); 419 return (0); 420 } 421 } 422 if (prev->free_down < a->size + a->offset + DMAR_PAGE_SIZE) 423 return (ENOMEM); 424 l = RB_LEFT(prev, rb_entry); 425 if (l != NULL) { 426 ret = dmar_gas_lowermatch(a, l); 427 if (ret == 0) 428 return (0); 429 } 430 l = RB_RIGHT(prev, rb_entry); 431 if (l != NULL) 432 return (dmar_gas_lowermatch(a, l)); 433 return (ENOMEM); 434} 435 436static int 437dmar_gas_uppermatch(struct dmar_gas_match_args *a) 438{ 439 struct dmar_map_entry *next, *prev, find_entry; 440 441 find_entry.start = a->common->highaddr; 442 next = RB_NFIND(dmar_gas_entries_tree, &a->domain->rb_root, 443 &find_entry); 444 if (next == NULL) 445 return (ENOMEM); 446 prev = RB_PREV(dmar_gas_entries_tree, &a->domain->rb_root, next); 447 KASSERT(prev != NULL, ("no prev %p %jx", a->domain, 448 (uintmax_t)find_entry.start)); 449 for (;;) { 450 a->entry->start = prev->start + DMAR_PAGE_SIZE; 451 if (a->entry->start < a->common->highaddr) 452 a->entry->start = a->common->highaddr; 453 a->entry->start = roundup2(a->entry->start, 454 a->common->alignment); 455 if (dmar_gas_match_one(a, prev, a->domain->end)) { 456 dmar_gas_match_insert(a, prev); 457 return (0); 458 } 459 460 /* 461 * XXXKIB. This falls back to linear iteration over 462 * the free space in the high region. But high 463 * regions are almost unused, the code should be 464 * enough to cover the case, although in the 465 * non-optimal way. 466 */ 467 prev = next; 468 next = RB_NEXT(dmar_gas_entries_tree, &a->domain->rb_root, 469 prev); 470 KASSERT(next != NULL, ("no next %p %jx", a->domain, 471 (uintmax_t)find_entry.start)); 472 if (next->end >= a->domain->end) 473 return (ENOMEM); 474 } 475} 476 477static int 478dmar_gas_find_space(struct dmar_domain *domain, 479 const struct bus_dma_tag_common *common, dmar_gaddr_t size, 480 int offset, u_int flags, struct dmar_map_entry *entry) 481{ 482 struct dmar_gas_match_args a; 483 int error; 484 485 DMAR_DOMAIN_ASSERT_LOCKED(domain); 486 KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); 487 KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); 488 489 a.domain = domain; 490 a.size = size; 491 a.offset = offset; 492 a.common = common; 493 a.gas_flags = flags; 494 a.entry = entry; 495 496 /* Handle lower region. */ 497 if (common->lowaddr > 0) { 498 error = dmar_gas_lowermatch(&a, RB_ROOT(&domain->rb_root)); 499 if (error == 0) 500 return (0); 501 KASSERT(error == ENOMEM, 502 ("error %d from dmar_gas_lowermatch", error)); 503 } 504 /* Handle upper region. */ 505 if (common->highaddr >= domain->end) 506 return (ENOMEM); 507 error = dmar_gas_uppermatch(&a); 508 KASSERT(error == ENOMEM, 509 ("error %d from dmar_gas_uppermatch", error)); 510 return (error); 511} 512 513static int 514dmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 515 u_int flags) 516{ 517 struct dmar_map_entry *next, *prev; 518 bool found; 519 520 DMAR_DOMAIN_ASSERT_LOCKED(domain); 521 522 if ((entry->start & DMAR_PAGE_MASK) != 0 || 523 (entry->end & DMAR_PAGE_MASK) != 0) 524 return (EINVAL); 525 if (entry->start >= entry->end) 526 return (EINVAL); 527 if (entry->end >= domain->end) 528 return (EINVAL); 529 530 next = RB_NFIND(dmar_gas_entries_tree, &domain->rb_root, entry); 531 KASSERT(next != NULL, ("next must be non-null %p %jx", domain, 532 (uintmax_t)entry->start)); 533 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, next); 534 /* prev could be NULL */ 535 536 /* 537 * Adapt to broken BIOSes which specify overlapping RMRR 538 * entries. 539 * 540 * XXXKIB: this does not handle a case when prev or next 541 * entries are completely covered by the current one, which 542 * extends both ways. 543 */ 544 if (prev != NULL && prev->end > entry->start && 545 (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 546 if ((flags & DMAR_GM_RMRR) == 0 || 547 (prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) 548 return (EBUSY); 549 entry->start = prev->end; 550 } 551 if (next != NULL && next->start < entry->end && 552 (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { 553 if ((flags & DMAR_GM_RMRR) == 0 || 554 (next->flags & DMAR_MAP_ENTRY_RMRR) == 0) 555 return (EBUSY); 556 entry->end = next->start; 557 } 558 if (entry->end == entry->start) 559 return (0); 560 561 if (prev != NULL && prev->end > entry->start) { 562 /* This assumes that prev is the placeholder entry. */ 563 dmar_gas_rb_remove(domain, prev); 564 prev = NULL; 565 } 566 if (next != NULL && next->start < entry->end) { 567 dmar_gas_rb_remove(domain, next); 568 next = NULL; 569 } 570 571 found = dmar_gas_rb_insert(domain, entry); 572 KASSERT(found, ("found RMRR dup %p start %jx end %jx", 573 domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); 574 if ((flags & DMAR_GM_RMRR) != 0) 575 entry->flags = DMAR_MAP_ENTRY_RMRR; 576 577#ifdef INVARIANTS 578 struct dmar_map_entry *ip, *in; 579 ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 580 in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 581 KASSERT(prev == NULL || ip == prev, 582 ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", 583 entry, entry->start, entry->end, prev, 584 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, 585 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); 586 KASSERT(next == NULL || in == next, 587 ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", 588 entry, entry->start, entry->end, next, 589 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, 590 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); 591#endif 592 593 return (0); 594} 595 596void 597dmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) 598{ 599 600 DMAR_DOMAIN_ASSERT_LOCKED(domain); 601 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 602 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, 603 ("permanent entry %p %p", domain, entry)); 604 605 dmar_gas_rb_remove(domain, entry); 606 entry->flags &= ~DMAR_MAP_ENTRY_MAP; 607#ifdef INVARIANTS 608 if (dmar_check_free) 609 dmar_gas_check_free(domain); 610#endif 611} 612 613void 614dmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) 615{ 616 struct dmar_map_entry *next, *prev; 617 618 DMAR_DOMAIN_ASSERT_LOCKED(domain); 619 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | 620 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, 621 ("non-RMRR entry %p %p", domain, entry)); 622 623 prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); 624 next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); 625 dmar_gas_rb_remove(domain, entry); 626 entry->flags &= ~DMAR_MAP_ENTRY_RMRR; 627 628 if (prev == NULL) 629 dmar_gas_rb_insert(domain, domain->first_place); 630 if (next == NULL) 631 dmar_gas_rb_insert(domain, domain->last_place); 632} 633 634int 635dmar_gas_map(struct dmar_domain *domain, 636 const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, 637 u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) 638{ 639 struct dmar_map_entry *entry; 640 int error; 641 642 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, 643 ("invalid flags 0x%x", flags)); 644 645 entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? 646 DMAR_PGF_WAITOK : 0); 647 if (entry == NULL) 648 return (ENOMEM); 649 DMAR_DOMAIN_LOCK(domain); 650 error = dmar_gas_find_space(domain, common, size, offset, flags, 651 entry); 652 if (error == ENOMEM) { 653 DMAR_DOMAIN_UNLOCK(domain); 654 dmar_gas_free_entry(domain, entry); 655 return (error); 656 } 657#ifdef INVARIANTS 658 if (dmar_check_free) 659 dmar_gas_check_free(domain); 660#endif 661 KASSERT(error == 0, 662 ("unexpected error %d from dmar_gas_find_entry", error)); 663 KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", 664 (uintmax_t)entry->end, (uintmax_t)domain->end)); 665 entry->flags |= eflags; 666 DMAR_DOMAIN_UNLOCK(domain); 667 668 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 669 ma, 670 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 671 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 672 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 673 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 674 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 675 if (error == ENOMEM) { 676 dmar_domain_unload_entry(entry, true); 677 return (error); 678 } 679 KASSERT(error == 0, 680 ("unexpected error %d from domain_map_buf", error)); 681 682 *res = entry; 683 return (0); 684} 685 686int 687dmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, 688 u_int eflags, u_int flags, vm_page_t *ma) 689{ 690 dmar_gaddr_t start; 691 int error; 692 693 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, 694 entry, entry->flags)); 695 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_RMRR)) == 0, 696 ("invalid flags 0x%x", flags)); 697 698 start = entry->start; 699 DMAR_DOMAIN_LOCK(domain); 700 error = dmar_gas_alloc_region(domain, entry, flags); 701 if (error != 0) { 702 DMAR_DOMAIN_UNLOCK(domain); 703 return (error); 704 } 705 entry->flags |= eflags; 706 DMAR_DOMAIN_UNLOCK(domain); 707 if (entry->end == entry->start) 708 return (0); 709 710 error = domain_map_buf(domain, entry->start, entry->end - entry->start, 711 ma + OFF_TO_IDX(start - entry->start), 712 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | 713 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | 714 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | 715 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), 716 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); 717 if (error == ENOMEM) { 718 dmar_domain_unload_entry(entry, false); 719 return (error); 720 } 721 KASSERT(error == 0, 722 ("unexpected error %d from domain_map_buf", error)); 723 724 return (0); 725} 726 727int 728dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, 729 dmar_gaddr_t end) 730{ 731 struct dmar_map_entry *entry; 732 int error; 733 734 entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); 735 entry->start = start; 736 entry->end = end; 737 DMAR_DOMAIN_LOCK(domain); 738 error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); 739 if (error == 0) 740 entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; 741 DMAR_DOMAIN_UNLOCK(domain); 742 if (error != 0) 743 dmar_gas_free_entry(domain, entry); 744 return (error); 745} 746