52#include <dev/pci/pcivar.h>
53#include <vm/vm.h>
54#include <vm/vm_extern.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_object.h>
57#include <vm/vm_page.h>
58#include <vm/vm_map.h>
59#include <vm/uma.h>
60#include <machine/atomic.h>
61#include <machine/bus.h>
62#include <machine/md_var.h>
63#include <machine/specialreg.h>
64#include <x86/include/busdma_impl.h>
65#include <x86/iommu/intel_reg.h>
66#include <x86/iommu/busdma_dmar.h>
67#include <x86/iommu/intel_dmar.h>
68
69/*
70 * Guest Address Space management.
71 */
72
73static uma_zone_t dmar_map_entry_zone;
74
75static void
76intel_gas_init(void)
77{
78
79 dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY",
80 sizeof(struct dmar_map_entry), NULL, NULL,
81 NULL, NULL, UMA_ALIGN_PTR, 0);
82}
83SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL);
84
85struct dmar_map_entry *
86dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags)
87{
88 struct dmar_map_entry *res;
89
90 KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0,
91 ("unsupported flags %x", flags));
92
93 res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) !=
94 0 ? M_WAITOK : M_NOWAIT) | M_ZERO);
95 if (res != NULL) {
96 res->ctx = ctx;
97 atomic_add_int(&ctx->entries_cnt, 1);
98 }
99 return (res);
100}
101
102void
103dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
104{
105
106 KASSERT(ctx == entry->ctx,
107 ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
108 entry, entry->ctx));
109 atomic_subtract_int(&ctx->entries_cnt, 1);
110 uma_zfree(dmar_map_entry_zone, entry);
111}
112
113static int
114dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b)
115{
116
117 /* Last entry have zero size, so <= */
118 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)",
119 a, (uintmax_t)a->start, (uintmax_t)a->end));
120 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)",
121 b, (uintmax_t)b->start, (uintmax_t)b->end));
122 KASSERT(a->end <= b->start || b->end <= a->start ||
123 a->end == a->start || b->end == b->start,
124 ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)",
125 a, (uintmax_t)a->start, (uintmax_t)a->end,
126 b, (uintmax_t)b->start, (uintmax_t)b->end));
127
128 if (a->end < b->end)
129 return (-1);
130 else if (b->end < a->end)
131 return (1);
132 return (0);
133}
134
135static void
136dmar_gas_augment_entry(struct dmar_map_entry *entry)
137{
138 struct dmar_map_entry *l, *r;
139
140 for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) {
141 l = RB_LEFT(entry, rb_entry);
142 r = RB_RIGHT(entry, rb_entry);
143 if (l == NULL && r == NULL) {
144 entry->free_down = entry->free_after;
145 } else if (l == NULL && r != NULL) {
146 entry->free_down = MAX(entry->free_after, r->free_down);
147 } else if (/*l != NULL && */ r == NULL) {
148 entry->free_down = MAX(entry->free_after, l->free_down);
149 } else /* if (l != NULL && r != NULL) */ {
150 entry->free_down = MAX(entry->free_after, l->free_down);
151 entry->free_down = MAX(entry->free_down, r->free_down);
152 }
153 }
154}
155
156RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
157 dmar_gas_cmp_entries);
158
159static void
160dmar_gas_fix_free(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
161{
162 struct dmar_map_entry *next;
163
164 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
165 entry->free_after = (next != NULL ? next->start : ctx->end) -
166 entry->end;
167 dmar_gas_augment_entry(entry);
168}
169
170#ifdef INVARIANTS
171static void
172dmar_gas_check_free(struct dmar_ctx *ctx)
173{
174 struct dmar_map_entry *entry, *next, *l, *r;
175 dmar_gaddr_t v;
176
177 RB_FOREACH(entry, dmar_gas_entries_tree, &ctx->rb_root) {
178 KASSERT(ctx == entry->ctx,
179 ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
180 entry, entry->ctx));
181 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
182 if (next == NULL) {
183 MPASS(entry->free_after == ctx->end - entry->end);
184 } else {
185 MPASS(entry->free_after = next->start - entry->end);
186 MPASS(entry->end <= next->start);
187 }
188 l = RB_LEFT(entry, rb_entry);
189 r = RB_RIGHT(entry, rb_entry);
190 if (l == NULL && r == NULL) {
191 MPASS(entry->free_down == entry->free_after);
192 } else if (l == NULL && r != NULL) {
193 MPASS(entry->free_down = MAX(entry->free_after,
194 r->free_down));
195 } else if (r == NULL) {
196 MPASS(entry->free_down = MAX(entry->free_after,
197 l->free_down));
198 } else {
199 v = MAX(entry->free_after, l->free_down);
200 v = MAX(entry->free_down, r->free_down);
201 MPASS(entry->free_down == v);
202 }
203 }
204}
205#endif
206
207static bool
208dmar_gas_rb_insert(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
209{
210 struct dmar_map_entry *prev, *found;
211
212 found = RB_INSERT(dmar_gas_entries_tree, &ctx->rb_root, entry);
213 dmar_gas_fix_free(ctx, entry);
214 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
215 if (prev != NULL)
216 dmar_gas_fix_free(ctx, prev);
217 return (found == NULL);
218}
219
220static void
221dmar_gas_rb_remove(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
222{
223 struct dmar_map_entry *prev;
224
225 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
226 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
227 if (prev != NULL)
228 dmar_gas_fix_free(ctx, prev);
229}
230
231void
232dmar_gas_init_ctx(struct dmar_ctx *ctx)
233{
234 struct dmar_map_entry *begin, *end;
235
236 begin = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
237 end = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
238
239 DMAR_CTX_LOCK(ctx);
240 KASSERT(ctx->entries_cnt == 2, ("dirty ctx %p", ctx));
241 KASSERT(RB_EMPTY(&ctx->rb_root), ("non-empty entries %p", ctx));
242
243 begin->start = 0;
244 begin->end = DMAR_PAGE_SIZE;
245 begin->free_after = ctx->end - begin->end;
246 begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED;
247 dmar_gas_rb_insert(ctx, begin);
248
249 end->start = ctx->end;
250 end->end = ctx->end;
251 end->free_after = 0;
252 end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED;
253 dmar_gas_rb_insert(ctx, end);
254
255 ctx->first_place = begin;
256 ctx->last_place = end;
257 DMAR_CTX_UNLOCK(ctx);
258}
259
260void
261dmar_gas_fini_ctx(struct dmar_ctx *ctx)
262{
263 struct dmar_map_entry *entry, *entry1;
264
265 DMAR_CTX_ASSERT_LOCKED(ctx);
266 KASSERT(ctx->entries_cnt == 2, ("ctx still in use %p", ctx));
267
268 entry = RB_MIN(dmar_gas_entries_tree, &ctx->rb_root);
269 KASSERT(entry->start == 0, ("start entry start %p", ctx));
270 KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", ctx));
271 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
272 ("start entry flags %p", ctx));
273 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
274 dmar_gas_free_entry(ctx, entry);
275
276 entry = RB_MAX(dmar_gas_entries_tree, &ctx->rb_root);
277 KASSERT(entry->start == ctx->end, ("end entry start %p", ctx));
278 KASSERT(entry->end == ctx->end, ("end entry end %p", ctx));
279 KASSERT(entry->free_after == 0, ("end entry free_after%p", ctx));
280 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
281 ("end entry flags %p", ctx));
282 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
283 dmar_gas_free_entry(ctx, entry);
284
285 RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &ctx->rb_root, entry1) {
286 KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0,
287 ("non-RMRR entry left %p", ctx));
288 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
289 dmar_gas_free_entry(ctx, entry);
290 }
291}
292
293struct dmar_gas_match_args {
294 struct dmar_ctx *ctx;
295 dmar_gaddr_t size;
296 const struct bus_dma_tag_common *common;
297 u_int gas_flags;
298 struct dmar_map_entry *entry;
299};
300
301static bool
302dmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev,
303 dmar_gaddr_t end)
304{
305 dmar_gaddr_t bs, start;
306
307 if (a->entry->start + a->size > end)
308 return (false);
309
310 /* DMAR_PAGE_SIZE to create gap after new entry. */
311 if (a->entry->start < prev->end + DMAR_PAGE_SIZE ||
312 a->entry->start + a->size + DMAR_PAGE_SIZE > prev->end +
313 prev->free_after)
314 return (false);
315
316 /* No boundary crossing. */
317 if (dmar_test_boundary(a->entry->start, a->size, a->common->boundary))
318 return (true);
319
320 /*
321 * The start to start + size region crosses the boundary.
322 * Check if there is enough space after the next boundary
323 * after the prev->end.
324 */
325 bs = (a->entry->start + a->common->boundary) & ~(a->common->boundary
326 - 1);
327 start = roundup2(bs, a->common->alignment);
328 /* DMAR_PAGE_SIZE to create gap after new entry. */
329 if (start + a->size + DMAR_PAGE_SIZE <= prev->end + prev->free_after &&
330 start + a->size <= end && dmar_test_boundary(start, a->size,
331 a->common->boundary)) {
332 a->entry->start = start;
333 return (true);
334 }
335
336 /*
337 * Not enough space to align at the requested boundary, or
338 * boundary is smaller than the size, but allowed to split.
339 * We already checked that start + size does not overlap end.
340 *
341 * XXXKIB. It is possible that bs is exactly at the start of
342 * the next entry, then we do not have gap. Ignore for now.
343 */
344 if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) {
345 a->size = bs - a->entry->start;
346 return (true);
347 }
348
349 return (false);
350}
351
352static void
353dmar_gas_match_insert(struct dmar_gas_match_args *a,
354 struct dmar_map_entry *prev)
355{
356 struct dmar_map_entry *next;
357 bool found;
358
359 /*
360 * The prev->end is always aligned on the page size, which
361 * causes page alignment for the entry->start too. The size
362 * is checked to be multiple of the page size.
363 *
364 * The page sized gap is created between consequent
365 * allocations to ensure that out-of-bounds accesses fault.
366 */
367 a->entry->end = a->entry->start + a->size;
368
369 next = RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, prev);
370 KASSERT(next->start >= a->entry->end &&
371 next->start - a->entry->start >= a->size &&
372 prev->end <= a->entry->end,
373 ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) "
374 "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->ctx,
375 (uintmax_t)prev->start, (uintmax_t)prev->end,
376 (uintmax_t)prev->free_after,
377 (uintmax_t)next->start, (uintmax_t)next->end,
378 (uintmax_t)a->entry->start, (uintmax_t)a->entry->end));
379
380 prev->free_after = a->entry->start - prev->end;
381 a->entry->free_after = next->start - a->entry->end;
382
383 found = dmar_gas_rb_insert(a->ctx, a->entry);
384 KASSERT(found, ("found dup %p start %jx size %jx",
385 a->ctx, (uintmax_t)a->entry->start, (uintmax_t)a->size));
386 a->entry->flags = DMAR_MAP_ENTRY_MAP;
387
388 KASSERT(RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root,
389 a->entry) == prev,
390 ("entry %p prev %p inserted prev %p", a->entry, prev,
391 RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
392 KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root,
393 a->entry) == next,
394 ("entry %p next %p inserted next %p", a->entry, next,
395 RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
396}
397
398static int
399dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev)
400{
401 struct dmar_map_entry *l;
402 int ret;
403
404 if (prev->end < a->common->lowaddr) {
405 a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE,
406 a->common->alignment);
407 if (dmar_gas_match_one(a, prev, a->common->lowaddr)) {
408 dmar_gas_match_insert(a, prev);
409 return (0);
410 }
411 }
412 if (prev->free_down < a->size + DMAR_PAGE_SIZE)
413 return (ENOMEM);
414 l = RB_LEFT(prev, rb_entry);
415 if (l != NULL) {
416 ret = dmar_gas_lowermatch(a, l);
417 if (ret == 0)
418 return (0);
419 }
420 l = RB_RIGHT(prev, rb_entry);
421 if (l != NULL)
422 return (dmar_gas_lowermatch(a, l));
423 return (ENOMEM);
424}
425
426static int
427dmar_gas_uppermatch(struct dmar_gas_match_args *a)
428{
429 struct dmar_map_entry *next, *prev, find_entry;
430
431 find_entry.start = a->common->highaddr;
432 next = RB_NFIND(dmar_gas_entries_tree, &a->ctx->rb_root, &find_entry);
433 if (next == NULL)
434 return (ENOMEM);
435 prev = RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root, next);
436 KASSERT(prev != NULL, ("no prev %p %jx", a->ctx,
437 (uintmax_t)find_entry.start));
438 for (;;) {
439 a->entry->start = prev->start + DMAR_PAGE_SIZE;
440 if (a->entry->start < a->common->highaddr)
441 a->entry->start = a->common->highaddr;
442 a->entry->start = roundup2(a->entry->start,
443 a->common->alignment);
444 if (dmar_gas_match_one(a, prev, a->ctx->end)) {
445 dmar_gas_match_insert(a, prev);
446 return (0);
447 }
448
449 /*
450 * XXXKIB. This falls back to linear iteration over
451 * the free space in the high region. But high
452 * regions are almost unused, the code should be
453 * enough to cover the case, although in the
454 * non-optimal way.
455 */
456 prev = next;
457 next = RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, prev);
458 KASSERT(next != NULL, ("no next %p %jx", a->ctx,
459 (uintmax_t)find_entry.start));
460 if (next->end >= a->ctx->end)
461 return (ENOMEM);
462 }
463}
464
465static int
466dmar_gas_find_space(struct dmar_ctx *ctx,
467 const struct bus_dma_tag_common *common, dmar_gaddr_t size,
468 u_int flags, struct dmar_map_entry *entry)
469{
470 struct dmar_gas_match_args a;
471 int error;
472
473 DMAR_CTX_ASSERT_LOCKED(ctx);
474 KASSERT(entry->flags == 0, ("dirty entry %p %p", ctx, entry));
475 KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size));
476
477 a.ctx = ctx;
478 a.size = size;
479 a.common = common;
480 a.gas_flags = flags;
481 a.entry = entry;
482
483 /* Handle lower region. */
484 if (common->lowaddr > 0) {
485 error = dmar_gas_lowermatch(&a, RB_ROOT(&ctx->rb_root));
486 if (error == 0)
487 return (0);
488 KASSERT(error == ENOMEM,
489 ("error %d from dmar_gas_lowermatch", error));
490 }
491 /* Handle upper region. */
492 if (common->highaddr >= ctx->end)
493 return (ENOMEM);
494 error = dmar_gas_uppermatch(&a);
495 KASSERT(error == ENOMEM,
496 ("error %d from dmar_gas_uppermatch", error));
497 return (error);
498}
499
500static int
501dmar_gas_alloc_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
502 u_int flags)
503{
504 struct dmar_map_entry *next, *prev;
505 bool found;
506
507 DMAR_CTX_ASSERT_LOCKED(ctx);
508
509 if ((entry->start & DMAR_PAGE_MASK) != 0 ||
510 (entry->end & DMAR_PAGE_MASK) != 0)
511 return (EINVAL);
512 if (entry->start >= entry->end)
513 return (EINVAL);
514 if (entry->end >= ctx->end)
515 return (EINVAL);
516
517 next = RB_NFIND(dmar_gas_entries_tree, &ctx->rb_root, entry);
518 KASSERT(next != NULL, ("next must be non-null %p %jx", ctx,
519 (uintmax_t)entry->start));
520 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, next);
521 /* prev could be NULL */
522
523 /*
524 * Adapt to broken BIOSes which specify overlapping RMRR
525 * entries.
526 *
527 * XXXKIB: this does not handle a case when prev or next
528 * entries are completely covered by the current one, which
529 * extends both ways.
530 */
531 if (prev != NULL && prev->end > entry->start &&
532 (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) {
533 if ((prev->flags & DMAR_MAP_ENTRY_RMRR) == 0)
534 return (EBUSY);
535 entry->start = prev->end;
536 }
537 if (next != NULL && next->start < entry->end &&
538 (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) {
539 if ((next->flags & DMAR_MAP_ENTRY_RMRR) == 0)
540 return (EBUSY);
541 entry->end = next->start;
542 }
543 if (entry->end == entry->start)
544 return (0);
545
546 if (prev != NULL && prev->end > entry->start) {
547 /* This assumes that prev is the placeholder entry. */
548 dmar_gas_rb_remove(ctx, prev);
549 prev = NULL;
550 }
551 if (next != NULL && next->start < entry->end) {
552 dmar_gas_rb_remove(ctx, next);
553 next = NULL;
554 }
555
556 found = dmar_gas_rb_insert(ctx, entry);
557 KASSERT(found, ("found RMRR dup %p start %jx end %jx",
558 ctx, (uintmax_t)entry->start, (uintmax_t)entry->end));
559 entry->flags = DMAR_MAP_ENTRY_RMRR;
560
561#ifdef INVARIANTS
562 struct dmar_map_entry *ip, *in;
563 ip = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
564 in = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
565 KASSERT(prev == NULL || ip == prev,
566 ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)",
567 entry, entry->start, entry->end, prev,
568 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end,
569 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end));
570 KASSERT(next == NULL || in == next,
571 ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)",
572 entry, entry->start, entry->end, next,
573 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end,
574 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end));
575#endif
576
577 return (0);
578}
579
580void
581dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
582{
583
584 DMAR_CTX_ASSERT_LOCKED(ctx);
585 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
586 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP,
587 ("permanent entry %p %p", ctx, entry));
588
589 dmar_gas_rb_remove(ctx, entry);
590 entry->flags &= ~DMAR_MAP_ENTRY_MAP;
591#ifdef INVARIANTS
592 if (dmar_check_free)
593 dmar_gas_check_free(ctx);
594#endif
595}
596
597void
598dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
599{
600 struct dmar_map_entry *next, *prev;
601
602 DMAR_CTX_ASSERT_LOCKED(ctx);
603 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
604 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR,
605 ("non-RMRR entry %p %p", ctx, entry));
606
607 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
608 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
609 dmar_gas_rb_remove(ctx, entry);
610 entry->flags &= ~DMAR_MAP_ENTRY_RMRR;
611
612 if (prev == NULL)
613 dmar_gas_rb_insert(ctx, ctx->first_place);
614 if (next == NULL)
615 dmar_gas_rb_insert(ctx, ctx->last_place);
616}
617
618int
619dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
620 dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
621 struct dmar_map_entry **res)
622{
623 struct dmar_map_entry *entry;
624 int error;
625
626 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0,
627 ("invalid flags 0x%x", flags));
628
629 entry = dmar_gas_alloc_entry(ctx, (flags & DMAR_GM_CANWAIT) != 0 ?
630 DMAR_PGF_WAITOK : 0);
631 if (entry == NULL)
632 return (ENOMEM);
633 DMAR_CTX_LOCK(ctx);
634 error = dmar_gas_find_space(ctx, common, size, flags, entry);
635 if (error == ENOMEM) {
636 DMAR_CTX_UNLOCK(ctx);
637 dmar_gas_free_entry(ctx, entry);
638 return (error);
639 }
640#ifdef INVARIANTS
641 if (dmar_check_free)
642 dmar_gas_check_free(ctx);
643#endif
644 KASSERT(error == 0,
645 ("unexpected error %d from dmar_gas_find_entry", error));
646 KASSERT(entry->end < ctx->end, ("allocated GPA %jx, max GPA %jx",
647 (uintmax_t)entry->end, (uintmax_t)ctx->end));
648 entry->flags |= eflags;
649 DMAR_CTX_UNLOCK(ctx);
650
651 error = ctx_map_buf(ctx, entry->start, size, ma,
652 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) |
653 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) |
654 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) |
655 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
656 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
657 if (error == ENOMEM) {
658 dmar_ctx_unload_entry(entry, true);
659 return (error);
660 }
661 KASSERT(error == 0,
662 ("unexpected error %d from ctx_map_buf", error));
663
664 *res = entry;
665 return (0);
666}
667
668int
669dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
670 u_int eflags, u_int flags, vm_page_t *ma)
671{
672 dmar_gaddr_t start;
673 int error;
674
675 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", ctx,
676 entry, entry->flags));
677 KASSERT((flags & ~(DMAR_GM_CANWAIT)) == 0,
678 ("invalid flags 0x%x", flags));
679
680 start = entry->start;
681 DMAR_CTX_LOCK(ctx);
682 error = dmar_gas_alloc_region(ctx, entry, flags);
683 if (error != 0) {
684 DMAR_CTX_UNLOCK(ctx);
685 return (error);
686 }
687 entry->flags |= eflags;
688 DMAR_CTX_UNLOCK(ctx);
689 if (entry->end == entry->start)
690 return (0);
691
692 error = ctx_map_buf(ctx, entry->start, entry->end - entry->start,
693 ma + OFF_TO_IDX(start - entry->start),
694 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) |
695 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) |
696 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) |
697 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
698 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
699 if (error == ENOMEM) {
700 dmar_ctx_unload_entry(entry, false);
701 return (error);
702 }
703 KASSERT(error == 0,
704 ("unexpected error %d from ctx_map_buf", error));
705
706 return (0);
707}
708
709int
710dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
711 dmar_gaddr_t end)
712{
713 struct dmar_map_entry *entry;
714 int error;
715
716 entry = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
717 entry->start = start;
718 entry->end = end;
719 DMAR_CTX_LOCK(ctx);
720 error = dmar_gas_alloc_region(ctx, entry, DMAR_GM_CANWAIT);
721 if (error == 0)
722 entry->flags |= DMAR_MAP_ENTRY_UNMAPPED;
723 DMAR_CTX_UNLOCK(ctx);
724 if (error != 0)
725 dmar_gas_free_entry(ctx, entry);
726 return (error);
727}
| 53#include <dev/pci/pcivar.h>
54#include <vm/vm.h>
55#include <vm/vm_extern.h>
56#include <vm/vm_kern.h>
57#include <vm/vm_object.h>
58#include <vm/vm_page.h>
59#include <vm/vm_map.h>
60#include <vm/uma.h>
61#include <machine/atomic.h>
62#include <machine/bus.h>
63#include <machine/md_var.h>
64#include <machine/specialreg.h>
65#include <x86/include/busdma_impl.h>
66#include <x86/iommu/intel_reg.h>
67#include <x86/iommu/busdma_dmar.h>
68#include <x86/iommu/intel_dmar.h>
69
70/*
71 * Guest Address Space management.
72 */
73
74static uma_zone_t dmar_map_entry_zone;
75
76static void
77intel_gas_init(void)
78{
79
80 dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY",
81 sizeof(struct dmar_map_entry), NULL, NULL,
82 NULL, NULL, UMA_ALIGN_PTR, 0);
83}
84SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL);
85
86struct dmar_map_entry *
87dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags)
88{
89 struct dmar_map_entry *res;
90
91 KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0,
92 ("unsupported flags %x", flags));
93
94 res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) !=
95 0 ? M_WAITOK : M_NOWAIT) | M_ZERO);
96 if (res != NULL) {
97 res->ctx = ctx;
98 atomic_add_int(&ctx->entries_cnt, 1);
99 }
100 return (res);
101}
102
103void
104dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
105{
106
107 KASSERT(ctx == entry->ctx,
108 ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
109 entry, entry->ctx));
110 atomic_subtract_int(&ctx->entries_cnt, 1);
111 uma_zfree(dmar_map_entry_zone, entry);
112}
113
114static int
115dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b)
116{
117
118 /* Last entry have zero size, so <= */
119 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)",
120 a, (uintmax_t)a->start, (uintmax_t)a->end));
121 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)",
122 b, (uintmax_t)b->start, (uintmax_t)b->end));
123 KASSERT(a->end <= b->start || b->end <= a->start ||
124 a->end == a->start || b->end == b->start,
125 ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)",
126 a, (uintmax_t)a->start, (uintmax_t)a->end,
127 b, (uintmax_t)b->start, (uintmax_t)b->end));
128
129 if (a->end < b->end)
130 return (-1);
131 else if (b->end < a->end)
132 return (1);
133 return (0);
134}
135
136static void
137dmar_gas_augment_entry(struct dmar_map_entry *entry)
138{
139 struct dmar_map_entry *l, *r;
140
141 for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) {
142 l = RB_LEFT(entry, rb_entry);
143 r = RB_RIGHT(entry, rb_entry);
144 if (l == NULL && r == NULL) {
145 entry->free_down = entry->free_after;
146 } else if (l == NULL && r != NULL) {
147 entry->free_down = MAX(entry->free_after, r->free_down);
148 } else if (/*l != NULL && */ r == NULL) {
149 entry->free_down = MAX(entry->free_after, l->free_down);
150 } else /* if (l != NULL && r != NULL) */ {
151 entry->free_down = MAX(entry->free_after, l->free_down);
152 entry->free_down = MAX(entry->free_down, r->free_down);
153 }
154 }
155}
156
157RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
158 dmar_gas_cmp_entries);
159
160static void
161dmar_gas_fix_free(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
162{
163 struct dmar_map_entry *next;
164
165 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
166 entry->free_after = (next != NULL ? next->start : ctx->end) -
167 entry->end;
168 dmar_gas_augment_entry(entry);
169}
170
171#ifdef INVARIANTS
172static void
173dmar_gas_check_free(struct dmar_ctx *ctx)
174{
175 struct dmar_map_entry *entry, *next, *l, *r;
176 dmar_gaddr_t v;
177
178 RB_FOREACH(entry, dmar_gas_entries_tree, &ctx->rb_root) {
179 KASSERT(ctx == entry->ctx,
180 ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
181 entry, entry->ctx));
182 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
183 if (next == NULL) {
184 MPASS(entry->free_after == ctx->end - entry->end);
185 } else {
186 MPASS(entry->free_after = next->start - entry->end);
187 MPASS(entry->end <= next->start);
188 }
189 l = RB_LEFT(entry, rb_entry);
190 r = RB_RIGHT(entry, rb_entry);
191 if (l == NULL && r == NULL) {
192 MPASS(entry->free_down == entry->free_after);
193 } else if (l == NULL && r != NULL) {
194 MPASS(entry->free_down = MAX(entry->free_after,
195 r->free_down));
196 } else if (r == NULL) {
197 MPASS(entry->free_down = MAX(entry->free_after,
198 l->free_down));
199 } else {
200 v = MAX(entry->free_after, l->free_down);
201 v = MAX(entry->free_down, r->free_down);
202 MPASS(entry->free_down == v);
203 }
204 }
205}
206#endif
207
208static bool
209dmar_gas_rb_insert(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
210{
211 struct dmar_map_entry *prev, *found;
212
213 found = RB_INSERT(dmar_gas_entries_tree, &ctx->rb_root, entry);
214 dmar_gas_fix_free(ctx, entry);
215 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
216 if (prev != NULL)
217 dmar_gas_fix_free(ctx, prev);
218 return (found == NULL);
219}
220
221static void
222dmar_gas_rb_remove(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
223{
224 struct dmar_map_entry *prev;
225
226 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
227 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
228 if (prev != NULL)
229 dmar_gas_fix_free(ctx, prev);
230}
231
232void
233dmar_gas_init_ctx(struct dmar_ctx *ctx)
234{
235 struct dmar_map_entry *begin, *end;
236
237 begin = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
238 end = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
239
240 DMAR_CTX_LOCK(ctx);
241 KASSERT(ctx->entries_cnt == 2, ("dirty ctx %p", ctx));
242 KASSERT(RB_EMPTY(&ctx->rb_root), ("non-empty entries %p", ctx));
243
244 begin->start = 0;
245 begin->end = DMAR_PAGE_SIZE;
246 begin->free_after = ctx->end - begin->end;
247 begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED;
248 dmar_gas_rb_insert(ctx, begin);
249
250 end->start = ctx->end;
251 end->end = ctx->end;
252 end->free_after = 0;
253 end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED;
254 dmar_gas_rb_insert(ctx, end);
255
256 ctx->first_place = begin;
257 ctx->last_place = end;
258 DMAR_CTX_UNLOCK(ctx);
259}
260
261void
262dmar_gas_fini_ctx(struct dmar_ctx *ctx)
263{
264 struct dmar_map_entry *entry, *entry1;
265
266 DMAR_CTX_ASSERT_LOCKED(ctx);
267 KASSERT(ctx->entries_cnt == 2, ("ctx still in use %p", ctx));
268
269 entry = RB_MIN(dmar_gas_entries_tree, &ctx->rb_root);
270 KASSERT(entry->start == 0, ("start entry start %p", ctx));
271 KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", ctx));
272 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
273 ("start entry flags %p", ctx));
274 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
275 dmar_gas_free_entry(ctx, entry);
276
277 entry = RB_MAX(dmar_gas_entries_tree, &ctx->rb_root);
278 KASSERT(entry->start == ctx->end, ("end entry start %p", ctx));
279 KASSERT(entry->end == ctx->end, ("end entry end %p", ctx));
280 KASSERT(entry->free_after == 0, ("end entry free_after%p", ctx));
281 KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
282 ("end entry flags %p", ctx));
283 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
284 dmar_gas_free_entry(ctx, entry);
285
286 RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &ctx->rb_root, entry1) {
287 KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0,
288 ("non-RMRR entry left %p", ctx));
289 RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
290 dmar_gas_free_entry(ctx, entry);
291 }
292}
293
294struct dmar_gas_match_args {
295 struct dmar_ctx *ctx;
296 dmar_gaddr_t size;
297 const struct bus_dma_tag_common *common;
298 u_int gas_flags;
299 struct dmar_map_entry *entry;
300};
301
302static bool
303dmar_gas_match_one(struct dmar_gas_match_args *a, struct dmar_map_entry *prev,
304 dmar_gaddr_t end)
305{
306 dmar_gaddr_t bs, start;
307
308 if (a->entry->start + a->size > end)
309 return (false);
310
311 /* DMAR_PAGE_SIZE to create gap after new entry. */
312 if (a->entry->start < prev->end + DMAR_PAGE_SIZE ||
313 a->entry->start + a->size + DMAR_PAGE_SIZE > prev->end +
314 prev->free_after)
315 return (false);
316
317 /* No boundary crossing. */
318 if (dmar_test_boundary(a->entry->start, a->size, a->common->boundary))
319 return (true);
320
321 /*
322 * The start to start + size region crosses the boundary.
323 * Check if there is enough space after the next boundary
324 * after the prev->end.
325 */
326 bs = (a->entry->start + a->common->boundary) & ~(a->common->boundary
327 - 1);
328 start = roundup2(bs, a->common->alignment);
329 /* DMAR_PAGE_SIZE to create gap after new entry. */
330 if (start + a->size + DMAR_PAGE_SIZE <= prev->end + prev->free_after &&
331 start + a->size <= end && dmar_test_boundary(start, a->size,
332 a->common->boundary)) {
333 a->entry->start = start;
334 return (true);
335 }
336
337 /*
338 * Not enough space to align at the requested boundary, or
339 * boundary is smaller than the size, but allowed to split.
340 * We already checked that start + size does not overlap end.
341 *
342 * XXXKIB. It is possible that bs is exactly at the start of
343 * the next entry, then we do not have gap. Ignore for now.
344 */
345 if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) {
346 a->size = bs - a->entry->start;
347 return (true);
348 }
349
350 return (false);
351}
352
353static void
354dmar_gas_match_insert(struct dmar_gas_match_args *a,
355 struct dmar_map_entry *prev)
356{
357 struct dmar_map_entry *next;
358 bool found;
359
360 /*
361 * The prev->end is always aligned on the page size, which
362 * causes page alignment for the entry->start too. The size
363 * is checked to be multiple of the page size.
364 *
365 * The page sized gap is created between consequent
366 * allocations to ensure that out-of-bounds accesses fault.
367 */
368 a->entry->end = a->entry->start + a->size;
369
370 next = RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, prev);
371 KASSERT(next->start >= a->entry->end &&
372 next->start - a->entry->start >= a->size &&
373 prev->end <= a->entry->end,
374 ("dmar_gas_match_insert hole failed %p prev (%jx, %jx) "
375 "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->ctx,
376 (uintmax_t)prev->start, (uintmax_t)prev->end,
377 (uintmax_t)prev->free_after,
378 (uintmax_t)next->start, (uintmax_t)next->end,
379 (uintmax_t)a->entry->start, (uintmax_t)a->entry->end));
380
381 prev->free_after = a->entry->start - prev->end;
382 a->entry->free_after = next->start - a->entry->end;
383
384 found = dmar_gas_rb_insert(a->ctx, a->entry);
385 KASSERT(found, ("found dup %p start %jx size %jx",
386 a->ctx, (uintmax_t)a->entry->start, (uintmax_t)a->size));
387 a->entry->flags = DMAR_MAP_ENTRY_MAP;
388
389 KASSERT(RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root,
390 a->entry) == prev,
391 ("entry %p prev %p inserted prev %p", a->entry, prev,
392 RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
393 KASSERT(RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root,
394 a->entry) == next,
395 ("entry %p next %p inserted next %p", a->entry, next,
396 RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
397}
398
399static int
400dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *prev)
401{
402 struct dmar_map_entry *l;
403 int ret;
404
405 if (prev->end < a->common->lowaddr) {
406 a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE,
407 a->common->alignment);
408 if (dmar_gas_match_one(a, prev, a->common->lowaddr)) {
409 dmar_gas_match_insert(a, prev);
410 return (0);
411 }
412 }
413 if (prev->free_down < a->size + DMAR_PAGE_SIZE)
414 return (ENOMEM);
415 l = RB_LEFT(prev, rb_entry);
416 if (l != NULL) {
417 ret = dmar_gas_lowermatch(a, l);
418 if (ret == 0)
419 return (0);
420 }
421 l = RB_RIGHT(prev, rb_entry);
422 if (l != NULL)
423 return (dmar_gas_lowermatch(a, l));
424 return (ENOMEM);
425}
426
427static int
428dmar_gas_uppermatch(struct dmar_gas_match_args *a)
429{
430 struct dmar_map_entry *next, *prev, find_entry;
431
432 find_entry.start = a->common->highaddr;
433 next = RB_NFIND(dmar_gas_entries_tree, &a->ctx->rb_root, &find_entry);
434 if (next == NULL)
435 return (ENOMEM);
436 prev = RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root, next);
437 KASSERT(prev != NULL, ("no prev %p %jx", a->ctx,
438 (uintmax_t)find_entry.start));
439 for (;;) {
440 a->entry->start = prev->start + DMAR_PAGE_SIZE;
441 if (a->entry->start < a->common->highaddr)
442 a->entry->start = a->common->highaddr;
443 a->entry->start = roundup2(a->entry->start,
444 a->common->alignment);
445 if (dmar_gas_match_one(a, prev, a->ctx->end)) {
446 dmar_gas_match_insert(a, prev);
447 return (0);
448 }
449
450 /*
451 * XXXKIB. This falls back to linear iteration over
452 * the free space in the high region. But high
453 * regions are almost unused, the code should be
454 * enough to cover the case, although in the
455 * non-optimal way.
456 */
457 prev = next;
458 next = RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, prev);
459 KASSERT(next != NULL, ("no next %p %jx", a->ctx,
460 (uintmax_t)find_entry.start));
461 if (next->end >= a->ctx->end)
462 return (ENOMEM);
463 }
464}
465
466static int
467dmar_gas_find_space(struct dmar_ctx *ctx,
468 const struct bus_dma_tag_common *common, dmar_gaddr_t size,
469 u_int flags, struct dmar_map_entry *entry)
470{
471 struct dmar_gas_match_args a;
472 int error;
473
474 DMAR_CTX_ASSERT_LOCKED(ctx);
475 KASSERT(entry->flags == 0, ("dirty entry %p %p", ctx, entry));
476 KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size));
477
478 a.ctx = ctx;
479 a.size = size;
480 a.common = common;
481 a.gas_flags = flags;
482 a.entry = entry;
483
484 /* Handle lower region. */
485 if (common->lowaddr > 0) {
486 error = dmar_gas_lowermatch(&a, RB_ROOT(&ctx->rb_root));
487 if (error == 0)
488 return (0);
489 KASSERT(error == ENOMEM,
490 ("error %d from dmar_gas_lowermatch", error));
491 }
492 /* Handle upper region. */
493 if (common->highaddr >= ctx->end)
494 return (ENOMEM);
495 error = dmar_gas_uppermatch(&a);
496 KASSERT(error == ENOMEM,
497 ("error %d from dmar_gas_uppermatch", error));
498 return (error);
499}
500
501static int
502dmar_gas_alloc_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
503 u_int flags)
504{
505 struct dmar_map_entry *next, *prev;
506 bool found;
507
508 DMAR_CTX_ASSERT_LOCKED(ctx);
509
510 if ((entry->start & DMAR_PAGE_MASK) != 0 ||
511 (entry->end & DMAR_PAGE_MASK) != 0)
512 return (EINVAL);
513 if (entry->start >= entry->end)
514 return (EINVAL);
515 if (entry->end >= ctx->end)
516 return (EINVAL);
517
518 next = RB_NFIND(dmar_gas_entries_tree, &ctx->rb_root, entry);
519 KASSERT(next != NULL, ("next must be non-null %p %jx", ctx,
520 (uintmax_t)entry->start));
521 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, next);
522 /* prev could be NULL */
523
524 /*
525 * Adapt to broken BIOSes which specify overlapping RMRR
526 * entries.
527 *
528 * XXXKIB: this does not handle a case when prev or next
529 * entries are completely covered by the current one, which
530 * extends both ways.
531 */
532 if (prev != NULL && prev->end > entry->start &&
533 (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) {
534 if ((prev->flags & DMAR_MAP_ENTRY_RMRR) == 0)
535 return (EBUSY);
536 entry->start = prev->end;
537 }
538 if (next != NULL && next->start < entry->end &&
539 (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) {
540 if ((next->flags & DMAR_MAP_ENTRY_RMRR) == 0)
541 return (EBUSY);
542 entry->end = next->start;
543 }
544 if (entry->end == entry->start)
545 return (0);
546
547 if (prev != NULL && prev->end > entry->start) {
548 /* This assumes that prev is the placeholder entry. */
549 dmar_gas_rb_remove(ctx, prev);
550 prev = NULL;
551 }
552 if (next != NULL && next->start < entry->end) {
553 dmar_gas_rb_remove(ctx, next);
554 next = NULL;
555 }
556
557 found = dmar_gas_rb_insert(ctx, entry);
558 KASSERT(found, ("found RMRR dup %p start %jx end %jx",
559 ctx, (uintmax_t)entry->start, (uintmax_t)entry->end));
560 entry->flags = DMAR_MAP_ENTRY_RMRR;
561
562#ifdef INVARIANTS
563 struct dmar_map_entry *ip, *in;
564 ip = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
565 in = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
566 KASSERT(prev == NULL || ip == prev,
567 ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)",
568 entry, entry->start, entry->end, prev,
569 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end,
570 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end));
571 KASSERT(next == NULL || in == next,
572 ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)",
573 entry, entry->start, entry->end, next,
574 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end,
575 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end));
576#endif
577
578 return (0);
579}
580
581void
582dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
583{
584
585 DMAR_CTX_ASSERT_LOCKED(ctx);
586 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
587 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP,
588 ("permanent entry %p %p", ctx, entry));
589
590 dmar_gas_rb_remove(ctx, entry);
591 entry->flags &= ~DMAR_MAP_ENTRY_MAP;
592#ifdef INVARIANTS
593 if (dmar_check_free)
594 dmar_gas_check_free(ctx);
595#endif
596}
597
598void
599dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
600{
601 struct dmar_map_entry *next, *prev;
602
603 DMAR_CTX_ASSERT_LOCKED(ctx);
604 KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
605 DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR,
606 ("non-RMRR entry %p %p", ctx, entry));
607
608 prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
609 next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
610 dmar_gas_rb_remove(ctx, entry);
611 entry->flags &= ~DMAR_MAP_ENTRY_RMRR;
612
613 if (prev == NULL)
614 dmar_gas_rb_insert(ctx, ctx->first_place);
615 if (next == NULL)
616 dmar_gas_rb_insert(ctx, ctx->last_place);
617}
618
619int
620dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
621 dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
622 struct dmar_map_entry **res)
623{
624 struct dmar_map_entry *entry;
625 int error;
626
627 KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0,
628 ("invalid flags 0x%x", flags));
629
630 entry = dmar_gas_alloc_entry(ctx, (flags & DMAR_GM_CANWAIT) != 0 ?
631 DMAR_PGF_WAITOK : 0);
632 if (entry == NULL)
633 return (ENOMEM);
634 DMAR_CTX_LOCK(ctx);
635 error = dmar_gas_find_space(ctx, common, size, flags, entry);
636 if (error == ENOMEM) {
637 DMAR_CTX_UNLOCK(ctx);
638 dmar_gas_free_entry(ctx, entry);
639 return (error);
640 }
641#ifdef INVARIANTS
642 if (dmar_check_free)
643 dmar_gas_check_free(ctx);
644#endif
645 KASSERT(error == 0,
646 ("unexpected error %d from dmar_gas_find_entry", error));
647 KASSERT(entry->end < ctx->end, ("allocated GPA %jx, max GPA %jx",
648 (uintmax_t)entry->end, (uintmax_t)ctx->end));
649 entry->flags |= eflags;
650 DMAR_CTX_UNLOCK(ctx);
651
652 error = ctx_map_buf(ctx, entry->start, size, ma,
653 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) |
654 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) |
655 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) |
656 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
657 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
658 if (error == ENOMEM) {
659 dmar_ctx_unload_entry(entry, true);
660 return (error);
661 }
662 KASSERT(error == 0,
663 ("unexpected error %d from ctx_map_buf", error));
664
665 *res = entry;
666 return (0);
667}
668
669int
670dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
671 u_int eflags, u_int flags, vm_page_t *ma)
672{
673 dmar_gaddr_t start;
674 int error;
675
676 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", ctx,
677 entry, entry->flags));
678 KASSERT((flags & ~(DMAR_GM_CANWAIT)) == 0,
679 ("invalid flags 0x%x", flags));
680
681 start = entry->start;
682 DMAR_CTX_LOCK(ctx);
683 error = dmar_gas_alloc_region(ctx, entry, flags);
684 if (error != 0) {
685 DMAR_CTX_UNLOCK(ctx);
686 return (error);
687 }
688 entry->flags |= eflags;
689 DMAR_CTX_UNLOCK(ctx);
690 if (entry->end == entry->start)
691 return (0);
692
693 error = ctx_map_buf(ctx, entry->start, entry->end - entry->start,
694 ma + OFF_TO_IDX(start - entry->start),
695 ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) |
696 ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) |
697 ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) |
698 ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
699 (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
700 if (error == ENOMEM) {
701 dmar_ctx_unload_entry(entry, false);
702 return (error);
703 }
704 KASSERT(error == 0,
705 ("unexpected error %d from ctx_map_buf", error));
706
707 return (0);
708}
709
710int
711dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
712 dmar_gaddr_t end)
713{
714 struct dmar_map_entry *entry;
715 int error;
716
717 entry = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
718 entry->start = start;
719 entry->end = end;
720 DMAR_CTX_LOCK(ctx);
721 error = dmar_gas_alloc_region(ctx, entry, DMAR_GM_CANWAIT);
722 if (error == 0)
723 entry->flags |= DMAR_MAP_ENTRY_UNMAPPED;
724 DMAR_CTX_UNLOCK(ctx);
725 if (error != 0)
726 dmar_gas_free_entry(ctx, entry);
727 return (error);
728}
|