x86/iommu/intel_gas.c

33 #define	RB_AUGMENT(entry) dmar_gas_augment_entry(entry)
103 dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
106 	KASSERT(ctx == entry->ctx,
107 	    ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
108 	    entry, entry->ctx));
110 	uma_zfree(dmar_map_entry_zone, entry);
117 	/* Last entry have zero size, so <= */
118 	KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)",
120 	KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)",
136 dmar_gas_augment_entry(struct dmar_map_entry *entry)
140 	for (; entry != NULL; entry = RB_PARENT(entry, rb_entry)) {
141 		l = RB_LEFT(entry, rb_entry);
142 		r = RB_RIGHT(entry, rb_entry);
144 			entry->free_down = entry->free_after;
146 			entry->free_down = MAX(entry->free_after, r->free_down);
148 			entry->free_down = MAX(entry->free_after, l->free_down);
150 			entry->free_down = MAX(entry->free_after, l->free_down);
151 			entry->free_down = MAX(entry->free_down, r->free_down);
160 dmar_gas_fix_free(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
164 	next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
165 	entry->free_after = (next != NULL ? next->start : ctx->end) -
166 	    entry->end;
167 	dmar_gas_augment_entry(entry);
174 	struct dmar_map_entry *entry, *next, *l, *r;
177 	RB_FOREACH(entry, dmar_gas_entries_tree, &ctx->rb_root) {
178 		KASSERT(ctx == entry->ctx,
179 		    ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
180 		    entry, entry->ctx));
181 		next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
183 			MPASS(entry->free_after == ctx->end - entry->end);
185 			MPASS(entry->free_after = next->start - entry->end);
186 			MPASS(entry->end <= next->start);
188 		l = RB_LEFT(entry, rb_entry);
189 		r = RB_RIGHT(entry, rb_entry);
191 			MPASS(entry->free_down == entry->free_after);
193 			MPASS(entry->free_down = MAX(entry->free_after,
196 			MPASS(entry->free_down = MAX(entry->free_after,
199 			v = MAX(entry->free_after, l->free_down);
200 			v = MAX(entry->free_down, r->free_down);
201 			MPASS(entry->free_down == v);
208 dmar_gas_rb_insert(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
212 	found = RB_INSERT(dmar_gas_entries_tree, &ctx->rb_root, entry);
213 	dmar_gas_fix_free(ctx, entry);
214 	prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
221 dmar_gas_rb_remove(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
225 	prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
226 	RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
263 	struct dmar_map_entry *entry, *entry1;
268 	entry = RB_MIN(dmar_gas_entries_tree, &ctx->rb_root);
269 	KASSERT(entry->start == 0, ("start entry start %p", ctx));
270 	KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", ctx));
271 	KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
272 	    ("start entry flags %p", ctx));
273 	RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
274 	dmar_gas_free_entry(ctx, entry);
276 	entry = RB_MAX(dmar_gas_entries_tree, &ctx->rb_root);
277 	KASSERT(entry->start == ctx->end, ("end entry start %p", ctx));
278 	KASSERT(entry->end == ctx->end, ("end entry end %p", ctx));
279 	KASSERT(entry->free_after == 0, ("end entry free_after%p", ctx));
280 	KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE,
281 	    ("end entry flags %p", ctx));
282 	RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
283 	dmar_gas_free_entry(ctx, entry);
285 	RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &ctx->rb_root, entry1) {
286 		KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0,
287 		    ("non-RMRR entry left %p", ctx));
288 		RB_REMOVE(dmar_gas_entries_tree, &ctx->rb_root, entry);
289 		dmar_gas_free_entry(ctx, entry);
299 	struct dmar_map_entry *entry;
308 	if (a->entry->start + a->size > end)
311 	/* DMAR_PAGE_SIZE to create gap after new entry. */
312 	if (a->entry->start < prev->end + DMAR_PAGE_SIZE ||
313 	    a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE >
318 	if (dmar_test_boundary(a->entry->start + a->offset, a->size,
327 	bs = (a->entry->start + a->offset + a->common->boundary) &
330 	/* DMAR_PAGE_SIZE to create gap after new entry. */
336 		a->entry->start = start;
346 	 * the next entry, then we do not have gap.  Ignore for now.
349 		a->size = bs - a->entry->start;
365 	 * causes page alignment for the entry->start too.  The size
371 	a->entry->end = a->entry->start + a->size;
374 	KASSERT(next->start >= a->entry->end &&
375 	    next->start - a->entry->start >= a->size &&
376 	    prev->end <= a->entry->end,
378 	    "free_after %jx next (%jx, %jx) entry (%jx, %jx)", a->ctx,
382 	    (uintmax_t)a->entry->start, (uintmax_t)a->entry->end));
384 	prev->free_after = a->entry->start - prev->end;
385 	a->entry->free_after = next->start - a->entry->end;
387 	found = dmar_gas_rb_insert(a->ctx, a->entry);
389 	    a->ctx, (uintmax_t)a->entry->start, (uintmax_t)a->size));
390 	a->entry->flags = DMAR_MAP_ENTRY_MAP;
393 	    a->entry) == prev,
394 	    ("entry %p prev %p inserted prev %p", a->entry, prev,
395 	    RB_PREV(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
397 	    a->entry) == next,
398 	    ("entry %p next %p inserted next %p", a->entry, next,
399 	    RB_NEXT(dmar_gas_entries_tree, &a->ctx->rb_root, a->entry)));
409 		a->entry->start = roundup2(prev->end + DMAR_PAGE_SIZE,
443 		a->entry->start = prev->start + DMAR_PAGE_SIZE;
444 		if (a->entry->start < a->common->highaddr)
445 			a->entry->start = a->common->highaddr;
446 		a->entry->start = roundup2(a->entry->start,
472     int offset, u_int flags, struct dmar_map_entry *entry)
478 	KASSERT(entry->flags == 0, ("dirty entry %p %p", ctx, entry));
486 	a.entry = entry;
506 dmar_gas_alloc_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
514 	if ((entry->start & DMAR_PAGE_MASK) != 0 ||
515 	    (entry->end & DMAR_PAGE_MASK) != 0)
517 	if (entry->start >= entry->end)
519 	if (entry->end >= ctx->end)
522 	next = RB_NFIND(dmar_gas_entries_tree, &ctx->rb_root, entry);
524 	    (uintmax_t)entry->start));
536 	if (prev != NULL && prev->end > entry->start &&
540 		entry->start = prev->end;
542 	if (next != NULL && next->start < entry->end &&
546 		entry->end = next->start;
548 	if (entry->end == entry->start)
551 	if (prev != NULL && prev->end > entry->start) {
552 		/* This assumes that prev is the placeholder entry. */
556 	if (next != NULL && next->start < entry->end) {
561 	found = dmar_gas_rb_insert(ctx, entry);
563 	    ctx, (uintmax_t)entry->start, (uintmax_t)entry->end));
564 	entry->flags = DMAR_MAP_ENTRY_RMRR;
568 	ip = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
569 	in = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
572 	    entry, entry->start, entry->end, prev,
577 	    entry, entry->start, entry->end, next,
586 dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
590 	KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
592 	    ("permanent entry %p %p", ctx, entry));
594 	dmar_gas_rb_remove(ctx, entry);
595 	entry->flags &= ~DMAR_MAP_ENTRY_MAP;
603 dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
608 	KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR |
610 	    ("non-RMRR entry %p %p", ctx, entry));
612 	prev = RB_PREV(dmar_gas_entries_tree, &ctx->rb_root, entry);
613 	next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
614 	dmar_gas_rb_remove(ctx, entry);
615 	entry->flags &= ~DMAR_MAP_ENTRY_RMRR;
628 	struct dmar_map_entry *entry;
634 	entry = dmar_gas_alloc_entry(ctx, (flags & DMAR_GM_CANWAIT) != 0 ?
636 	if (entry == NULL)
639 	error = dmar_gas_find_space(ctx, common, size, offset, flags, entry);
642 		dmar_gas_free_entry(ctx, entry);
651 	KASSERT(entry->end < ctx->end, ("allocated GPA %jx, max GPA %jx",
652 	    (uintmax_t)entry->end, (uintmax_t)ctx->end));
653 	entry->flags |= eflags;
656 	error = ctx_map_buf(ctx, entry->start, entry->end - entry->start, ma,
663 		dmar_ctx_unload_entry(entry, true);
669 	*res = entry;
674 dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
680 	KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", ctx,
681 	    entry, entry->flags));
685 	start = entry->start;
687 	error = dmar_gas_alloc_region(ctx, entry, flags);
692 	entry->flags |= eflags;
694 	if (entry->end == entry->start)
697 	error = ctx_map_buf(ctx, entry->start, entry->end - entry->start,
698 	    ma + OFF_TO_IDX(start - entry->start),
705 		dmar_ctx_unload_entry(entry, false);
718 	struct dmar_map_entry *entry;
721 	entry = dmar_gas_alloc_entry(ctx, DMAR_PGF_WAITOK);
722 	entry->start = start;
723 	entry->end = end;
725 	error = dmar_gas_alloc_region(ctx, entry, DMAR_GM_CANWAIT);
727 		entry->flags |= DMAR_MAP_ENTRY_UNMAPPED;
730 		dmar_gas_free_entry(ctx, entry);