Deleted Added
sdiff udiff text old ( 257251 ) new ( 259512 )
full compact
intel_idpgtbl.c (257251) intel_idpgtbl.c (259512)
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_idpgtbl.c 257251 2013-10-28 13:33:29Z kib $");
31__FBSDID("$FreeBSD: stable/10/sys/x86/iommu/intel_idpgtbl.c 259512 2013-12-17 13:49:35Z kib $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/malloc.h>
36#include <sys/bus.h>
37#include <sys/interrupt.h>
38#include <sys/kernel.h>
39#include <sys/ktr.h>

--- 22 unchanged lines hidden (view full) ---

62#include <machine/specialreg.h>
63#include <x86/include/busdma_impl.h>
64#include <x86/iommu/intel_reg.h>
65#include <x86/iommu/busdma_dmar.h>
66#include <x86/iommu/intel_dmar.h>
67
68static int ctx_unmap_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
69 dmar_gaddr_t size, int flags);
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/malloc.h>
36#include <sys/bus.h>
37#include <sys/interrupt.h>
38#include <sys/kernel.h>
39#include <sys/ktr.h>

--- 22 unchanged lines hidden (view full) ---

62#include <machine/specialreg.h>
63#include <x86/include/busdma_impl.h>
64#include <x86/iommu/intel_reg.h>
65#include <x86/iommu/busdma_dmar.h>
66#include <x86/iommu/intel_dmar.h>
67
68static int ctx_unmap_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
69 dmar_gaddr_t size, int flags);
70static void ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base,
71 dmar_gaddr_t size, int flags);
72
73/*
74 * The cache of the identity mapping page tables for the DMARs. Using
75 * the cache saves significant amount of memory for page tables by
76 * reusing the page tables, since usually DMARs are identical and have
77 * the same capabilities. Still, cache records the information needed
78 * to match DMAR capabilities and page table format, to correctly
79 * handle different DMARs.

--- 84 unchanged lines hidden (view full) ---

164ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr)
165{
166 struct dmar_unit *unit;
167 struct idpgtbl *tbl;
168 vm_object_t res;
169 vm_page_t m;
170 int leaf, i;
171
70
71/*
72 * The cache of the identity mapping page tables for the DMARs. Using
73 * the cache saves significant amount of memory for page tables by
74 * reusing the page tables, since usually DMARs are identical and have
75 * the same capabilities. Still, cache records the information needed
76 * to match DMAR capabilities and page table format, to correctly
77 * handle different DMARs.

--- 84 unchanged lines hidden (view full) ---

162ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr)
163{
164 struct dmar_unit *unit;
165 struct idpgtbl *tbl;
166 vm_object_t res;
167 vm_page_t m;
168 int leaf, i;
169
170 leaf = 0; /* silence gcc */
171
172 /*
173 * First, determine where to stop the paging structures.
174 */
175 for (i = 0; i < ctx->pglvl; i++) {
176 if (i == ctx->pglvl - 1 || ctx_is_sp_lvl(ctx, i)) {
177 leaf = i;
178 break;
179 }

--- 227 unchanged lines hidden (view full) ---

407 pte += ctx_pgtbl_pte_off(ctx, base, lvl);
408 return (pte);
409}
410
411static int
412ctx_map_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
413 vm_page_t *ma, uint64_t pflags, int flags)
414{
172 /*
173 * First, determine where to stop the paging structures.
174 */
175 for (i = 0; i < ctx->pglvl; i++) {
176 if (i == ctx->pglvl - 1 || ctx_is_sp_lvl(ctx, i)) {
177 leaf = i;
178 break;
179 }

--- 227 unchanged lines hidden (view full) ---

407 pte += ctx_pgtbl_pte_off(ctx, base, lvl);
408 return (pte);
409}
410
411static int
412ctx_map_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
413 vm_page_t *ma, uint64_t pflags, int flags)
414{
415 struct dmar_unit *unit;
416 dmar_pte_t *pte;
417 struct sf_buf *sf;
418 dmar_gaddr_t pg_sz, base1, size1;
419 vm_pindex_t pi, c, idx, run_sz;
420 int lvl;
421 bool superpage;
422
423 DMAR_CTX_ASSERT_PGLOCKED(ctx);

--- 53 unchanged lines hidden (view full) ---

477 return (ENOMEM);
478 }
479 dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags |
480 (superpage ? DMAR_PTE_SP : 0));
481 sf_buf_page(sf)->wire_count += 1;
482 }
483 if (sf != NULL)
484 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
415 dmar_pte_t *pte;
416 struct sf_buf *sf;
417 dmar_gaddr_t pg_sz, base1, size1;
418 vm_pindex_t pi, c, idx, run_sz;
419 int lvl;
420 bool superpage;
421
422 DMAR_CTX_ASSERT_PGLOCKED(ctx);

--- 53 unchanged lines hidden (view full) ---

476 return (ENOMEM);
477 }
478 dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags |
479 (superpage ? DMAR_PTE_SP : 0));
480 sf_buf_page(sf)->wire_count += 1;
481 }
482 if (sf != NULL)
483 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
485 DMAR_CTX_PGUNLOCK(ctx);
486 unit = ctx->dmar;
487 if ((unit->hw_cap & DMAR_CAP_CM) != 0)
488 ctx_flush_iotlb(ctx, base1, size1, flags);
489 else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
490 /* See 11.1 Write Buffer Flushing. */
491 DMAR_LOCK(unit);
492 dmar_flush_write_bufs(unit);
493 DMAR_UNLOCK(unit);
494 }
495
496 TD_PINNED_ASSERT;
497 return (0);
498}
499
500int
501ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
502 vm_page_t *ma, uint64_t pflags, int flags)
503{
484 TD_PINNED_ASSERT;
485 return (0);
486}
487
488int
489ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
490 vm_page_t *ma, uint64_t pflags, int flags)
491{
492 struct dmar_unit *unit;
493 int error;
504
494
495 unit = ctx->dmar;
496
505 KASSERT((ctx->flags & DMAR_CTX_IDMAP) == 0,
506 ("modifying idmap pagetable ctx %p", ctx));
507 KASSERT((base & DMAR_PAGE_MASK) == 0,
508 ("non-aligned base %p %jx %jx", ctx, (uintmax_t)base,
509 (uintmax_t)size));
510 KASSERT((size & DMAR_PAGE_MASK) == 0,
511 ("non-aligned size %p %jx %jx", ctx, (uintmax_t)base,
512 (uintmax_t)size));

--- 9 unchanged lines hidden (view full) ---

522 ("size overflow %p %jx %jx", ctx, (uintmax_t)base,
523 (uintmax_t)size));
524 KASSERT((pflags & (DMAR_PTE_R | DMAR_PTE_W)) != 0,
525 ("neither read nor write %jx", (uintmax_t)pflags));
526 KASSERT((pflags & ~(DMAR_PTE_R | DMAR_PTE_W | DMAR_PTE_SNP |
527 DMAR_PTE_TM)) == 0,
528 ("invalid pte flags %jx", (uintmax_t)pflags));
529 KASSERT((pflags & DMAR_PTE_SNP) == 0 ||
497 KASSERT((ctx->flags & DMAR_CTX_IDMAP) == 0,
498 ("modifying idmap pagetable ctx %p", ctx));
499 KASSERT((base & DMAR_PAGE_MASK) == 0,
500 ("non-aligned base %p %jx %jx", ctx, (uintmax_t)base,
501 (uintmax_t)size));
502 KASSERT((size & DMAR_PAGE_MASK) == 0,
503 ("non-aligned size %p %jx %jx", ctx, (uintmax_t)base,
504 (uintmax_t)size));

--- 9 unchanged lines hidden (view full) ---

514 ("size overflow %p %jx %jx", ctx, (uintmax_t)base,
515 (uintmax_t)size));
516 KASSERT((pflags & (DMAR_PTE_R | DMAR_PTE_W)) != 0,
517 ("neither read nor write %jx", (uintmax_t)pflags));
518 KASSERT((pflags & ~(DMAR_PTE_R | DMAR_PTE_W | DMAR_PTE_SNP |
519 DMAR_PTE_TM)) == 0,
520 ("invalid pte flags %jx", (uintmax_t)pflags));
521 KASSERT((pflags & DMAR_PTE_SNP) == 0 ||
530 (ctx->dmar->hw_ecap & DMAR_ECAP_SC) != 0,
522 (unit->hw_ecap & DMAR_ECAP_SC) != 0,
531 ("PTE_SNP for dmar without snoop control %p %jx",
532 ctx, (uintmax_t)pflags));
533 KASSERT((pflags & DMAR_PTE_TM) == 0 ||
523 ("PTE_SNP for dmar without snoop control %p %jx",
524 ctx, (uintmax_t)pflags));
525 KASSERT((pflags & DMAR_PTE_TM) == 0 ||
534 (ctx->dmar->hw_ecap & DMAR_ECAP_DI) != 0,
526 (unit->hw_ecap & DMAR_ECAP_DI) != 0,
535 ("PTE_TM for dmar without DIOTLB %p %jx",
536 ctx, (uintmax_t)pflags));
537 KASSERT((flags & ~DMAR_PGF_WAITOK) == 0, ("invalid flags %x", flags));
538
539 DMAR_CTX_PGLOCK(ctx);
527 ("PTE_TM for dmar without DIOTLB %p %jx",
528 ctx, (uintmax_t)pflags));
529 KASSERT((flags & ~DMAR_PGF_WAITOK) == 0, ("invalid flags %x", flags));
530
531 DMAR_CTX_PGLOCK(ctx);
540 return (ctx_map_buf_locked(ctx, base, size, ma, pflags, flags));
532 error = ctx_map_buf_locked(ctx, base, size, ma, pflags, flags);
533 DMAR_CTX_PGUNLOCK(ctx);
534 if (error != 0)
535 return (error);
536
537 if ((unit->hw_cap & DMAR_CAP_CM) != 0)
538 ctx_flush_iotlb_sync(ctx, base, size);
539 else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
540 /* See 11.1 Write Buffer Flushing. */
541 DMAR_LOCK(unit);
542 dmar_flush_write_bufs(unit);
543 DMAR_UNLOCK(unit);
544 }
545 return (0);
541}
542
543static void ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gaddr_t base,
544 int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_fs);
545
546static void
547ctx_free_pgtbl_pde(struct dmar_ctx *ctx, dmar_gaddr_t base, int lvl, int flags)
548{

--- 62 unchanged lines hidden (view full) ---

611 KASSERT(base + size < (1ULL << ctx->agaw),
612 ("end too high %p %jx %jx agaw %d", ctx, (uintmax_t)base,
613 (uintmax_t)size, ctx->agaw));
614 KASSERT(base + size > base,
615 ("size overflow %p %jx %jx", ctx, (uintmax_t)base,
616 (uintmax_t)size));
617 KASSERT((flags & ~DMAR_PGF_WAITOK) == 0, ("invalid flags %x", flags));
618
546}
547
548static void ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gaddr_t base,
549 int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_fs);
550
551static void
552ctx_free_pgtbl_pde(struct dmar_ctx *ctx, dmar_gaddr_t base, int lvl, int flags)
553{

--- 62 unchanged lines hidden (view full) ---

616 KASSERT(base + size < (1ULL << ctx->agaw),
617 ("end too high %p %jx %jx agaw %d", ctx, (uintmax_t)base,
618 (uintmax_t)size, ctx->agaw));
619 KASSERT(base + size > base,
620 ("size overflow %p %jx %jx", ctx, (uintmax_t)base,
621 (uintmax_t)size));
622 KASSERT((flags & ~DMAR_PGF_WAITOK) == 0, ("invalid flags %x", flags));
623
624 pg_sz = 0; /* silence gcc */
619 base1 = base;
620 size1 = size;
621 flags |= DMAR_PGF_OBJL;
622 TD_PREP_PINNED_ASSERT;
623
624 for (sf = NULL; size > 0; base += pg_sz, size -= pg_sz) {
625 for (lvl = 0; lvl < ctx->pglvl; lvl++) {
626 if (lvl != ctx->pglvl - 1 && !ctx_is_sp_lvl(ctx, lvl))

--- 14 unchanged lines hidden (view full) ---

641 }
642 }
643 KASSERT(size >= pg_sz,
644 ("unmapping loop overflow %p %jx %jx %jx", ctx,
645 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz));
646 }
647 if (sf != NULL)
648 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
625 base1 = base;
626 size1 = size;
627 flags |= DMAR_PGF_OBJL;
628 TD_PREP_PINNED_ASSERT;
629
630 for (sf = NULL; size > 0; base += pg_sz, size -= pg_sz) {
631 for (lvl = 0; lvl < ctx->pglvl; lvl++) {
632 if (lvl != ctx->pglvl - 1 && !ctx_is_sp_lvl(ctx, lvl))

--- 14 unchanged lines hidden (view full) ---

647 }
648 }
649 KASSERT(size >= pg_sz,
650 ("unmapping loop overflow %p %jx %jx %jx", ctx,
651 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz));
652 }
653 if (sf != NULL)
654 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
649 DMAR_CTX_PGUNLOCK(ctx);
650 ctx_flush_iotlb(ctx, base1, size1, flags);
651 /*
652 * See 11.1 Write Buffer Flushing for an explanation why RWBF
653 * can be ignored there.
654 */
655
656 TD_PINNED_ASSERT;
657 return (0);
658}
659
660int
661ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
662 int flags)
663{
655 /*
656 * See 11.1 Write Buffer Flushing for an explanation why RWBF
657 * can be ignored there.
658 */
659
660 TD_PINNED_ASSERT;
661 return (0);
662}
663
664int
665ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
666 int flags)
667{
668 int error;
664
665 DMAR_CTX_PGLOCK(ctx);
669
670 DMAR_CTX_PGLOCK(ctx);
666 return (ctx_unmap_buf_locked(ctx, base, size, flags));
671 error = ctx_unmap_buf_locked(ctx, base, size, flags);
672 DMAR_CTX_PGUNLOCK(ctx);
673 return (error);
667}
668
669int
670ctx_alloc_pgtbl(struct dmar_ctx *ctx)
671{
672 vm_page_t m;
673
674 KASSERT(ctx->pgtbl_obj == NULL, ("already initialized %p", ctx));

--- 50 unchanged lines hidden (view full) ---

725 iotlbr = dmar_read8(unit, iro + DMAR_IOTLB_REG_OFF);
726 if ((iotlbr & DMAR_IOTLB_IVT) == 0)
727 break;
728 cpu_spinwait();
729 }
730 return (iotlbr);
731}
732
674}
675
676int
677ctx_alloc_pgtbl(struct dmar_ctx *ctx)
678{
679 vm_page_t m;
680
681 KASSERT(ctx->pgtbl_obj == NULL, ("already initialized %p", ctx));

--- 50 unchanged lines hidden (view full) ---

732 iotlbr = dmar_read8(unit, iro + DMAR_IOTLB_REG_OFF);
733 if ((iotlbr & DMAR_IOTLB_IVT) == 0)
734 break;
735 cpu_spinwait();
736 }
737 return (iotlbr);
738}
739
733/*
734 * flags is only intended for PGF_WAITOK, to disallow queued
735 * invalidation.
736 */
737static void
738ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
739 int flags)
740void
741ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size)
740{
741 struct dmar_unit *unit;
742 dmar_gaddr_t isize;
743 uint64_t iotlbr;
744 int am, iro;
745
746 unit = ctx->dmar;
742{
743 struct dmar_unit *unit;
744 dmar_gaddr_t isize;
745 uint64_t iotlbr;
746 int am, iro;
747
748 unit = ctx->dmar;
747#if 0
748 if ((unit->hw_ecap & DMAR_ECAP_QI) != 0 &&
749 (flags & DMAR_PGF_WAITOK) != 0) {
750 /*
751 * XXXKIB: There, a queued invalidation interface
752 * could be used. But since queued and registered
753 * interfaces cannot be used simultaneously, and we
754 * must use sleep-less (i.e. register) interface when
755 * DMAR_PGF_WAITOK is not specified, only register
756 * interface is suitable.
757 */
758 return;
759 }
760#endif
749 KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call",
750 unit->unit));
761 iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16;
762 DMAR_LOCK(unit);
763 if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) {
764 iotlbr = ctx_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM |
765 DMAR_IOTLB_DID(ctx->domain), iro);
766 KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
767 DMAR_IOTLB_IAIG_INVLD,
768 ("dmar%d: invalidation failed %jx", unit->unit,
769 (uintmax_t)iotlbr));
770 } else {
771 for (; size > 0; base += isize, size -= isize) {
751 iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16;
752 DMAR_LOCK(unit);
753 if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) {
754 iotlbr = ctx_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM |
755 DMAR_IOTLB_DID(ctx->domain), iro);
756 KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
757 DMAR_IOTLB_IAIG_INVLD,
758 ("dmar%d: invalidation failed %jx", unit->unit,
759 (uintmax_t)iotlbr));
760 } else {
761 for (; size > 0; base += isize, size -= isize) {
772 for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) {
773 isize = 1ULL << (am + DMAR_PAGE_SHIFT);
774 if ((base & (isize - 1)) == 0 && size >= isize)
775 break;
776 if (am == 0)
777 break;
778 }
762 am = calc_am(unit, base, size, &isize);
779 dmar_write8(unit, iro, base | am);
780 iotlbr = ctx_wait_iotlb_flush(unit,
781 DMAR_IOTLB_IIRG_PAGE | DMAR_IOTLB_DID(ctx->domain),
782 iro);
783 KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
784 DMAR_IOTLB_IAIG_INVLD,
785 ("dmar%d: PSI invalidation failed "
786 "iotlbr 0x%jx base 0x%jx size 0x%jx am %d",

--- 13 unchanged lines hidden --- 		763 dmar_write8(unit, iro, base | am);
764 iotlbr = ctx_wait_iotlb_flush(unit,
765 DMAR_IOTLB_IIRG_PAGE | DMAR_IOTLB_DID(ctx->domain),
766 iro);
767 KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
768 DMAR_IOTLB_IAIG_INVLD,
769 ("dmar%d: PSI invalidation failed "
770 "iotlbr 0x%jx base 0x%jx size 0x%jx am %d",

--- 13 unchanged lines hidden ---