| intel_idpgtbl.c (257900) | intel_idpgtbl.c (277023) |
|---|---|
| 1/*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 14 unchanged lines hidden (view full) --- 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 14 unchanged lines hidden (view full) --- 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> |
| 31__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_idpgtbl.c 257900 2013-11-09 20:36:52Z dim $"); | 31__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_idpgtbl.c 277023 2015-01-11 20:27:15Z kib $"); |
| 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/malloc.h> 36#include <sys/bus.h> 37#include <sys/interrupt.h> 38#include <sys/kernel.h> 39#include <sys/ktr.h> --- 101 unchanged lines hidden (view full) --- 141 m1 = dmar_pgalloc(tbl->pgtbl_obj, base + i, 142 DMAR_PGF_NOALLOC); 143 KASSERT(m1 != NULL, ("lost page table page")); 144 pte[i].pte = (DMAR_PTE_ADDR_MASK & 145 VM_PAGE_TO_PHYS(m1)) | DMAR_PTE_R | DMAR_PTE_W; 146 } 147 } 148 /* ctx_get_idmap_pgtbl flushes CPU cache if needed. */ | 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/malloc.h> 36#include <sys/bus.h> 37#include <sys/interrupt.h> 38#include <sys/kernel.h> 39#include <sys/ktr.h> --- 101 unchanged lines hidden (view full) --- 141 m1 = dmar_pgalloc(tbl->pgtbl_obj, base + i, 142 DMAR_PGF_NOALLOC); 143 KASSERT(m1 != NULL, ("lost page table page")); 144 pte[i].pte = (DMAR_PTE_ADDR_MASK & 145 VM_PAGE_TO_PHYS(m1)) | DMAR_PTE_R | DMAR_PTE_W; 146 } 147 } 148 /* ctx_get_idmap_pgtbl flushes CPU cache if needed. */ |
| 149 dmar_unmap_pgtbl(sf, true); | 149 dmar_unmap_pgtbl(sf); |
| 150 VM_OBJECT_WLOCK(tbl->pgtbl_obj); 151} 152 153/* 154 * Find a ready and compatible identity-mapping page table in the 155 * cache. If not found, populate the identity-mapping page table for 156 * the context, up to the maxaddr. The maxaddr byte is allowed to be 157 * not mapped, which is aligned with the definition of Maxmem as the --- 198 unchanged lines hidden (view full) --- 356 DMAR_CTX_ASSERT_PGLOCKED(ctx); 357 KASSERT((flags & DMAR_PGF_OBJL) != 0, ("lost PGF_OBJL")); 358 359 idx = ctx_pgtbl_get_pindex(ctx, base, lvl); 360 if (*sf != NULL && idx == *idxp) { 361 pte = (dmar_pte_t *)sf_buf_kva(*sf); 362 } else { 363 if (*sf != NULL) | 150 VM_OBJECT_WLOCK(tbl->pgtbl_obj); 151} 152 153/* 154 * Find a ready and compatible identity-mapping page table in the 155 * cache. If not found, populate the identity-mapping page table for 156 * the context, up to the maxaddr. The maxaddr byte is allowed to be 157 * not mapped, which is aligned with the definition of Maxmem as the --- 198 unchanged lines hidden (view full) --- 356 DMAR_CTX_ASSERT_PGLOCKED(ctx); 357 KASSERT((flags & DMAR_PGF_OBJL) != 0, ("lost PGF_OBJL")); 358 359 idx = ctx_pgtbl_get_pindex(ctx, base, lvl); 360 if (*sf != NULL && idx == *idxp) { 361 pte = (dmar_pte_t *)sf_buf_kva(*sf); 362 } else { 363 if (*sf != NULL) |
| 364 dmar_unmap_pgtbl(*sf, DMAR_IS_COHERENT(ctx->dmar)); | 364 dmar_unmap_pgtbl(*sf); |
| 365 *idxp = idx; 366retry: 367 pte = dmar_map_pgtbl(ctx->pgtbl_obj, idx, flags, sf); 368 if (pte == NULL) { 369 KASSERT(lvl > 0, ("lost root page table page %p", ctx)); 370 /* 371 * Page table page does not exists, allocate 372 * it and create pte in the up level. --- 19 unchanged lines hidden (view full) --- 392 KASSERT(m->pindex != 0, 393 ("loosing root page %p", ctx)); 394 m->wire_count--; 395 dmar_pgfree(ctx->pgtbl_obj, m->pindex, flags); 396 return (NULL); 397 } 398 dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | 399 VM_PAGE_TO_PHYS(m)); | 365 *idxp = idx; 366retry: 367 pte = dmar_map_pgtbl(ctx->pgtbl_obj, idx, flags, sf); 368 if (pte == NULL) { 369 KASSERT(lvl > 0, ("lost root page table page %p", ctx)); 370 /* 371 * Page table page does not exists, allocate 372 * it and create pte in the up level. --- 19 unchanged lines hidden (view full) --- 392 KASSERT(m->pindex != 0, 393 ("loosing root page %p", ctx)); 394 m->wire_count--; 395 dmar_pgfree(ctx->pgtbl_obj, m->pindex, flags); 396 return (NULL); 397 } 398 dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | 399 VM_PAGE_TO_PHYS(m)); |
| 400 dmar_flush_pte_to_ram(ctx->dmar, ptep); |
|
| 400 sf_buf_page(sfp)->wire_count += 1; 401 m->wire_count--; | 401 sf_buf_page(sfp)->wire_count += 1; 402 m->wire_count--; |
| 402 dmar_unmap_pgtbl(sfp, DMAR_IS_COHERENT(ctx->dmar)); | 403 dmar_unmap_pgtbl(sfp); |
| 403 /* Only executed once. */ 404 goto retry; 405 } 406 } 407 pte += ctx_pgtbl_pte_off(ctx, base, lvl); 408 return (pte); 409} 410 --- 51 unchanged lines hidden (view full) --- 462 } 463 KASSERT(size >= pg_sz, 464 ("mapping loop overflow %p %jx %jx %jx", ctx, 465 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); 466 pte = ctx_pgtbl_map_pte(ctx, base, lvl, flags, &idx, &sf); 467 if (pte == NULL) { 468 KASSERT((flags & DMAR_PGF_WAITOK) == 0, 469 ("failed waitable pte alloc %p", ctx)); | 404 /* Only executed once. */ 405 goto retry; 406 } 407 } 408 pte += ctx_pgtbl_pte_off(ctx, base, lvl); 409 return (pte); 410} 411 --- 51 unchanged lines hidden (view full) --- 463 } 464 KASSERT(size >= pg_sz, 465 ("mapping loop overflow %p %jx %jx %jx", ctx, 466 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); 467 pte = ctx_pgtbl_map_pte(ctx, base, lvl, flags, &idx, &sf); 468 if (pte == NULL) { 469 KASSERT((flags & DMAR_PGF_WAITOK) == 0, 470 ("failed waitable pte alloc %p", ctx)); |
| 470 if (sf != NULL) { 471 dmar_unmap_pgtbl(sf, 472 DMAR_IS_COHERENT(ctx->dmar)); 473 } | 471 if (sf != NULL) 472 dmar_unmap_pgtbl(sf); |
| 474 ctx_unmap_buf_locked(ctx, base1, base - base1, flags); 475 TD_PINNED_ASSERT; 476 return (ENOMEM); 477 } 478 dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | 479 (superpage ? DMAR_PTE_SP : 0)); | 473 ctx_unmap_buf_locked(ctx, base1, base - base1, flags); 474 TD_PINNED_ASSERT; 475 return (ENOMEM); 476 } 477 dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | 478 (superpage ? DMAR_PTE_SP : 0)); |
| 479 dmar_flush_pte_to_ram(ctx->dmar, pte); |
|
| 480 sf_buf_page(sf)->wire_count += 1; 481 } 482 if (sf != NULL) | 480 sf_buf_page(sf)->wire_count += 1; 481 } 482 if (sf != NULL) |
| 483 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar)); | 483 dmar_unmap_pgtbl(sf); |
| 484 TD_PINNED_ASSERT; 485 return (0); 486} 487 488int 489ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size, 490 vm_page_t *ma, uint64_t pflags, int flags) 491{ --- 70 unchanged lines hidden (view full) --- 562 563static void 564ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gaddr_t base, int lvl, 565 int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) 566{ 567 vm_page_t m; 568 569 dmar_pte_clear(&pte->pte); | 484 TD_PINNED_ASSERT; 485 return (0); 486} 487 488int 489ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size, 490 vm_page_t *ma, uint64_t pflags, int flags) 491{ --- 70 unchanged lines hidden (view full) --- 562 563static void 564ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gaddr_t base, int lvl, 565 int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) 566{ 567 vm_page_t m; 568 569 dmar_pte_clear(&pte->pte); |
| 570 dmar_flush_pte_to_ram(ctx->dmar, pte); |
|
| 570 m = sf_buf_page(*sf); 571 if (free_sf) { | 571 m = sf_buf_page(*sf); 572 if (free_sf) { |
| 572 dmar_unmap_pgtbl(*sf, DMAR_IS_COHERENT(ctx->dmar)); | 573 dmar_unmap_pgtbl(*sf); |
| 573 *sf = NULL; 574 } 575 m->wire_count--; 576 if (m->wire_count != 0) 577 return; 578 KASSERT(lvl != 0, 579 ("lost reference (lvl) on root pg ctx %p base %jx lvl %d", 580 ctx, (uintmax_t)base, lvl)); --- 65 unchanged lines hidden (view full) --- 646 break; 647 } 648 } 649 KASSERT(size >= pg_sz, 650 ("unmapping loop overflow %p %jx %jx %jx", ctx, 651 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); 652 } 653 if (sf != NULL) | 574 *sf = NULL; 575 } 576 m->wire_count--; 577 if (m->wire_count != 0) 578 return; 579 KASSERT(lvl != 0, 580 ("lost reference (lvl) on root pg ctx %p base %jx lvl %d", 581 ctx, (uintmax_t)base, lvl)); --- 65 unchanged lines hidden (view full) --- 647 break; 648 } 649 } 650 KASSERT(size >= pg_sz, 651 ("unmapping loop overflow %p %jx %jx %jx", ctx, 652 (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); 653 } 654 if (sf != NULL) |
| 654 dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar)); | 655 dmar_unmap_pgtbl(sf); |
| 655 /* 656 * See 11.1 Write Buffer Flushing for an explanation why RWBF 657 * can be ignored there. 658 */ 659 660 TD_PINNED_ASSERT; 661 return (0); 662} --- 121 unchanged lines hidden --- | 656 /* 657 * See 11.1 Write Buffer Flushing for an explanation why RWBF 658 * can be ignored there. 659 */ 660 661 TD_PINNED_ASSERT; 662 return (0); 663} --- 121 unchanged lines hidden --- |