196998Sjake/*- 296998Sjake * Copyright (c) 2001 Jake Burkholder. 396998Sjake * All rights reserved. 496998Sjake * 596998Sjake * Redistribution and use in source and binary forms, with or without 696998Sjake * modification, are permitted provided that the following conditions 796998Sjake * are met: 896998Sjake * 1. Redistributions of source code must retain the above copyright 996998Sjake * notice, this list of conditions and the following disclaimer. 1096998Sjake * 2. Redistributions in binary form must reproduce the above copyright 1196998Sjake * notice, this list of conditions and the following disclaimer in the 1296998Sjake * documentation and/or other materials provided with the distribution. 1396998Sjake * 1496998Sjake * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1596998Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1696998Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1796998Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1896998Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1996998Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2096998Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2196998Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2296998Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2396998Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2496998Sjake * SUCH DAMAGE. 2596998Sjake */ 2696998Sjake 27176994Smarius#include <sys/cdefs.h> 28176994Smarius__FBSDID("$FreeBSD$"); 29176994Smarius 30108700Sjake#include "opt_pmap.h" 31108700Sjake 3296998Sjake#include <sys/param.h> 3396998Sjake#include <sys/systm.h> 3498031Sjake#include <sys/ktr.h> 3596998Sjake#include <sys/pcpu.h> 36108187Sjake#include <sys/lock.h> 37108187Sjake#include <sys/mutex.h> 3896998Sjake#include <sys/smp.h> 39108700Sjake#include <sys/sysctl.h> 4096998Sjake 4196998Sjake#include <vm/vm.h> 4296998Sjake#include <vm/pmap.h> 4396998Sjake 4496998Sjake#include <machine/pmap.h> 4596998Sjake#include <machine/smp.h> 4696998Sjake#include <machine/tlb.h> 47182877Smarius#include <machine/vmparam.h> 4896998Sjake 49108700SjakePMAP_STATS_VAR(tlb_ncontext_demap); 50108700SjakePMAP_STATS_VAR(tlb_npage_demap); 51108700SjakePMAP_STATS_VAR(tlb_nrange_demap); 52108700Sjake 53176994Smariustlb_flush_nonlocked_t *tlb_flush_nonlocked; 54113453Sjaketlb_flush_user_t *tlb_flush_user; 5598031Sjake 5696998Sjake/* 5796998Sjake * Some tlb operations must be atomic, so no interrupt or trap can be allowed 5896998Sjake * while they are in progress. Traps should not happen, but interrupts need to 5996998Sjake * be explicitely disabled. critical_enter() cannot be used here, since it only 6096998Sjake * disables soft interrupts. 6196998Sjake */ 6296998Sjake 6396998Sjakevoid 6496998Sjaketlb_context_demap(struct pmap *pm) 6596998Sjake{ 6696998Sjake void *cookie; 67181701Smarius register_t s; 6896998Sjake 6996998Sjake /* 7096998Sjake * It is important that we are not interrupted or preempted while 7196998Sjake * doing the IPIs. The interrupted CPU may hold locks, and since 7296998Sjake * it will wait for the CPU that sent the IPI, this can lead 7396998Sjake * to a deadlock when an interrupt comes in on that CPU and it's 7496998Sjake * handler tries to grab one of that locks. This will only happen for 7596998Sjake * spin locks, but these IPI types are delivered even if normal 7696998Sjake * interrupts are disabled, so the lock critical section will not 7796998Sjake * protect the target processor from entering the IPI handler with 7896998Sjake * the lock held. 7996998Sjake */ 80108700Sjake PMAP_STATS_INC(tlb_ncontext_demap); 8196998Sjake cookie = ipi_tlb_context_demap(pm); 82216891Smarius s = intr_disable(); 83223346Smarius if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { 84181701Smarius KASSERT(pm->pm_context[curcpu] != -1, 8596998Sjake ("tlb_context_demap: inactive pmap?")); 8696998Sjake stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_DMMU_DEMAP, 0); 8796998Sjake stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_IMMU_DEMAP, 0); 88182877Smarius flush(KERNBASE); 8996998Sjake } 90216891Smarius intr_restore(s); 9196998Sjake ipi_wait(cookie); 9296998Sjake} 9396998Sjake 9496998Sjakevoid 95100718Sjaketlb_page_demap(struct pmap *pm, vm_offset_t va) 9696998Sjake{ 9796998Sjake u_long flags; 9896998Sjake void *cookie; 99181701Smarius register_t s; 10096998Sjake 101108700Sjake PMAP_STATS_INC(tlb_npage_demap); 102100718Sjake cookie = ipi_tlb_page_demap(pm, va); 103216891Smarius s = intr_disable(); 104223346Smarius if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { 105181701Smarius KASSERT(pm->pm_context[curcpu] != -1, 10696998Sjake ("tlb_page_demap: inactive pmap?")); 10796998Sjake if (pm == kernel_pmap) 10896998Sjake flags = TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE; 10996998Sjake else 11096998Sjake flags = TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE; 111176994Smarius 112100718Sjake stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); 113100718Sjake stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); 114182877Smarius flush(KERNBASE); 11596998Sjake } 116216891Smarius intr_restore(s); 11796998Sjake ipi_wait(cookie); 11896998Sjake} 11996998Sjake 12096998Sjakevoid 12196998Sjaketlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) 12296998Sjake{ 12396998Sjake vm_offset_t va; 12496998Sjake void *cookie; 12596998Sjake u_long flags; 126181701Smarius register_t s; 12796998Sjake 128108700Sjake PMAP_STATS_INC(tlb_nrange_demap); 12996998Sjake cookie = ipi_tlb_range_demap(pm, start, end); 130216891Smarius s = intr_disable(); 131223346Smarius if (CPU_ISSET(PCPU_GET(cpuid), &pm->pm_active)) { 132181701Smarius KASSERT(pm->pm_context[curcpu] != -1, 13396998Sjake ("tlb_range_demap: inactive pmap?")); 13496998Sjake if (pm == kernel_pmap) 13596998Sjake flags = TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE; 13696998Sjake else 13796998Sjake flags = TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE; 138176994Smarius 13996998Sjake for (va = start; va < end; va += PAGE_SIZE) { 14096998Sjake stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); 14196998Sjake stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); 142182877Smarius flush(KERNBASE); 14396998Sjake } 14496998Sjake } 145216891Smarius intr_restore(s); 14696998Sjake ipi_wait(cookie); 14796998Sjake} 148