mmu_oea.c revision 254025
1139825Simp/*- 290643Sbenno * Copyright (c) 2001 The NetBSD Foundation, Inc. 390643Sbenno * All rights reserved. 490643Sbenno * 590643Sbenno * This code is derived from software contributed to The NetBSD Foundation 690643Sbenno * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc. 790643Sbenno * 890643Sbenno * Redistribution and use in source and binary forms, with or without 990643Sbenno * modification, are permitted provided that the following conditions 1090643Sbenno * are met: 1190643Sbenno * 1. Redistributions of source code must retain the above copyright 1290643Sbenno * notice, this list of conditions and the following disclaimer. 1390643Sbenno * 2. Redistributions in binary form must reproduce the above copyright 1490643Sbenno * notice, this list of conditions and the following disclaimer in the 1590643Sbenno * documentation and/or other materials provided with the distribution. 1690643Sbenno * 3. All advertising materials mentioning features or use of this software 1790643Sbenno * must display the following acknowledgement: 1890643Sbenno * This product includes software developed by the NetBSD 1990643Sbenno * Foundation, Inc. and its contributors. 2090643Sbenno * 4. Neither the name of The NetBSD Foundation nor the names of its 2190643Sbenno * contributors may be used to endorse or promote products derived 2290643Sbenno * from this software without specific prior written permission. 2390643Sbenno * 2490643Sbenno * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 2590643Sbenno * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 2690643Sbenno * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 2790643Sbenno * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 2890643Sbenno * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2990643Sbenno * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 3090643Sbenno * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 3190643Sbenno * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 3290643Sbenno * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 3390643Sbenno * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3490643Sbenno * POSSIBILITY OF SUCH DAMAGE. 3590643Sbenno */ 36139825Simp/*- 3777957Sbenno * Copyright (C) 1995, 1996 Wolfgang Solfrank. 3877957Sbenno * Copyright (C) 1995, 1996 TooLs GmbH. 3977957Sbenno * All rights reserved. 4077957Sbenno * 4177957Sbenno * Redistribution and use in source and binary forms, with or without 4277957Sbenno * modification, are permitted provided that the following conditions 4377957Sbenno * are met: 4477957Sbenno * 1. Redistributions of source code must retain the above copyright 4577957Sbenno * notice, this list of conditions and the following disclaimer. 4677957Sbenno * 2. Redistributions in binary form must reproduce the above copyright 4777957Sbenno * notice, this list of conditions and the following disclaimer in the 4877957Sbenno * documentation and/or other materials provided with the distribution. 4977957Sbenno * 3. All advertising materials mentioning features or use of this software 5077957Sbenno * must display the following acknowledgement: 5177957Sbenno * This product includes software developed by TooLs GmbH. 5277957Sbenno * 4. The name of TooLs GmbH may not be used to endorse or promote products 5377957Sbenno * derived from this software without specific prior written permission. 5477957Sbenno * 5577957Sbenno * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 5677957Sbenno * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 5777957Sbenno * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 5877957Sbenno * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 5977957Sbenno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 6077957Sbenno * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 6177957Sbenno * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 6277957Sbenno * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 6377957Sbenno * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 6477957Sbenno * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 6577957Sbenno * 6678880Sbenno * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ 6777957Sbenno */ 68139825Simp/*- 6977957Sbenno * Copyright (C) 2001 Benno Rice. 7077957Sbenno * All rights reserved. 7177957Sbenno * 7277957Sbenno * Redistribution and use in source and binary forms, with or without 7377957Sbenno * modification, are permitted provided that the following conditions 7477957Sbenno * are met: 7577957Sbenno * 1. Redistributions of source code must retain the above copyright 7677957Sbenno * notice, this list of conditions and the following disclaimer. 7777957Sbenno * 2. Redistributions in binary form must reproduce the above copyright 7877957Sbenno * notice, this list of conditions and the following disclaimer in the 7977957Sbenno * documentation and/or other materials provided with the distribution. 8077957Sbenno * 8177957Sbenno * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 8277957Sbenno * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 8377957Sbenno * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 8477957Sbenno * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 8577957Sbenno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 8677957Sbenno * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 8777957Sbenno * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 8877957Sbenno * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 8977957Sbenno * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 9077957Sbenno * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 9177957Sbenno */ 9277957Sbenno 93113038Sobrien#include <sys/cdefs.h> 94113038Sobrien__FBSDID("$FreeBSD: head/sys/powerpc/aim/mmu_oea.c 254025 2013-08-07 06:21:20Z jeff $"); 9577957Sbenno 9690643Sbenno/* 9790643Sbenno * Manages physical address maps. 9890643Sbenno * 9990643Sbenno * Since the information managed by this module is also stored by the 10090643Sbenno * logical address mapping module, this module may throw away valid virtual 10190643Sbenno * to physical mappings at almost any time. However, invalidations of 10290643Sbenno * mappings must be done as requested. 10390643Sbenno * 10490643Sbenno * In order to cope with hardware architectures which make virtual to 10590643Sbenno * physical map invalidates expensive, this module may delay invalidate 10690643Sbenno * reduced protection operations until such time as they are actually 10790643Sbenno * necessary. This module is given full information as to which processors 10890643Sbenno * are currently using which maps, and to when physical maps must be made 10990643Sbenno * correct. 11090643Sbenno */ 11190643Sbenno 112118239Speter#include "opt_kstack_pages.h" 113118239Speter 11477957Sbenno#include <sys/param.h> 11580431Speter#include <sys/kernel.h> 116222813Sattilio#include <sys/queue.h> 117222813Sattilio#include <sys/cpuset.h> 11890643Sbenno#include <sys/ktr.h> 11990643Sbenno#include <sys/lock.h> 12090643Sbenno#include <sys/msgbuf.h> 12190643Sbenno#include <sys/mutex.h> 12277957Sbenno#include <sys/proc.h> 123238159Salc#include <sys/rwlock.h> 124222813Sattilio#include <sys/sched.h> 12590643Sbenno#include <sys/sysctl.h> 12690643Sbenno#include <sys/systm.h> 12777957Sbenno#include <sys/vmmeter.h> 12877957Sbenno 12990643Sbenno#include <dev/ofw/openfirm.h> 13090643Sbenno 131152180Sgrehan#include <vm/vm.h> 13277957Sbenno#include <vm/vm_param.h> 13377957Sbenno#include <vm/vm_kern.h> 13477957Sbenno#include <vm/vm_page.h> 13577957Sbenno#include <vm/vm_map.h> 13677957Sbenno#include <vm/vm_object.h> 13777957Sbenno#include <vm/vm_extern.h> 13877957Sbenno#include <vm/vm_pageout.h> 13992847Sjeff#include <vm/uma.h> 14077957Sbenno 141125687Sgrehan#include <machine/cpu.h> 142192067Snwhitehorn#include <machine/platform.h> 14383730Smp#include <machine/bat.h> 14490643Sbenno#include <machine/frame.h> 14590643Sbenno#include <machine/md_var.h> 14690643Sbenno#include <machine/psl.h> 14777957Sbenno#include <machine/pte.h> 148178628Smarcel#include <machine/smp.h> 14990643Sbenno#include <machine/sr.h> 150152180Sgrehan#include <machine/mmuvar.h> 151228609Snwhitehorn#include <machine/trap_aim.h> 15277957Sbenno 153152180Sgrehan#include "mmu_if.h" 15477957Sbenno 155152180Sgrehan#define MOEA_DEBUG 156152180Sgrehan 15790643Sbenno#define TODO panic("%s: not implemented", __func__); 15877957Sbenno 15990643Sbenno#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) 16090643Sbenno#define VSID_TO_SR(vsid) ((vsid) & 0xf) 16190643Sbenno#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) 16290643Sbenno 16390643Sbennostruct ofw_map { 16490643Sbenno vm_offset_t om_va; 16590643Sbenno vm_size_t om_len; 16690643Sbenno vm_offset_t om_pa; 16790643Sbenno u_int om_mode; 16890643Sbenno}; 16977957Sbenno 170249864Sjhibbitsextern unsigned char _etext[]; 171249864Sjhibbitsextern unsigned char _end[]; 172249864Sjhibbits 173249864Sjhibbitsextern int dumpsys_minidump; 174249864Sjhibbits 17590643Sbenno/* 17690643Sbenno * Map of physical memory regions. 17790643Sbenno */ 17897346Sbennostatic struct mem_region *regions; 17997346Sbennostatic struct mem_region *pregions; 180209975Snwhitehornstatic u_int phys_avail_count; 181209975Snwhitehornstatic int regions_sz, pregions_sz; 182100319Sbennostatic struct ofw_map *translations; 18377957Sbenno 18490643Sbenno/* 185134535Salc * Lock for the pteg and pvo tables. 186134535Salc */ 187152180Sgrehanstruct mtx moea_table_mutex; 188212278Snwhitehornstruct mtx moea_vsid_mutex; 189134535Salc 190183094Smarcel/* tlbie instruction synchronization */ 191183094Smarcelstatic struct mtx tlbie_mtx; 192183094Smarcel 193134535Salc/* 19490643Sbenno * PTEG data. 19590643Sbenno */ 196152180Sgrehanstatic struct pteg *moea_pteg_table; 197152180Sgrehanu_int moea_pteg_count; 198152180Sgrehanu_int moea_pteg_mask; 19977957Sbenno 20090643Sbenno/* 20190643Sbenno * PVO data. 20290643Sbenno */ 203152180Sgrehanstruct pvo_head *moea_pvo_table; /* pvo entries by pteg index */ 204152180Sgrehanstruct pvo_head moea_pvo_kunmanaged = 205152180Sgrehan LIST_HEAD_INITIALIZER(moea_pvo_kunmanaged); /* list of unmanaged pages */ 20677957Sbenno 207242534Sattiliostatic struct rwlock_padalign pvh_global_lock; 208238159Salc 209152180Sgrehanuma_zone_t moea_upvo_zone; /* zone for pvo entries for unmanaged pages */ 210152180Sgrehanuma_zone_t moea_mpvo_zone; /* zone for pvo entries for managed pages */ 21177957Sbenno 21299037Sbenno#define BPVO_POOL_SIZE 32768 213152180Sgrehanstatic struct pvo_entry *moea_bpvo_pool; 214152180Sgrehanstatic int moea_bpvo_pool_index = 0; 21577957Sbenno 21690643Sbenno#define VSID_NBPW (sizeof(u_int32_t) * 8) 217152180Sgrehanstatic u_int moea_vsid_bitmap[NPMAPS / VSID_NBPW]; 21877957Sbenno 219152180Sgrehanstatic boolean_t moea_initialized = FALSE; 22077957Sbenno 22190643Sbenno/* 22290643Sbenno * Statistics. 22390643Sbenno */ 224152180Sgrehanu_int moea_pte_valid = 0; 225152180Sgrehanu_int moea_pte_overflow = 0; 226152180Sgrehanu_int moea_pte_replacements = 0; 227152180Sgrehanu_int moea_pvo_entries = 0; 228152180Sgrehanu_int moea_pvo_enter_calls = 0; 229152180Sgrehanu_int moea_pvo_remove_calls = 0; 230152180Sgrehanu_int moea_pte_spills = 0; 231152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pte_valid, CTLFLAG_RD, &moea_pte_valid, 23290643Sbenno 0, ""); 233152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pte_overflow, CTLFLAG_RD, 234152180Sgrehan &moea_pte_overflow, 0, ""); 235152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pte_replacements, CTLFLAG_RD, 236152180Sgrehan &moea_pte_replacements, 0, ""); 237152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pvo_entries, CTLFLAG_RD, &moea_pvo_entries, 23890643Sbenno 0, ""); 239152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pvo_enter_calls, CTLFLAG_RD, 240152180Sgrehan &moea_pvo_enter_calls, 0, ""); 241152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pvo_remove_calls, CTLFLAG_RD, 242152180Sgrehan &moea_pvo_remove_calls, 0, ""); 243152180SgrehanSYSCTL_INT(_machdep, OID_AUTO, moea_pte_spills, CTLFLAG_RD, 244152180Sgrehan &moea_pte_spills, 0, ""); 24577957Sbenno 24690643Sbenno/* 247152180Sgrehan * Allocate physical memory for use in moea_bootstrap. 24890643Sbenno */ 249152180Sgrehanstatic vm_offset_t moea_bootstrap_alloc(vm_size_t, u_int); 25077957Sbenno 25190643Sbenno/* 25290643Sbenno * PTE calls. 25390643Sbenno */ 254152180Sgrehanstatic int moea_pte_insert(u_int, struct pte *); 25577957Sbenno 25677957Sbenno/* 25790643Sbenno * PVO calls. 25877957Sbenno */ 259152180Sgrehanstatic int moea_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, 26090643Sbenno vm_offset_t, vm_offset_t, u_int, int); 261152180Sgrehanstatic void moea_pvo_remove(struct pvo_entry *, int); 262152180Sgrehanstatic struct pvo_entry *moea_pvo_find_va(pmap_t, vm_offset_t, int *); 263152180Sgrehanstatic struct pte *moea_pvo_to_pte(const struct pvo_entry *, int); 26490643Sbenno 26590643Sbenno/* 26690643Sbenno * Utility routines. 26790643Sbenno */ 268159303Salcstatic void moea_enter_locked(pmap_t, vm_offset_t, vm_page_t, 269159303Salc vm_prot_t, boolean_t); 270152180Sgrehanstatic void moea_syncicache(vm_offset_t, vm_size_t); 271152180Sgrehanstatic boolean_t moea_query_bit(vm_page_t, int); 272208990Salcstatic u_int moea_clear_bit(vm_page_t, int); 273152180Sgrehanstatic void moea_kremove(mmu_t, vm_offset_t); 274152180Sgrehanint moea_pte_spill(vm_offset_t); 27590643Sbenno 276152180Sgrehan/* 277152180Sgrehan * Kernel MMU interface 278152180Sgrehan */ 279152180Sgrehanvoid moea_change_wiring(mmu_t, pmap_t, vm_offset_t, boolean_t); 280152180Sgrehanvoid moea_clear_modify(mmu_t, vm_page_t); 281152180Sgrehanvoid moea_clear_reference(mmu_t, vm_page_t); 282152180Sgrehanvoid moea_copy_page(mmu_t, vm_page_t, vm_page_t); 283248280Skibvoid moea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 284248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize); 285152180Sgrehanvoid moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); 286159303Salcvoid moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 287159303Salc vm_prot_t); 288159627Supsvoid moea_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 289152180Sgrehanvm_paddr_t moea_extract(mmu_t, pmap_t, vm_offset_t); 290152180Sgrehanvm_page_t moea_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t); 291152180Sgrehanvoid moea_init(mmu_t); 292152180Sgrehanboolean_t moea_is_modified(mmu_t, vm_page_t); 293214617Salcboolean_t moea_is_prefaultable(mmu_t, pmap_t, vm_offset_t); 294207155Salcboolean_t moea_is_referenced(mmu_t, vm_page_t); 295238357Salcint moea_ts_referenced(mmu_t, vm_page_t); 296235936Srajvm_offset_t moea_map(mmu_t, vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 297152180Sgrehanboolean_t moea_page_exists_quick(mmu_t, pmap_t, vm_page_t); 298173708Salcint moea_page_wired_mappings(mmu_t, vm_page_t); 299152180Sgrehanvoid moea_pinit(mmu_t, pmap_t); 300152180Sgrehanvoid moea_pinit0(mmu_t, pmap_t); 301152180Sgrehanvoid moea_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 302152180Sgrehanvoid moea_qenter(mmu_t, vm_offset_t, vm_page_t *, int); 303152180Sgrehanvoid moea_qremove(mmu_t, vm_offset_t, int); 304152180Sgrehanvoid moea_release(mmu_t, pmap_t); 305152180Sgrehanvoid moea_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t); 306152180Sgrehanvoid moea_remove_all(mmu_t, vm_page_t); 307160889Salcvoid moea_remove_write(mmu_t, vm_page_t); 308152180Sgrehanvoid moea_zero_page(mmu_t, vm_page_t); 309152180Sgrehanvoid moea_zero_page_area(mmu_t, vm_page_t, int, int); 310152180Sgrehanvoid moea_zero_page_idle(mmu_t, vm_page_t); 311152180Sgrehanvoid moea_activate(mmu_t, struct thread *); 312152180Sgrehanvoid moea_deactivate(mmu_t, struct thread *); 313190681Snwhitehornvoid moea_cpu_bootstrap(mmu_t, int); 314152180Sgrehanvoid moea_bootstrap(mmu_t, vm_offset_t, vm_offset_t); 315235936Srajvoid *moea_mapdev(mmu_t, vm_paddr_t, vm_size_t); 316213307Snwhitehornvoid *moea_mapdev_attr(mmu_t, vm_offset_t, vm_size_t, vm_memattr_t); 317152180Sgrehanvoid moea_unmapdev(mmu_t, vm_offset_t, vm_size_t); 318235936Srajvm_paddr_t moea_kextract(mmu_t, vm_offset_t); 319213307Snwhitehornvoid moea_kenter_attr(mmu_t, vm_offset_t, vm_offset_t, vm_memattr_t); 320235936Srajvoid moea_kenter(mmu_t, vm_offset_t, vm_paddr_t); 321213307Snwhitehornvoid moea_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma); 322235936Srajboolean_t moea_dev_direct_mapped(mmu_t, vm_paddr_t, vm_size_t); 323198341Smarcelstatic void moea_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t); 324249864Sjhibbitsvm_offset_t moea_dumpsys_map(mmu_t mmu, struct pmap_md *md, vm_size_t ofs, 325249864Sjhibbits vm_size_t *sz); 326249864Sjhibbitsstruct pmap_md * moea_scan_md(mmu_t mmu, struct pmap_md *prev); 327152180Sgrehan 328152180Sgrehanstatic mmu_method_t moea_methods[] = { 329152180Sgrehan MMUMETHOD(mmu_change_wiring, moea_change_wiring), 330152180Sgrehan MMUMETHOD(mmu_clear_modify, moea_clear_modify), 331152180Sgrehan MMUMETHOD(mmu_clear_reference, moea_clear_reference), 332152180Sgrehan MMUMETHOD(mmu_copy_page, moea_copy_page), 333248280Skib MMUMETHOD(mmu_copy_pages, moea_copy_pages), 334152180Sgrehan MMUMETHOD(mmu_enter, moea_enter), 335159303Salc MMUMETHOD(mmu_enter_object, moea_enter_object), 336152180Sgrehan MMUMETHOD(mmu_enter_quick, moea_enter_quick), 337152180Sgrehan MMUMETHOD(mmu_extract, moea_extract), 338152180Sgrehan MMUMETHOD(mmu_extract_and_hold, moea_extract_and_hold), 339152180Sgrehan MMUMETHOD(mmu_init, moea_init), 340152180Sgrehan MMUMETHOD(mmu_is_modified, moea_is_modified), 341214617Salc MMUMETHOD(mmu_is_prefaultable, moea_is_prefaultable), 342207155Salc MMUMETHOD(mmu_is_referenced, moea_is_referenced), 343152180Sgrehan MMUMETHOD(mmu_ts_referenced, moea_ts_referenced), 344152180Sgrehan MMUMETHOD(mmu_map, moea_map), 345152180Sgrehan MMUMETHOD(mmu_page_exists_quick,moea_page_exists_quick), 346173708Salc MMUMETHOD(mmu_page_wired_mappings,moea_page_wired_mappings), 347152180Sgrehan MMUMETHOD(mmu_pinit, moea_pinit), 348152180Sgrehan MMUMETHOD(mmu_pinit0, moea_pinit0), 349152180Sgrehan MMUMETHOD(mmu_protect, moea_protect), 350152180Sgrehan MMUMETHOD(mmu_qenter, moea_qenter), 351152180Sgrehan MMUMETHOD(mmu_qremove, moea_qremove), 352152180Sgrehan MMUMETHOD(mmu_release, moea_release), 353152180Sgrehan MMUMETHOD(mmu_remove, moea_remove), 354152180Sgrehan MMUMETHOD(mmu_remove_all, moea_remove_all), 355160889Salc MMUMETHOD(mmu_remove_write, moea_remove_write), 356198341Smarcel MMUMETHOD(mmu_sync_icache, moea_sync_icache), 357152180Sgrehan MMUMETHOD(mmu_zero_page, moea_zero_page), 358152180Sgrehan MMUMETHOD(mmu_zero_page_area, moea_zero_page_area), 359152180Sgrehan MMUMETHOD(mmu_zero_page_idle, moea_zero_page_idle), 360152180Sgrehan MMUMETHOD(mmu_activate, moea_activate), 361152180Sgrehan MMUMETHOD(mmu_deactivate, moea_deactivate), 362213307Snwhitehorn MMUMETHOD(mmu_page_set_memattr, moea_page_set_memattr), 363152180Sgrehan 364152180Sgrehan /* Internal interfaces */ 365152180Sgrehan MMUMETHOD(mmu_bootstrap, moea_bootstrap), 366190681Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, moea_cpu_bootstrap), 367213307Snwhitehorn MMUMETHOD(mmu_mapdev_attr, moea_mapdev_attr), 368152180Sgrehan MMUMETHOD(mmu_mapdev, moea_mapdev), 369152180Sgrehan MMUMETHOD(mmu_unmapdev, moea_unmapdev), 370152180Sgrehan MMUMETHOD(mmu_kextract, moea_kextract), 371152180Sgrehan MMUMETHOD(mmu_kenter, moea_kenter), 372213307Snwhitehorn MMUMETHOD(mmu_kenter_attr, moea_kenter_attr), 373152180Sgrehan MMUMETHOD(mmu_dev_direct_mapped,moea_dev_direct_mapped), 374249864Sjhibbits MMUMETHOD(mmu_scan_md, moea_scan_md), 375249864Sjhibbits MMUMETHOD(mmu_dumpsys_map, moea_dumpsys_map), 376152180Sgrehan 377152180Sgrehan { 0, 0 } 378152180Sgrehan}; 379152180Sgrehan 380212627SgrehanMMU_DEF(oea_mmu, MMU_TYPE_OEA, moea_methods, 0); 381152180Sgrehan 382213307Snwhitehornstatic __inline uint32_t 383213307Snwhitehornmoea_calc_wimg(vm_offset_t pa, vm_memattr_t ma) 384213307Snwhitehorn{ 385213307Snwhitehorn uint32_t pte_lo; 386213307Snwhitehorn int i; 387212627Sgrehan 388213307Snwhitehorn if (ma != VM_MEMATTR_DEFAULT) { 389213307Snwhitehorn switch (ma) { 390213307Snwhitehorn case VM_MEMATTR_UNCACHEABLE: 391213307Snwhitehorn return (PTE_I | PTE_G); 392213307Snwhitehorn case VM_MEMATTR_WRITE_COMBINING: 393213307Snwhitehorn case VM_MEMATTR_WRITE_BACK: 394213307Snwhitehorn case VM_MEMATTR_PREFETCHABLE: 395213307Snwhitehorn return (PTE_I); 396213307Snwhitehorn case VM_MEMATTR_WRITE_THROUGH: 397213307Snwhitehorn return (PTE_W | PTE_M); 398213307Snwhitehorn } 399213307Snwhitehorn } 400213307Snwhitehorn 401213307Snwhitehorn /* 402213307Snwhitehorn * Assume the page is cache inhibited and access is guarded unless 403213307Snwhitehorn * it's in our available memory array. 404213307Snwhitehorn */ 405213307Snwhitehorn pte_lo = PTE_I | PTE_G; 406213307Snwhitehorn for (i = 0; i < pregions_sz; i++) { 407213307Snwhitehorn if ((pa >= pregions[i].mr_start) && 408213307Snwhitehorn (pa < (pregions[i].mr_start + pregions[i].mr_size))) { 409213307Snwhitehorn pte_lo = PTE_M; 410213307Snwhitehorn break; 411213307Snwhitehorn } 412213307Snwhitehorn } 413213307Snwhitehorn 414213307Snwhitehorn return pte_lo; 415213307Snwhitehorn} 416213307Snwhitehorn 417183094Smarcelstatic void 418183094Smarceltlbie(vm_offset_t va) 419183094Smarcel{ 420152180Sgrehan 421183094Smarcel mtx_lock_spin(&tlbie_mtx); 422213407Snwhitehorn __asm __volatile("ptesync"); 423183094Smarcel __asm __volatile("tlbie %0" :: "r"(va)); 424213407Snwhitehorn __asm __volatile("eieio; tlbsync; ptesync"); 425183094Smarcel mtx_unlock_spin(&tlbie_mtx); 426183094Smarcel} 427183094Smarcel 428183094Smarcelstatic void 429183094Smarceltlbia(void) 430183094Smarcel{ 431183094Smarcel vm_offset_t va; 432183094Smarcel 433183094Smarcel for (va = 0; va < 0x00040000; va += 0x00001000) { 434183094Smarcel __asm __volatile("tlbie %0" :: "r"(va)); 435183094Smarcel powerpc_sync(); 436183094Smarcel } 437183094Smarcel __asm __volatile("tlbsync"); 438183094Smarcel powerpc_sync(); 439183094Smarcel} 440183094Smarcel 44190643Sbennostatic __inline int 44290643Sbennova_to_sr(u_int *sr, vm_offset_t va) 44377957Sbenno{ 44490643Sbenno return (sr[(uintptr_t)va >> ADDR_SR_SHFT]); 44590643Sbenno} 44677957Sbenno 44790643Sbennostatic __inline u_int 44890643Sbennova_to_pteg(u_int sr, vm_offset_t addr) 44990643Sbenno{ 45090643Sbenno u_int hash; 45190643Sbenno 45290643Sbenno hash = (sr & SR_VSID_MASK) ^ (((u_int)addr & ADDR_PIDX) >> 45390643Sbenno ADDR_PIDX_SHFT); 454152180Sgrehan return (hash & moea_pteg_mask); 45577957Sbenno} 45677957Sbenno 45790643Sbennostatic __inline struct pvo_head * 45890643Sbennovm_page_to_pvoh(vm_page_t m) 45990643Sbenno{ 46090643Sbenno 46190643Sbenno return (&m->md.mdpg_pvoh); 46290643Sbenno} 46390643Sbenno 46477957Sbennostatic __inline void 465152180Sgrehanmoea_attr_clear(vm_page_t m, int ptebit) 46677957Sbenno{ 46790643Sbenno 468238159Salc rw_assert(&pvh_global_lock, RA_WLOCKED); 46990643Sbenno m->md.mdpg_attrs &= ~ptebit; 47077957Sbenno} 47177957Sbenno 47277957Sbennostatic __inline int 473152180Sgrehanmoea_attr_fetch(vm_page_t m) 47477957Sbenno{ 47577957Sbenno 47690643Sbenno return (m->md.mdpg_attrs); 47777957Sbenno} 47877957Sbenno 47990643Sbennostatic __inline void 480152180Sgrehanmoea_attr_save(vm_page_t m, int ptebit) 48190643Sbenno{ 48290643Sbenno 483238159Salc rw_assert(&pvh_global_lock, RA_WLOCKED); 48490643Sbenno m->md.mdpg_attrs |= ptebit; 48590643Sbenno} 48690643Sbenno 48777957Sbennostatic __inline int 488152180Sgrehanmoea_pte_compare(const struct pte *pt, const struct pte *pvo_pt) 48977957Sbenno{ 49090643Sbenno if (pt->pte_hi == pvo_pt->pte_hi) 49190643Sbenno return (1); 49290643Sbenno 49390643Sbenno return (0); 49477957Sbenno} 49577957Sbenno 49677957Sbennostatic __inline int 497152180Sgrehanmoea_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) 49877957Sbenno{ 49990643Sbenno return (pt->pte_hi & ~PTE_VALID) == 50090643Sbenno (((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 50190643Sbenno ((va >> ADDR_API_SHFT) & PTE_API) | which); 50290643Sbenno} 50377957Sbenno 50490643Sbennostatic __inline void 505152180Sgrehanmoea_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) 50690643Sbenno{ 507159928Salc 508159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 509159928Salc 51090643Sbenno /* 51190643Sbenno * Construct a PTE. Default to IMB initially. Valid bit only gets 51290643Sbenno * set when the real pte is set in memory. 51390643Sbenno * 51490643Sbenno * Note: Don't set the valid bit for correct operation of tlb update. 51590643Sbenno */ 51690643Sbenno pt->pte_hi = ((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 51790643Sbenno (((va & ADDR_PIDX) >> ADDR_API_SHFT) & PTE_API); 51890643Sbenno pt->pte_lo = pte_lo; 51977957Sbenno} 52077957Sbenno 52190643Sbennostatic __inline void 522152180Sgrehanmoea_pte_synch(struct pte *pt, struct pte *pvo_pt) 52377957Sbenno{ 52477957Sbenno 525159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 52690643Sbenno pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); 52777957Sbenno} 52877957Sbenno 52990643Sbennostatic __inline void 530152180Sgrehanmoea_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) 53177957Sbenno{ 53277957Sbenno 533159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 534159928Salc 53590643Sbenno /* 53690643Sbenno * As shown in Section 7.6.3.2.3 53790643Sbenno */ 53890643Sbenno pt->pte_lo &= ~ptebit; 539183094Smarcel tlbie(va); 54077957Sbenno} 54177957Sbenno 54290643Sbennostatic __inline void 543152180Sgrehanmoea_pte_set(struct pte *pt, struct pte *pvo_pt) 54477957Sbenno{ 54577957Sbenno 546159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 54790643Sbenno pvo_pt->pte_hi |= PTE_VALID; 54890643Sbenno 54977957Sbenno /* 55090643Sbenno * Update the PTE as defined in section 7.6.3.1. 551253976Sjhibbits * Note that the REF/CHG bits are from pvo_pt and thus should have 55290643Sbenno * been saved so this routine can restore them (if desired). 55377957Sbenno */ 55490643Sbenno pt->pte_lo = pvo_pt->pte_lo; 555183094Smarcel powerpc_sync(); 55690643Sbenno pt->pte_hi = pvo_pt->pte_hi; 557183094Smarcel powerpc_sync(); 558152180Sgrehan moea_pte_valid++; 55990643Sbenno} 56077957Sbenno 56190643Sbennostatic __inline void 562152180Sgrehanmoea_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 56390643Sbenno{ 56490643Sbenno 565159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 56690643Sbenno pvo_pt->pte_hi &= ~PTE_VALID; 56790643Sbenno 56877957Sbenno /* 56990643Sbenno * Force the reg & chg bits back into the PTEs. 57077957Sbenno */ 571183094Smarcel powerpc_sync(); 57277957Sbenno 57390643Sbenno /* 57490643Sbenno * Invalidate the pte. 57590643Sbenno */ 57690643Sbenno pt->pte_hi &= ~PTE_VALID; 57777957Sbenno 578183094Smarcel tlbie(va); 57977957Sbenno 58090643Sbenno /* 58190643Sbenno * Save the reg & chg bits. 58290643Sbenno */ 583152180Sgrehan moea_pte_synch(pt, pvo_pt); 584152180Sgrehan moea_pte_valid--; 58577957Sbenno} 58677957Sbenno 58790643Sbennostatic __inline void 588152180Sgrehanmoea_pte_change(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 58990643Sbenno{ 59090643Sbenno 59190643Sbenno /* 59290643Sbenno * Invalidate the PTE 59390643Sbenno */ 594152180Sgrehan moea_pte_unset(pt, pvo_pt, va); 595152180Sgrehan moea_pte_set(pt, pvo_pt); 59690643Sbenno} 59790643Sbenno 59877957Sbenno/* 59990643Sbenno * Quick sort callout for comparing memory regions. 60077957Sbenno */ 60190643Sbennostatic int om_cmp(const void *a, const void *b); 60290643Sbenno 60390643Sbennostatic int 60490643Sbennoom_cmp(const void *a, const void *b) 60590643Sbenno{ 60690643Sbenno const struct ofw_map *mapa; 60790643Sbenno const struct ofw_map *mapb; 60890643Sbenno 60990643Sbenno mapa = a; 61090643Sbenno mapb = b; 61190643Sbenno if (mapa->om_pa < mapb->om_pa) 61290643Sbenno return (-1); 61390643Sbenno else if (mapa->om_pa > mapb->om_pa) 61490643Sbenno return (1); 61590643Sbenno else 61690643Sbenno return (0); 61777957Sbenno} 61877957Sbenno 61977957Sbennovoid 620190681Snwhitehornmoea_cpu_bootstrap(mmu_t mmup, int ap) 621178628Smarcel{ 622178628Smarcel u_int sdr; 623178628Smarcel int i; 624178628Smarcel 625178628Smarcel if (ap) { 626183094Smarcel powerpc_sync(); 627178628Smarcel __asm __volatile("mtdbatu 0,%0" :: "r"(battable[0].batu)); 628178628Smarcel __asm __volatile("mtdbatl 0,%0" :: "r"(battable[0].batl)); 629178628Smarcel isync(); 630178628Smarcel __asm __volatile("mtibatu 0,%0" :: "r"(battable[0].batu)); 631178628Smarcel __asm __volatile("mtibatl 0,%0" :: "r"(battable[0].batl)); 632178628Smarcel isync(); 633178628Smarcel } 634178628Smarcel 635243370Sadrian#ifdef WII 636243370Sadrian /* 637243370Sadrian * Special case for the Wii: don't install the PCI BAT. 638243370Sadrian */ 639243370Sadrian if (strcmp(installed_platform(), "wii") != 0) { 640243370Sadrian#endif 641243370Sadrian __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); 642243370Sadrian __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); 643243370Sadrian#ifdef WII 644243370Sadrian } 645243370Sadrian#endif 646178629Smarcel isync(); 647178628Smarcel 648178629Smarcel __asm __volatile("mtibatu 1,%0" :: "r"(0)); 649178629Smarcel __asm __volatile("mtdbatu 2,%0" :: "r"(0)); 650178629Smarcel __asm __volatile("mtibatu 2,%0" :: "r"(0)); 651178629Smarcel __asm __volatile("mtdbatu 3,%0" :: "r"(0)); 652178629Smarcel __asm __volatile("mtibatu 3,%0" :: "r"(0)); 653178628Smarcel isync(); 654178628Smarcel 655178628Smarcel for (i = 0; i < 16; i++) 656215163Snwhitehorn mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); 657183094Smarcel powerpc_sync(); 658178628Smarcel 659178628Smarcel sdr = (u_int)moea_pteg_table | (moea_pteg_mask >> 10); 660178628Smarcel __asm __volatile("mtsdr1 %0" :: "r"(sdr)); 661178628Smarcel isync(); 662178628Smarcel 663179254Smarcel tlbia(); 664178628Smarcel} 665178628Smarcel 666178628Smarcelvoid 667152180Sgrehanmoea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 66877957Sbenno{ 66997346Sbenno ihandle_t mmui; 67090643Sbenno phandle_t chosen, mmu; 67190643Sbenno int sz; 67290643Sbenno int i, j; 673143200Sgrehan vm_size_t size, physsz, hwphyssz; 67490643Sbenno vm_offset_t pa, va, off; 675194784Sjeff void *dpcpu; 676209369Snwhitehorn register_t msr; 67777957Sbenno 67899037Sbenno /* 679103604Sgrehan * Set up BAT0 to map the lowest 256 MB area 68099037Sbenno */ 68199037Sbenno battable[0x0].batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 68299037Sbenno battable[0x0].batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 68399037Sbenno 684243370Sadrian /* 685243370Sadrian * Map PCI memory space. 686243370Sadrian */ 687243370Sadrian battable[0x8].batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 688243370Sadrian battable[0x8].batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 68999037Sbenno 690243370Sadrian battable[0x9].batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 691243370Sadrian battable[0x9].batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 69299037Sbenno 693243370Sadrian battable[0xa].batl = BATL(0xa0000000, BAT_I|BAT_G, BAT_PP_RW); 694243370Sadrian battable[0xa].batu = BATU(0xa0000000, BAT_BL_256M, BAT_Vs); 69599037Sbenno 696243370Sadrian battable[0xb].batl = BATL(0xb0000000, BAT_I|BAT_G, BAT_PP_RW); 697243370Sadrian battable[0xb].batu = BATU(0xb0000000, BAT_BL_256M, BAT_Vs); 69899037Sbenno 699243370Sadrian /* 700243370Sadrian * Map obio devices. 701243370Sadrian */ 702243370Sadrian battable[0xf].batl = BATL(0xf0000000, BAT_I|BAT_G, BAT_PP_RW); 703243370Sadrian battable[0xf].batu = BATU(0xf0000000, BAT_BL_256M, BAT_Vs); 70499037Sbenno 70577957Sbenno /* 70690643Sbenno * Use an IBAT and a DBAT to map the bottom segment of memory 707209369Snwhitehorn * where we are. Turn off instruction relocation temporarily 708209369Snwhitehorn * to prevent faults while reprogramming the IBAT. 70977957Sbenno */ 710209369Snwhitehorn msr = mfmsr(); 711209369Snwhitehorn mtmsr(msr & ~PSL_IR); 712152180Sgrehan __asm (".balign 32; \n" 713149958Sgrehan "mtibatu 0,%0; mtibatl 0,%1; isync; \n" 714131808Sgrehan "mtdbatu 0,%0; mtdbatl 0,%1; isync" 715178628Smarcel :: "r"(battable[0].batu), "r"(battable[0].batl)); 716209369Snwhitehorn mtmsr(msr); 71799037Sbenno 718243370Sadrian#ifdef WII 719243370Sadrian if (strcmp(installed_platform(), "wii") != 0) { 720243370Sadrian#endif 721243370Sadrian /* map pci space */ 722243370Sadrian __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); 723243370Sadrian __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); 724243370Sadrian#ifdef WII 725243370Sadrian } 726243370Sadrian#endif 727178628Smarcel isync(); 72877957Sbenno 729190681Snwhitehorn /* set global direct map flag */ 730190681Snwhitehorn hw_direct_map = 1; 731190681Snwhitehorn 73297346Sbenno mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 733152180Sgrehan CTR0(KTR_PMAP, "moea_bootstrap: physical memory"); 73497346Sbenno 73597346Sbenno for (i = 0; i < pregions_sz; i++) { 736103604Sgrehan vm_offset_t pa; 737103604Sgrehan vm_offset_t end; 738103604Sgrehan 73997346Sbenno CTR3(KTR_PMAP, "physregion: %#x - %#x (%#x)", 74097346Sbenno pregions[i].mr_start, 74197346Sbenno pregions[i].mr_start + pregions[i].mr_size, 74297346Sbenno pregions[i].mr_size); 743103604Sgrehan /* 744103604Sgrehan * Install entries into the BAT table to allow all 745103604Sgrehan * of physmem to be convered by on-demand BAT entries. 746103604Sgrehan * The loop will sometimes set the same battable element 747103604Sgrehan * twice, but that's fine since they won't be used for 748103604Sgrehan * a while yet. 749103604Sgrehan */ 750103604Sgrehan pa = pregions[i].mr_start & 0xf0000000; 751103604Sgrehan end = pregions[i].mr_start + pregions[i].mr_size; 752103604Sgrehan do { 753103604Sgrehan u_int n = pa >> ADDR_SR_SHFT; 754152180Sgrehan 755103604Sgrehan battable[n].batl = BATL(pa, BAT_M, BAT_PP_RW); 756103604Sgrehan battable[n].batu = BATU(pa, BAT_BL_256M, BAT_Vs); 757103604Sgrehan pa += SEGMENT_LENGTH; 758103604Sgrehan } while (pa < end); 75997346Sbenno } 76097346Sbenno 76197346Sbenno if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) 762152180Sgrehan panic("moea_bootstrap: phys_avail too small"); 763222614Snwhitehorn 76490643Sbenno phys_avail_count = 0; 76591793Sbenno physsz = 0; 766143234Sgrehan hwphyssz = 0; 767143234Sgrehan TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 76897346Sbenno for (i = 0, j = 0; i < regions_sz; i++, j += 2) { 76990643Sbenno CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start, 77090643Sbenno regions[i].mr_start + regions[i].mr_size, 77190643Sbenno regions[i].mr_size); 772143200Sgrehan if (hwphyssz != 0 && 773143200Sgrehan (physsz + regions[i].mr_size) >= hwphyssz) { 774143200Sgrehan if (physsz < hwphyssz) { 775143200Sgrehan phys_avail[j] = regions[i].mr_start; 776143200Sgrehan phys_avail[j + 1] = regions[i].mr_start + 777143200Sgrehan hwphyssz - physsz; 778143200Sgrehan physsz = hwphyssz; 779143200Sgrehan phys_avail_count++; 780143200Sgrehan } 781143200Sgrehan break; 782143200Sgrehan } 78390643Sbenno phys_avail[j] = regions[i].mr_start; 78490643Sbenno phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 78590643Sbenno phys_avail_count++; 78691793Sbenno physsz += regions[i].mr_size; 78777957Sbenno } 788228609Snwhitehorn 789228609Snwhitehorn /* Check for overlap with the kernel and exception vectors */ 790228609Snwhitehorn for (j = 0; j < 2*phys_avail_count; j+=2) { 791228609Snwhitehorn if (phys_avail[j] < EXC_LAST) 792228609Snwhitehorn phys_avail[j] += EXC_LAST; 793228609Snwhitehorn 794228609Snwhitehorn if (kernelstart >= phys_avail[j] && 795228609Snwhitehorn kernelstart < phys_avail[j+1]) { 796228609Snwhitehorn if (kernelend < phys_avail[j+1]) { 797228609Snwhitehorn phys_avail[2*phys_avail_count] = 798228609Snwhitehorn (kernelend & ~PAGE_MASK) + PAGE_SIZE; 799228609Snwhitehorn phys_avail[2*phys_avail_count + 1] = 800228609Snwhitehorn phys_avail[j+1]; 801228609Snwhitehorn phys_avail_count++; 802228609Snwhitehorn } 803228609Snwhitehorn 804228609Snwhitehorn phys_avail[j+1] = kernelstart & ~PAGE_MASK; 805228609Snwhitehorn } 806228609Snwhitehorn 807228609Snwhitehorn if (kernelend >= phys_avail[j] && 808228609Snwhitehorn kernelend < phys_avail[j+1]) { 809228609Snwhitehorn if (kernelstart > phys_avail[j]) { 810228609Snwhitehorn phys_avail[2*phys_avail_count] = phys_avail[j]; 811228609Snwhitehorn phys_avail[2*phys_avail_count + 1] = 812228609Snwhitehorn kernelstart & ~PAGE_MASK; 813228609Snwhitehorn phys_avail_count++; 814228609Snwhitehorn } 815228609Snwhitehorn 816228609Snwhitehorn phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE; 817228609Snwhitehorn } 818228609Snwhitehorn } 819228609Snwhitehorn 82091793Sbenno physmem = btoc(physsz); 82177957Sbenno 82277957Sbenno /* 82390643Sbenno * Allocate PTEG table. 82477957Sbenno */ 82590643Sbenno#ifdef PTEGCOUNT 826152180Sgrehan moea_pteg_count = PTEGCOUNT; 82790643Sbenno#else 828152180Sgrehan moea_pteg_count = 0x1000; 82977957Sbenno 830152180Sgrehan while (moea_pteg_count < physmem) 831152180Sgrehan moea_pteg_count <<= 1; 83277957Sbenno 833152180Sgrehan moea_pteg_count >>= 1; 83490643Sbenno#endif /* PTEGCOUNT */ 83577957Sbenno 836152180Sgrehan size = moea_pteg_count * sizeof(struct pteg); 837152180Sgrehan CTR2(KTR_PMAP, "moea_bootstrap: %d PTEGs, %d bytes", moea_pteg_count, 83890643Sbenno size); 839152180Sgrehan moea_pteg_table = (struct pteg *)moea_bootstrap_alloc(size, size); 840152180Sgrehan CTR1(KTR_PMAP, "moea_bootstrap: PTEG table at %p", moea_pteg_table); 841152180Sgrehan bzero((void *)moea_pteg_table, moea_pteg_count * sizeof(struct pteg)); 842152180Sgrehan moea_pteg_mask = moea_pteg_count - 1; 84377957Sbenno 84490643Sbenno /* 84594839Sbenno * Allocate pv/overflow lists. 84690643Sbenno */ 847152180Sgrehan size = sizeof(struct pvo_head) * moea_pteg_count; 848152180Sgrehan moea_pvo_table = (struct pvo_head *)moea_bootstrap_alloc(size, 84990643Sbenno PAGE_SIZE); 850152180Sgrehan CTR1(KTR_PMAP, "moea_bootstrap: PVO table at %p", moea_pvo_table); 851152180Sgrehan for (i = 0; i < moea_pteg_count; i++) 852152180Sgrehan LIST_INIT(&moea_pvo_table[i]); 85377957Sbenno 85490643Sbenno /* 855134535Salc * Initialize the lock that synchronizes access to the pteg and pvo 856134535Salc * tables. 857134535Salc */ 858159928Salc mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF | 859159928Salc MTX_RECURSE); 860212278Snwhitehorn mtx_init(&moea_vsid_mutex, "VSID table", NULL, MTX_DEF); 861134535Salc 862183094Smarcel mtx_init(&tlbie_mtx, "tlbie", NULL, MTX_SPIN); 863183094Smarcel 864134535Salc /* 86590643Sbenno * Initialise the unmanaged pvo pool. 86690643Sbenno */ 867152180Sgrehan moea_bpvo_pool = (struct pvo_entry *)moea_bootstrap_alloc( 86899037Sbenno BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); 869152180Sgrehan moea_bpvo_pool_index = 0; 87077957Sbenno 87177957Sbenno /* 87290643Sbenno * Make sure kernel vsid is allocated as well as VSID 0. 87377957Sbenno */ 874152180Sgrehan moea_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW] 87590643Sbenno |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 876152180Sgrehan moea_vsid_bitmap[0] |= 1; 87777957Sbenno 87890643Sbenno /* 879215163Snwhitehorn * Initialize the kernel pmap (which is statically allocated). 88090643Sbenno */ 881215163Snwhitehorn PMAP_LOCK_INIT(kernel_pmap); 882215163Snwhitehorn for (i = 0; i < 16; i++) 883215163Snwhitehorn kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i; 884222813Sattilio CPU_FILL(&kernel_pmap->pm_active); 885235689Snwhitehorn RB_INIT(&kernel_pmap->pmap_pvo); 886215163Snwhitehorn 887238159Salc /* 888238159Salc * Initialize the global pv list lock. 889238159Salc */ 890238159Salc rw_init(&pvh_global_lock, "pmap pv global"); 891238159Salc 892215163Snwhitehorn /* 893215163Snwhitehorn * Set up the Open Firmware mappings 894215163Snwhitehorn */ 895228609Snwhitehorn chosen = OF_finddevice("/chosen"); 896228609Snwhitehorn if (chosen != -1 && OF_getprop(chosen, "mmu", &mmui, 4) != -1 && 897228609Snwhitehorn (mmu = OF_instance_to_package(mmui)) != -1 && 898228609Snwhitehorn (sz = OF_getproplen(mmu, "translations")) != -1) { 899228609Snwhitehorn translations = NULL; 900228609Snwhitehorn for (i = 0; phys_avail[i] != 0; i += 2) { 901228609Snwhitehorn if (phys_avail[i + 1] >= sz) { 902228609Snwhitehorn translations = (struct ofw_map *)phys_avail[i]; 903228609Snwhitehorn break; 904228609Snwhitehorn } 905131401Sgrehan } 906228609Snwhitehorn if (translations == NULL) 907228609Snwhitehorn panic("moea_bootstrap: no space to copy translations"); 908228609Snwhitehorn bzero(translations, sz); 909228609Snwhitehorn if (OF_getprop(mmu, "translations", translations, sz) == -1) 910228609Snwhitehorn panic("moea_bootstrap: can't get ofw translations"); 911228609Snwhitehorn CTR0(KTR_PMAP, "moea_bootstrap: translations"); 912228609Snwhitehorn sz /= sizeof(*translations); 913228609Snwhitehorn qsort(translations, sz, sizeof (*translations), om_cmp); 914228609Snwhitehorn for (i = 0; i < sz; i++) { 915228609Snwhitehorn CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x", 916228609Snwhitehorn translations[i].om_pa, translations[i].om_va, 917228609Snwhitehorn translations[i].om_len); 91877957Sbenno 919228609Snwhitehorn /* 920228609Snwhitehorn * If the mapping is 1:1, let the RAM and device 921228609Snwhitehorn * on-demand BAT tables take care of the translation. 922228609Snwhitehorn */ 923228609Snwhitehorn if (translations[i].om_va == translations[i].om_pa) 924228609Snwhitehorn continue; 92577957Sbenno 926228609Snwhitehorn /* Enter the pages */ 927228609Snwhitehorn for (off = 0; off < translations[i].om_len; 928228609Snwhitehorn off += PAGE_SIZE) 929228609Snwhitehorn moea_kenter(mmup, translations[i].om_va + off, 930228609Snwhitehorn translations[i].om_pa + off); 931228609Snwhitehorn } 93277957Sbenno } 93377957Sbenno 93490643Sbenno /* 935178261Smarcel * Calculate the last available physical address. 936178261Smarcel */ 937178261Smarcel for (i = 0; phys_avail[i + 2] != 0; i += 2) 938178261Smarcel ; 939178261Smarcel Maxmem = powerpc_btop(phys_avail[i + 1]); 940178261Smarcel 941190681Snwhitehorn moea_cpu_bootstrap(mmup,0); 94277957Sbenno 94390643Sbenno pmap_bootstrapped++; 944178261Smarcel 945178261Smarcel /* 946178261Smarcel * Set the start and end of kva. 947178261Smarcel */ 948178261Smarcel virtual_avail = VM_MIN_KERNEL_ADDRESS; 949204128Snwhitehorn virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 950178261Smarcel 951178261Smarcel /* 952178261Smarcel * Allocate a kernel stack with a guard page for thread0 and map it 953178261Smarcel * into the kernel page map. 954178261Smarcel */ 955178261Smarcel pa = moea_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, PAGE_SIZE); 956178261Smarcel va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 957178261Smarcel virtual_avail = va + KSTACK_PAGES * PAGE_SIZE; 958178261Smarcel CTR2(KTR_PMAP, "moea_bootstrap: kstack0 at %#x (%#x)", pa, va); 959178261Smarcel thread0.td_kstack = va; 960178261Smarcel thread0.td_kstack_pages = KSTACK_PAGES; 961178261Smarcel for (i = 0; i < KSTACK_PAGES; i++) { 962201758Smbr moea_kenter(mmup, va, pa); 963178261Smarcel pa += PAGE_SIZE; 964178261Smarcel va += PAGE_SIZE; 965178261Smarcel } 966178261Smarcel 967178261Smarcel /* 968178261Smarcel * Allocate virtual address space for the message buffer. 969178261Smarcel */ 970217688Spluknet pa = msgbuf_phys = moea_bootstrap_alloc(msgbufsize, PAGE_SIZE); 971178261Smarcel msgbufp = (struct msgbuf *)virtual_avail; 972178261Smarcel va = virtual_avail; 973217688Spluknet virtual_avail += round_page(msgbufsize); 974178261Smarcel while (va < virtual_avail) { 975201758Smbr moea_kenter(mmup, va, pa); 976178261Smarcel pa += PAGE_SIZE; 977178261Smarcel va += PAGE_SIZE; 978178261Smarcel } 979194784Sjeff 980194784Sjeff /* 981194784Sjeff * Allocate virtual address space for the dynamic percpu area. 982194784Sjeff */ 983194784Sjeff pa = moea_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE); 984194784Sjeff dpcpu = (void *)virtual_avail; 985194784Sjeff va = virtual_avail; 986194784Sjeff virtual_avail += DPCPU_SIZE; 987194784Sjeff while (va < virtual_avail) { 988201758Smbr moea_kenter(mmup, va, pa); 989194784Sjeff pa += PAGE_SIZE; 990194784Sjeff va += PAGE_SIZE; 991194784Sjeff } 992194784Sjeff dpcpu_init(dpcpu, 0); 99377957Sbenno} 99477957Sbenno 99577957Sbenno/* 99690643Sbenno * Activate a user pmap. The pmap must be activated before it's address 99790643Sbenno * space can be accessed in any way. 99877957Sbenno */ 99977957Sbennovoid 1000152180Sgrehanmoea_activate(mmu_t mmu, struct thread *td) 100177957Sbenno{ 100296250Sbenno pmap_t pm, pmr; 100377957Sbenno 100477957Sbenno /* 1005103604Sgrehan * Load all the data we need up front to encourage the compiler to 100690643Sbenno * not issue any loads while we have interrupts disabled below. 100777957Sbenno */ 100890643Sbenno pm = &td->td_proc->p_vmspace->vm_pmap; 1009183290Snwhitehorn pmr = pm->pmap_phys; 101077957Sbenno 1011223758Sattilio CPU_SET(PCPU_GET(cpuid), &pm->pm_active); 101296250Sbenno PCPU_SET(curpmap, pmr); 101377957Sbenno} 101477957Sbenno 101591483Sbennovoid 1016152180Sgrehanmoea_deactivate(mmu_t mmu, struct thread *td) 101791483Sbenno{ 101891483Sbenno pmap_t pm; 101991483Sbenno 102091483Sbenno pm = &td->td_proc->p_vmspace->vm_pmap; 1021223758Sattilio CPU_CLR(PCPU_GET(cpuid), &pm->pm_active); 102296250Sbenno PCPU_SET(curpmap, NULL); 102391483Sbenno} 102491483Sbenno 102577957Sbennovoid 1026152180Sgrehanmoea_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired) 102777957Sbenno{ 102896353Sbenno struct pvo_entry *pvo; 102996353Sbenno 1030134329Salc PMAP_LOCK(pm); 1031152180Sgrehan pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 103296353Sbenno 103396353Sbenno if (pvo != NULL) { 103496353Sbenno if (wired) { 103596353Sbenno if ((pvo->pvo_vaddr & PVO_WIRED) == 0) 103696353Sbenno pm->pm_stats.wired_count++; 103796353Sbenno pvo->pvo_vaddr |= PVO_WIRED; 103896353Sbenno } else { 103996353Sbenno if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 104096353Sbenno pm->pm_stats.wired_count--; 104196353Sbenno pvo->pvo_vaddr &= ~PVO_WIRED; 104296353Sbenno } 104396353Sbenno } 1044134329Salc PMAP_UNLOCK(pm); 104577957Sbenno} 104677957Sbenno 104777957Sbennovoid 1048152180Sgrehanmoea_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) 104977957Sbenno{ 105097385Sbenno vm_offset_t dst; 105197385Sbenno vm_offset_t src; 105297385Sbenno 105397385Sbenno dst = VM_PAGE_TO_PHYS(mdst); 105497385Sbenno src = VM_PAGE_TO_PHYS(msrc); 105597385Sbenno 1056234156Snwhitehorn bcopy((void *)src, (void *)dst, PAGE_SIZE); 105777957Sbenno} 105877957Sbenno 1059248280Skibvoid 1060248280Skibmoea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, 1061248280Skib vm_page_t *mb, vm_offset_t b_offset, int xfersize) 1062248280Skib{ 1063248280Skib void *a_cp, *b_cp; 1064248280Skib vm_offset_t a_pg_offset, b_pg_offset; 1065248280Skib int cnt; 1066248280Skib 1067248280Skib while (xfersize > 0) { 1068248280Skib a_pg_offset = a_offset & PAGE_MASK; 1069248280Skib cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 1070248280Skib a_cp = (char *)VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]) + 1071248280Skib a_pg_offset; 1072248280Skib b_pg_offset = b_offset & PAGE_MASK; 1073248280Skib cnt = min(cnt, PAGE_SIZE - b_pg_offset); 1074248280Skib b_cp = (char *)VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]) + 1075248280Skib b_pg_offset; 1076248280Skib bcopy(a_cp, b_cp, cnt); 1077248280Skib a_offset += cnt; 1078248280Skib b_offset += cnt; 1079248280Skib xfersize -= cnt; 1080248280Skib } 1081248280Skib} 1082248280Skib 108377957Sbenno/* 108490643Sbenno * Zero a page of physical memory by temporarily mapping it into the tlb. 108577957Sbenno */ 108677957Sbennovoid 1087152180Sgrehanmoea_zero_page(mmu_t mmu, vm_page_t m) 108877957Sbenno{ 108994777Speter vm_offset_t pa = VM_PAGE_TO_PHYS(m); 1090178265Smarcel void *va = (void *)pa; 109177957Sbenno 109290643Sbenno bzero(va, PAGE_SIZE); 109377957Sbenno} 109477957Sbenno 109577957Sbennovoid 1096152180Sgrehanmoea_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) 109777957Sbenno{ 109899666Sbenno vm_offset_t pa = VM_PAGE_TO_PHYS(m); 1099178265Smarcel void *va = (void *)(pa + off); 110099666Sbenno 1101178265Smarcel bzero(va, size); 110277957Sbenno} 110377957Sbenno 110499571Spetervoid 1105152180Sgrehanmoea_zero_page_idle(mmu_t mmu, vm_page_t m) 110699571Speter{ 1107178265Smarcel vm_offset_t pa = VM_PAGE_TO_PHYS(m); 1108178265Smarcel void *va = (void *)pa; 110999571Speter 1110178265Smarcel bzero(va, PAGE_SIZE); 111199571Speter} 111299571Speter 111377957Sbenno/* 111490643Sbenno * Map the given physical page at the specified virtual address in the 111590643Sbenno * target pmap with the protection requested. If specified the page 111690643Sbenno * will be wired down. 111777957Sbenno */ 111877957Sbennovoid 1119152180Sgrehanmoea_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 112090643Sbenno boolean_t wired) 112177957Sbenno{ 1122159303Salc 1123238159Salc rw_wlock(&pvh_global_lock); 1124159303Salc PMAP_LOCK(pmap); 1125159324Salc moea_enter_locked(pmap, va, m, prot, wired); 1126238159Salc rw_wunlock(&pvh_global_lock); 1127159303Salc PMAP_UNLOCK(pmap); 1128159303Salc} 1129159303Salc 1130159303Salc/* 1131159303Salc * Map the given physical page at the specified virtual address in the 1132159303Salc * target pmap with the protection requested. If specified the page 1133159303Salc * will be wired down. 1134159303Salc * 1135159303Salc * The page queues and pmap must be locked. 1136159303Salc */ 1137159303Salcstatic void 1138159303Salcmoea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1139159303Salc boolean_t wired) 1140159303Salc{ 114190643Sbenno struct pvo_head *pvo_head; 114292847Sjeff uma_zone_t zone; 114396250Sbenno vm_page_t pg; 1144233949Snwhitehorn u_int pte_lo, pvo_flags; 114590643Sbenno int error; 114677957Sbenno 1147152180Sgrehan if (!moea_initialized) { 1148152180Sgrehan pvo_head = &moea_pvo_kunmanaged; 1149152180Sgrehan zone = moea_upvo_zone; 115090643Sbenno pvo_flags = 0; 115196250Sbenno pg = NULL; 115290643Sbenno } else { 1153110172Sgrehan pvo_head = vm_page_to_pvoh(m); 1154110172Sgrehan pg = m; 1155152180Sgrehan zone = moea_mpvo_zone; 115690643Sbenno pvo_flags = PVO_MANAGED; 115790643Sbenno } 1158134535Salc if (pmap_bootstrapped) 1159238159Salc rw_assert(&pvh_global_lock, RA_WLOCKED); 1160159303Salc PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1161247400Sattilio if ((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) == 0) 1162250747Salc VM_OBJECT_ASSERT_LOCKED(m->object); 116377957Sbenno 1164142416Sgrehan /* XXX change the pvo head for fake pages */ 1165224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) { 1166189675Snwhitehorn pvo_flags &= ~PVO_MANAGED; 1167152180Sgrehan pvo_head = &moea_pvo_kunmanaged; 1168189675Snwhitehorn zone = moea_upvo_zone; 1169189675Snwhitehorn } 1170142416Sgrehan 1171213335Snwhitehorn pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); 117277957Sbenno 1173164229Salc if (prot & VM_PROT_WRITE) { 117490643Sbenno pte_lo |= PTE_BW; 1175208810Salc if (pmap_bootstrapped && 1176224746Skib (m->oflags & VPO_UNMANAGED) == 0) 1177225418Skib vm_page_aflag_set(m, PGA_WRITEABLE); 1178164229Salc } else 117990643Sbenno pte_lo |= PTE_BR; 118077957Sbenno 1181142416Sgrehan if (prot & VM_PROT_EXECUTE) 1182142416Sgrehan pvo_flags |= PVO_EXECUTABLE; 118377957Sbenno 118490643Sbenno if (wired) 118590643Sbenno pvo_flags |= PVO_WIRED; 118677957Sbenno 1187152180Sgrehan error = moea_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), 118896250Sbenno pte_lo, pvo_flags); 118990643Sbenno 119096250Sbenno /* 1191233949Snwhitehorn * Flush the real page from the instruction cache. This has be done 1192233949Snwhitehorn * for all user mappings to prevent information leakage via the 1193234149Snwhitehorn * instruction cache. moea_pvo_enter() returns ENOENT for the first 1194234149Snwhitehorn * mapping for a page. 119596250Sbenno */ 1196234149Snwhitehorn if (pmap != kernel_pmap && error == ENOENT && 1197234149Snwhitehorn (pte_lo & (PTE_I | PTE_G)) == 0) 1198152180Sgrehan moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 119977957Sbenno} 120077957Sbenno 1201159303Salc/* 1202159303Salc * Maps a sequence of resident pages belonging to the same object. 1203159303Salc * The sequence begins with the given page m_start. This page is 1204159303Salc * mapped at the given virtual address start. Each subsequent page is 1205159303Salc * mapped at a virtual address that is offset from start by the same 1206159303Salc * amount as the page is offset from m_start within the object. The 1207159303Salc * last page in the sequence is the page with the largest offset from 1208159303Salc * m_start that can be mapped at a virtual address less than the given 1209159303Salc * virtual address end. Not every virtual page between start and end 1210159303Salc * is mapped; only those for which a resident page exists with the 1211159303Salc * corresponding offset from m_start are mapped. 1212159303Salc */ 1213159303Salcvoid 1214159303Salcmoea_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, 1215159303Salc vm_page_t m_start, vm_prot_t prot) 1216159303Salc{ 1217159303Salc vm_page_t m; 1218159303Salc vm_pindex_t diff, psize; 1219159303Salc 1220250884Sattilio VM_OBJECT_ASSERT_LOCKED(m_start->object); 1221250884Sattilio 1222159303Salc psize = atop(end - start); 1223159303Salc m = m_start; 1224238159Salc rw_wlock(&pvh_global_lock); 1225159303Salc PMAP_LOCK(pm); 1226159303Salc while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1227159303Salc moea_enter_locked(pm, start + ptoa(diff), m, prot & 1228159303Salc (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1229159303Salc m = TAILQ_NEXT(m, listq); 1230159303Salc } 1231238159Salc rw_wunlock(&pvh_global_lock); 1232159303Salc PMAP_UNLOCK(pm); 1233159303Salc} 1234159303Salc 1235159627Supsvoid 1236152180Sgrehanmoea_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, 1237159627Sups vm_prot_t prot) 1238117045Salc{ 1239117045Salc 1240238159Salc rw_wlock(&pvh_global_lock); 1241159303Salc PMAP_LOCK(pm); 1242159303Salc moea_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1243152180Sgrehan FALSE); 1244238159Salc rw_wunlock(&pvh_global_lock); 1245159303Salc PMAP_UNLOCK(pm); 1246117045Salc} 1247117045Salc 1248131658Salcvm_paddr_t 1249152180Sgrehanmoea_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) 125077957Sbenno{ 125196353Sbenno struct pvo_entry *pvo; 1252134329Salc vm_paddr_t pa; 125396353Sbenno 1254134329Salc PMAP_LOCK(pm); 1255152180Sgrehan pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 1256134329Salc if (pvo == NULL) 1257134329Salc pa = 0; 1258134329Salc else 1259183290Snwhitehorn pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 1260134329Salc PMAP_UNLOCK(pm); 1261134329Salc return (pa); 126277957Sbenno} 126377957Sbenno 126477957Sbenno/* 1265120336Sgrehan * Atomically extract and hold the physical page with the given 1266120336Sgrehan * pmap and virtual address pair if that mapping permits the given 1267120336Sgrehan * protection. 1268120336Sgrehan */ 1269120336Sgrehanvm_page_t 1270152180Sgrehanmoea_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) 1271120336Sgrehan{ 1272132666Salc struct pvo_entry *pvo; 1273120336Sgrehan vm_page_t m; 1274207410Skmacy vm_paddr_t pa; 1275207410Skmacy 1276120336Sgrehan m = NULL; 1277207410Skmacy pa = 0; 1278134329Salc PMAP_LOCK(pmap); 1279207410Skmacyretry: 1280152180Sgrehan pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); 1281183290Snwhitehorn if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID) && 1282183290Snwhitehorn ((pvo->pvo_pte.pte.pte_lo & PTE_PP) == PTE_RW || 1283132666Salc (prot & VM_PROT_WRITE) == 0)) { 1284207410Skmacy if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.pte.pte_lo & PTE_RPGN, &pa)) 1285207410Skmacy goto retry; 1286183290Snwhitehorn m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN); 1287120336Sgrehan vm_page_hold(m); 1288120336Sgrehan } 1289207410Skmacy PA_UNLOCK_COND(pa); 1290134329Salc PMAP_UNLOCK(pmap); 1291120336Sgrehan return (m); 1292120336Sgrehan} 1293120336Sgrehan 129490643Sbennovoid 1295152180Sgrehanmoea_init(mmu_t mmu) 129677957Sbenno{ 129777957Sbenno 1298152180Sgrehan moea_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), 1299125442Sgrehan NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1300125442Sgrehan UMA_ZONE_VM | UMA_ZONE_NOFREE); 1301152180Sgrehan moea_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), 1302125442Sgrehan NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1303125442Sgrehan UMA_ZONE_VM | UMA_ZONE_NOFREE); 1304152180Sgrehan moea_initialized = TRUE; 130577957Sbenno} 130677957Sbenno 130790643Sbennoboolean_t 1308207155Salcmoea_is_referenced(mmu_t mmu, vm_page_t m) 1309207155Salc{ 1310238357Salc boolean_t rv; 1311207155Salc 1312224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1313208574Salc ("moea_is_referenced: page %p is not managed", m)); 1314238357Salc rw_wlock(&pvh_global_lock); 1315238357Salc rv = moea_query_bit(m, PTE_REF); 1316238357Salc rw_wunlock(&pvh_global_lock); 1317238357Salc return (rv); 1318207155Salc} 1319207155Salc 1320207155Salcboolean_t 1321152180Sgrehanmoea_is_modified(mmu_t mmu, vm_page_t m) 132290643Sbenno{ 1323238357Salc boolean_t rv; 132496353Sbenno 1325224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1326208504Salc ("moea_is_modified: page %p is not managed", m)); 1327208504Salc 1328208504Salc /* 1329225418Skib * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be 1330225418Skib * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 1331208504Salc * is clear, no PTEs can have PTE_CHG set. 1332208504Salc */ 1333248084Sattilio VM_OBJECT_ASSERT_WLOCKED(m->object); 1334208504Salc if ((m->oflags & VPO_BUSY) == 0 && 1335225418Skib (m->aflags & PGA_WRITEABLE) == 0) 133696353Sbenno return (FALSE); 1337238357Salc rw_wlock(&pvh_global_lock); 1338238357Salc rv = moea_query_bit(m, PTE_CHG); 1339238357Salc rw_wunlock(&pvh_global_lock); 1340238357Salc return (rv); 134190643Sbenno} 134290643Sbenno 1343214617Salcboolean_t 1344214617Salcmoea_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t va) 1345214617Salc{ 1346214617Salc struct pvo_entry *pvo; 1347214617Salc boolean_t rv; 1348214617Salc 1349214617Salc PMAP_LOCK(pmap); 1350214617Salc pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); 1351214617Salc rv = pvo == NULL || (pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0; 1352214617Salc PMAP_UNLOCK(pmap); 1353214617Salc return (rv); 1354214617Salc} 1355214617Salc 135690643Sbennovoid 1357152180Sgrehanmoea_clear_reference(mmu_t mmu, vm_page_t m) 135890643Sbenno{ 1359110172Sgrehan 1360224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1361208504Salc ("moea_clear_reference: page %p is not managed", m)); 1362238357Salc rw_wlock(&pvh_global_lock); 1363208990Salc moea_clear_bit(m, PTE_REF); 1364238357Salc rw_wunlock(&pvh_global_lock); 136590643Sbenno} 136690643Sbenno 1367110172Sgrehanvoid 1368152180Sgrehanmoea_clear_modify(mmu_t mmu, vm_page_t m) 1369110172Sgrehan{ 1370110172Sgrehan 1371224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1372208504Salc ("moea_clear_modify: page %p is not managed", m)); 1373248084Sattilio VM_OBJECT_ASSERT_WLOCKED(m->object); 1374208504Salc KASSERT((m->oflags & VPO_BUSY) == 0, 1375208504Salc ("moea_clear_modify: page %p is busy", m)); 1376208504Salc 1377208504Salc /* 1378225418Skib * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_CHG 1379208504Salc * set. If the object containing the page is locked and the page is 1380225418Skib * not VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 1381208504Salc */ 1382225418Skib if ((m->aflags & PGA_WRITEABLE) == 0) 1383110172Sgrehan return; 1384238357Salc rw_wlock(&pvh_global_lock); 1385208990Salc moea_clear_bit(m, PTE_CHG); 1386238357Salc rw_wunlock(&pvh_global_lock); 1387110172Sgrehan} 1388110172Sgrehan 138991403Ssilby/* 1390160889Salc * Clear the write and modified bits in each of the given page's mappings. 1391160889Salc */ 1392160889Salcvoid 1393160889Salcmoea_remove_write(mmu_t mmu, vm_page_t m) 1394160889Salc{ 1395160889Salc struct pvo_entry *pvo; 1396160889Salc struct pte *pt; 1397160889Salc pmap_t pmap; 1398160889Salc u_int lo; 1399160889Salc 1400224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1401208175Salc ("moea_remove_write: page %p is not managed", m)); 1402208175Salc 1403208175Salc /* 1404225418Skib * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 1405225418Skib * another thread while the object is locked. Thus, if PGA_WRITEABLE 1406208175Salc * is clear, no page table entries need updating. 1407208175Salc */ 1408248084Sattilio VM_OBJECT_ASSERT_WLOCKED(m->object); 1409208175Salc if ((m->oflags & VPO_BUSY) == 0 && 1410225418Skib (m->aflags & PGA_WRITEABLE) == 0) 1411160889Salc return; 1412238159Salc rw_wlock(&pvh_global_lock); 1413160889Salc lo = moea_attr_fetch(m); 1414183094Smarcel powerpc_sync(); 1415160889Salc LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1416160889Salc pmap = pvo->pvo_pmap; 1417160889Salc PMAP_LOCK(pmap); 1418183290Snwhitehorn if ((pvo->pvo_pte.pte.pte_lo & PTE_PP) != PTE_BR) { 1419160889Salc pt = moea_pvo_to_pte(pvo, -1); 1420183290Snwhitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; 1421183290Snwhitehorn pvo->pvo_pte.pte.pte_lo |= PTE_BR; 1422160889Salc if (pt != NULL) { 1423183290Snwhitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 1424183290Snwhitehorn lo |= pvo->pvo_pte.pte.pte_lo; 1425183290Snwhitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_CHG; 1426183290Snwhitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, 1427160889Salc pvo->pvo_vaddr); 1428160889Salc mtx_unlock(&moea_table_mutex); 1429160889Salc } 1430160889Salc } 1431160889Salc PMAP_UNLOCK(pmap); 1432160889Salc } 1433160889Salc if ((lo & PTE_CHG) != 0) { 1434160889Salc moea_attr_clear(m, PTE_CHG); 1435160889Salc vm_page_dirty(m); 1436160889Salc } 1437225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1438238159Salc rw_wunlock(&pvh_global_lock); 1439160889Salc} 1440160889Salc 1441160889Salc/* 1442152180Sgrehan * moea_ts_referenced: 144391403Ssilby * 144491403Ssilby * Return a count of reference bits for a page, clearing those bits. 144591403Ssilby * It is not necessary for every reference bit to be cleared, but it 144691403Ssilby * is necessary that 0 only be returned when there are truly no 144791403Ssilby * reference bits set. 144891403Ssilby * 144991403Ssilby * XXX: The exact number of bits to check and clear is a matter that 145091403Ssilby * should be tested and standardized at some point in the future for 145191403Ssilby * optimal aging of shared pages. 145291403Ssilby */ 1453238357Salcint 1454152180Sgrehanmoea_ts_referenced(mmu_t mmu, vm_page_t m) 145590643Sbenno{ 1456238357Salc int count; 1457110172Sgrehan 1458224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1459208990Salc ("moea_ts_referenced: page %p is not managed", m)); 1460238357Salc rw_wlock(&pvh_global_lock); 1461238357Salc count = moea_clear_bit(m, PTE_REF); 1462238357Salc rw_wunlock(&pvh_global_lock); 1463238357Salc return (count); 146490643Sbenno} 146590643Sbenno 146677957Sbenno/* 1467213307Snwhitehorn * Modify the WIMG settings of all mappings for a page. 1468213307Snwhitehorn */ 1469213307Snwhitehornvoid 1470213307Snwhitehornmoea_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma) 1471213307Snwhitehorn{ 1472213307Snwhitehorn struct pvo_entry *pvo; 1473213335Snwhitehorn struct pvo_head *pvo_head; 1474213307Snwhitehorn struct pte *pt; 1475213307Snwhitehorn pmap_t pmap; 1476213307Snwhitehorn u_int lo; 1477213307Snwhitehorn 1478224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) { 1479213335Snwhitehorn m->md.mdpg_cache_attrs = ma; 1480213335Snwhitehorn return; 1481213335Snwhitehorn } 1482213335Snwhitehorn 1483238159Salc rw_wlock(&pvh_global_lock); 1484213335Snwhitehorn pvo_head = vm_page_to_pvoh(m); 1485213307Snwhitehorn lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), ma); 1486213335Snwhitehorn 1487213335Snwhitehorn LIST_FOREACH(pvo, pvo_head, pvo_vlink) { 1488213307Snwhitehorn pmap = pvo->pvo_pmap; 1489213307Snwhitehorn PMAP_LOCK(pmap); 1490213307Snwhitehorn pt = moea_pvo_to_pte(pvo, -1); 1491213307Snwhitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_WIMG; 1492213307Snwhitehorn pvo->pvo_pte.pte.pte_lo |= lo; 1493213307Snwhitehorn if (pt != NULL) { 1494213307Snwhitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, 1495213307Snwhitehorn pvo->pvo_vaddr); 1496213307Snwhitehorn if (pvo->pvo_pmap == kernel_pmap) 1497213307Snwhitehorn isync(); 1498213307Snwhitehorn } 1499213307Snwhitehorn mtx_unlock(&moea_table_mutex); 1500213307Snwhitehorn PMAP_UNLOCK(pmap); 1501213307Snwhitehorn } 1502213307Snwhitehorn m->md.mdpg_cache_attrs = ma; 1503238159Salc rw_wunlock(&pvh_global_lock); 1504213307Snwhitehorn} 1505213307Snwhitehorn 1506213307Snwhitehorn/* 150790643Sbenno * Map a wired page into kernel virtual address space. 150877957Sbenno */ 150977957Sbennovoid 1510235936Srajmoea_kenter(mmu_t mmu, vm_offset_t va, vm_paddr_t pa) 151177957Sbenno{ 1512213307Snwhitehorn 1513213307Snwhitehorn moea_kenter_attr(mmu, va, pa, VM_MEMATTR_DEFAULT); 1514213307Snwhitehorn} 1515213307Snwhitehorn 1516213307Snwhitehornvoid 1517213307Snwhitehornmoea_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma) 1518213307Snwhitehorn{ 151990643Sbenno u_int pte_lo; 152090643Sbenno int error; 152177957Sbenno 152290643Sbenno#if 0 152390643Sbenno if (va < VM_MIN_KERNEL_ADDRESS) 1524152180Sgrehan panic("moea_kenter: attempt to enter non-kernel address %#x", 152590643Sbenno va); 152690643Sbenno#endif 152777957Sbenno 1528213307Snwhitehorn pte_lo = moea_calc_wimg(pa, ma); 152977957Sbenno 1530135172Salc PMAP_LOCK(kernel_pmap); 1531152180Sgrehan error = moea_pvo_enter(kernel_pmap, moea_upvo_zone, 1532152180Sgrehan &moea_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED); 153390643Sbenno 153490643Sbenno if (error != 0 && error != ENOENT) 1535152180Sgrehan panic("moea_kenter: failed to enter va %#x pa %#x: %d", va, 153690643Sbenno pa, error); 153790643Sbenno 1538135172Salc PMAP_UNLOCK(kernel_pmap); 153977957Sbenno} 154077957Sbenno 154194838Sbenno/* 154294838Sbenno * Extract the physical page address associated with the given kernel virtual 154394838Sbenno * address. 154494838Sbenno */ 1545235936Srajvm_paddr_t 1546152180Sgrehanmoea_kextract(mmu_t mmu, vm_offset_t va) 154777957Sbenno{ 154894838Sbenno struct pvo_entry *pvo; 1549134329Salc vm_paddr_t pa; 155094838Sbenno 1551125185Sgrehan /* 1552183290Snwhitehorn * Allow direct mappings on 32-bit OEA 1553125185Sgrehan */ 1554125185Sgrehan if (va < VM_MIN_KERNEL_ADDRESS) { 1555125185Sgrehan return (va); 1556125185Sgrehan } 1557125185Sgrehan 1558134329Salc PMAP_LOCK(kernel_pmap); 1559152180Sgrehan pvo = moea_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); 1560152180Sgrehan KASSERT(pvo != NULL, ("moea_kextract: no addr found")); 1561183290Snwhitehorn pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF); 1562134329Salc PMAP_UNLOCK(kernel_pmap); 1563134329Salc return (pa); 156477957Sbenno} 156577957Sbenno 156691456Sbenno/* 156791456Sbenno * Remove a wired page from kernel virtual address space. 156891456Sbenno */ 156977957Sbennovoid 1570152180Sgrehanmoea_kremove(mmu_t mmu, vm_offset_t va) 157177957Sbenno{ 157291456Sbenno 1573152180Sgrehan moea_remove(mmu, kernel_pmap, va, va + PAGE_SIZE); 157477957Sbenno} 157577957Sbenno 157677957Sbenno/* 157790643Sbenno * Map a range of physical addresses into kernel virtual address space. 157890643Sbenno * 157990643Sbenno * The value passed in *virt is a suggested virtual address for the mapping. 158090643Sbenno * Architectures which can support a direct-mapped physical to virtual region 158190643Sbenno * can return the appropriate address within that region, leaving '*virt' 158290643Sbenno * unchanged. We cannot and therefore do not; *virt is updated with the 158390643Sbenno * first usable address after the mapped region. 158477957Sbenno */ 158590643Sbennovm_offset_t 1586235936Srajmoea_map(mmu_t mmu, vm_offset_t *virt, vm_paddr_t pa_start, 1587235936Sraj vm_paddr_t pa_end, int prot) 158877957Sbenno{ 158990643Sbenno vm_offset_t sva, va; 159077957Sbenno 159190643Sbenno sva = *virt; 159290643Sbenno va = sva; 159390643Sbenno for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) 1594152180Sgrehan moea_kenter(mmu, va, pa_start); 159590643Sbenno *virt = va; 159690643Sbenno return (sva); 159777957Sbenno} 159877957Sbenno 159977957Sbenno/* 160091403Ssilby * Returns true if the pmap's pv is one of the first 160191403Ssilby * 16 pvs linked to from this page. This count may 160291403Ssilby * be changed upwards or downwards in the future; it 160391403Ssilby * is only necessary that true be returned for a small 160491403Ssilby * subset of pmaps for proper page aging. 160591403Ssilby */ 160690643Sbennoboolean_t 1607152180Sgrehanmoea_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) 160890643Sbenno{ 1609110172Sgrehan int loops; 1610110172Sgrehan struct pvo_entry *pvo; 1611208990Salc boolean_t rv; 1612110172Sgrehan 1613224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1614208990Salc ("moea_page_exists_quick: page %p is not managed", m)); 1615110172Sgrehan loops = 0; 1616208990Salc rv = FALSE; 1617238159Salc rw_wlock(&pvh_global_lock); 1618110172Sgrehan LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1619208990Salc if (pvo->pvo_pmap == pmap) { 1620208990Salc rv = TRUE; 1621208990Salc break; 1622208990Salc } 1623110172Sgrehan if (++loops >= 16) 1624110172Sgrehan break; 1625110172Sgrehan } 1626238159Salc rw_wunlock(&pvh_global_lock); 1627208990Salc return (rv); 162890643Sbenno} 162977957Sbenno 1630173708Salc/* 1631173708Salc * Return the number of managed mappings to the given physical page 1632173708Salc * that are wired. 1633173708Salc */ 1634173708Salcint 1635173708Salcmoea_page_wired_mappings(mmu_t mmu, vm_page_t m) 1636173708Salc{ 1637173708Salc struct pvo_entry *pvo; 1638173708Salc int count; 1639173708Salc 1640173708Salc count = 0; 1641224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) 1642173708Salc return (count); 1643238159Salc rw_wlock(&pvh_global_lock); 1644173708Salc LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) 1645173708Salc if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 1646173708Salc count++; 1647238159Salc rw_wunlock(&pvh_global_lock); 1648173708Salc return (count); 1649173708Salc} 1650173708Salc 1651152180Sgrehanstatic u_int moea_vsidcontext; 165277957Sbenno 165390643Sbennovoid 1654152180Sgrehanmoea_pinit(mmu_t mmu, pmap_t pmap) 165590643Sbenno{ 165690643Sbenno int i, mask; 165790643Sbenno u_int entropy; 165877957Sbenno 1659152180Sgrehan KASSERT((int)pmap < VM_MIN_KERNEL_ADDRESS, ("moea_pinit: virt pmap")); 1660134329Salc PMAP_LOCK_INIT(pmap); 1661235689Snwhitehorn RB_INIT(&pmap->pmap_pvo); 1662126478Sgrehan 166390643Sbenno entropy = 0; 166490643Sbenno __asm __volatile("mftb %0" : "=r"(entropy)); 166577957Sbenno 1666183290Snwhitehorn if ((pmap->pmap_phys = (pmap_t)moea_kextract(mmu, (vm_offset_t)pmap)) 1667183290Snwhitehorn == NULL) { 1668183290Snwhitehorn pmap->pmap_phys = pmap; 1669183290Snwhitehorn } 1670183290Snwhitehorn 1671183290Snwhitehorn 1672212278Snwhitehorn mtx_lock(&moea_vsid_mutex); 167390643Sbenno /* 167490643Sbenno * Allocate some segment registers for this pmap. 167590643Sbenno */ 167690643Sbenno for (i = 0; i < NPMAPS; i += VSID_NBPW) { 167790643Sbenno u_int hash, n; 167877957Sbenno 167977957Sbenno /* 168090643Sbenno * Create a new value by mutiplying by a prime and adding in 168190643Sbenno * entropy from the timebase register. This is to make the 168290643Sbenno * VSID more random so that the PT hash function collides 168390643Sbenno * less often. (Note that the prime casues gcc to do shifts 168490643Sbenno * instead of a multiply.) 168577957Sbenno */ 1686152180Sgrehan moea_vsidcontext = (moea_vsidcontext * 0x1105) + entropy; 1687152180Sgrehan hash = moea_vsidcontext & (NPMAPS - 1); 168890643Sbenno if (hash == 0) /* 0 is special, avoid it */ 168990643Sbenno continue; 169090643Sbenno n = hash >> 5; 169190643Sbenno mask = 1 << (hash & (VSID_NBPW - 1)); 1692152180Sgrehan hash = (moea_vsidcontext & 0xfffff); 1693152180Sgrehan if (moea_vsid_bitmap[n] & mask) { /* collision? */ 169490643Sbenno /* anything free in this bucket? */ 1695152180Sgrehan if (moea_vsid_bitmap[n] == 0xffffffff) { 1696152180Sgrehan entropy = (moea_vsidcontext >> 20); 169790643Sbenno continue; 169890643Sbenno } 1699212322Snwhitehorn i = ffs(~moea_vsid_bitmap[n]) - 1; 170090643Sbenno mask = 1 << i; 170190643Sbenno hash &= 0xfffff & ~(VSID_NBPW - 1); 170290643Sbenno hash |= i; 170377957Sbenno } 1704227627Snwhitehorn KASSERT(!(moea_vsid_bitmap[n] & mask), 1705227627Snwhitehorn ("Allocating in-use VSID group %#x\n", hash)); 1706152180Sgrehan moea_vsid_bitmap[n] |= mask; 170790643Sbenno for (i = 0; i < 16; i++) 170890643Sbenno pmap->pm_sr[i] = VSID_MAKE(i, hash); 1709212278Snwhitehorn mtx_unlock(&moea_vsid_mutex); 171090643Sbenno return; 171190643Sbenno } 171277957Sbenno 1713212278Snwhitehorn mtx_unlock(&moea_vsid_mutex); 1714152180Sgrehan panic("moea_pinit: out of segments"); 171577957Sbenno} 171677957Sbenno 171777957Sbenno/* 171890643Sbenno * Initialize the pmap associated with process 0. 171977957Sbenno */ 172077957Sbennovoid 1721152180Sgrehanmoea_pinit0(mmu_t mmu, pmap_t pm) 172277957Sbenno{ 172377957Sbenno 1724152180Sgrehan moea_pinit(mmu, pm); 172590643Sbenno bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 172677957Sbenno} 172777957Sbenno 172894838Sbenno/* 172994838Sbenno * Set the physical protection on the specified range of this map as requested. 173094838Sbenno */ 173190643Sbennovoid 1732152180Sgrehanmoea_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, 1733152180Sgrehan vm_prot_t prot) 173490643Sbenno{ 1735235689Snwhitehorn struct pvo_entry *pvo, *tpvo, key; 173694838Sbenno struct pte *pt; 173794838Sbenno 173894838Sbenno KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1739152180Sgrehan ("moea_protect: non current pmap")); 174094838Sbenno 174194838Sbenno if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1742152180Sgrehan moea_remove(mmu, pm, sva, eva); 174394838Sbenno return; 174494838Sbenno } 174594838Sbenno 1746238159Salc rw_wlock(&pvh_global_lock); 1747134329Salc PMAP_LOCK(pm); 1748235689Snwhitehorn key.pvo_vaddr = sva; 1749235689Snwhitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1750235689Snwhitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 1751235689Snwhitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 175294838Sbenno if ((prot & VM_PROT_EXECUTE) == 0) 175394838Sbenno pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 175494838Sbenno 175594838Sbenno /* 175694838Sbenno * Grab the PTE pointer before we diddle with the cached PTE 175794838Sbenno * copy. 175894838Sbenno */ 1759235689Snwhitehorn pt = moea_pvo_to_pte(pvo, -1); 176094838Sbenno /* 176194838Sbenno * Change the protection of the page. 176294838Sbenno */ 1763183290Snwhitehorn pvo->pvo_pte.pte.pte_lo &= ~PTE_PP; 1764183290Snwhitehorn pvo->pvo_pte.pte.pte_lo |= PTE_BR; 176594838Sbenno 176694838Sbenno /* 176794838Sbenno * If the PVO is in the page table, update that pte as well. 176894838Sbenno */ 1769159928Salc if (pt != NULL) { 1770183290Snwhitehorn moea_pte_change(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); 1771159928Salc mtx_unlock(&moea_table_mutex); 1772159928Salc } 177394838Sbenno } 1774238159Salc rw_wunlock(&pvh_global_lock); 1775134329Salc PMAP_UNLOCK(pm); 177677957Sbenno} 177777957Sbenno 177891456Sbenno/* 177991456Sbenno * Map a list of wired pages into kernel virtual address space. This is 178091456Sbenno * intended for temporary mappings which do not need page modification or 178191456Sbenno * references recorded. Existing mappings in the region are overwritten. 178291456Sbenno */ 178390643Sbennovoid 1784152180Sgrehanmoea_qenter(mmu_t mmu, vm_offset_t sva, vm_page_t *m, int count) 178577957Sbenno{ 1786110172Sgrehan vm_offset_t va; 178777957Sbenno 1788110172Sgrehan va = sva; 1789110172Sgrehan while (count-- > 0) { 1790152180Sgrehan moea_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); 1791110172Sgrehan va += PAGE_SIZE; 1792110172Sgrehan m++; 1793110172Sgrehan } 179490643Sbenno} 179577957Sbenno 179691456Sbenno/* 179791456Sbenno * Remove page mappings from kernel virtual address space. Intended for 1798152180Sgrehan * temporary mappings entered by moea_qenter. 179991456Sbenno */ 180090643Sbennovoid 1801152180Sgrehanmoea_qremove(mmu_t mmu, vm_offset_t sva, int count) 180290643Sbenno{ 1803110172Sgrehan vm_offset_t va; 180491456Sbenno 1805110172Sgrehan va = sva; 1806110172Sgrehan while (count-- > 0) { 1807152180Sgrehan moea_kremove(mmu, va); 1808110172Sgrehan va += PAGE_SIZE; 1809110172Sgrehan } 181077957Sbenno} 181177957Sbenno 181290643Sbennovoid 1813152180Sgrehanmoea_release(mmu_t mmu, pmap_t pmap) 181490643Sbenno{ 1815103604Sgrehan int idx, mask; 1816103604Sgrehan 1817103604Sgrehan /* 1818103604Sgrehan * Free segment register's VSID 1819103604Sgrehan */ 1820103604Sgrehan if (pmap->pm_sr[0] == 0) 1821152180Sgrehan panic("moea_release"); 1822103604Sgrehan 1823212278Snwhitehorn mtx_lock(&moea_vsid_mutex); 1824103604Sgrehan idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1); 1825103604Sgrehan mask = 1 << (idx % VSID_NBPW); 1826103604Sgrehan idx /= VSID_NBPW; 1827152180Sgrehan moea_vsid_bitmap[idx] &= ~mask; 1828212278Snwhitehorn mtx_unlock(&moea_vsid_mutex); 1829134329Salc PMAP_LOCK_DESTROY(pmap); 183077957Sbenno} 183177957Sbenno 183291456Sbenno/* 183391456Sbenno * Remove the given range of addresses from the specified map. 183491456Sbenno */ 183590643Sbennovoid 1836152180Sgrehanmoea_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) 183777957Sbenno{ 1838235689Snwhitehorn struct pvo_entry *pvo, *tpvo, key; 183991456Sbenno 1840238159Salc rw_wlock(&pvh_global_lock); 1841134329Salc PMAP_LOCK(pm); 1842235689Snwhitehorn key.pvo_vaddr = sva; 1843235689Snwhitehorn for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); 1844235689Snwhitehorn pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { 1845235689Snwhitehorn tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); 1846235689Snwhitehorn moea_pvo_remove(pvo, -1); 184791456Sbenno } 1848140538Sgrehan PMAP_UNLOCK(pm); 1849238159Salc rw_wunlock(&pvh_global_lock); 185077957Sbenno} 185177957Sbenno 185294838Sbenno/* 1853152180Sgrehan * Remove physical page from all pmaps in which it resides. moea_pvo_remove() 1854110172Sgrehan * will reflect changes in pte's back to the vm_page. 1855110172Sgrehan */ 1856110172Sgrehanvoid 1857152180Sgrehanmoea_remove_all(mmu_t mmu, vm_page_t m) 1858110172Sgrehan{ 1859110172Sgrehan struct pvo_head *pvo_head; 1860110172Sgrehan struct pvo_entry *pvo, *next_pvo; 1861134329Salc pmap_t pmap; 1862110172Sgrehan 1863238159Salc rw_wlock(&pvh_global_lock); 1864110172Sgrehan pvo_head = vm_page_to_pvoh(m); 1865110172Sgrehan for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 1866110172Sgrehan next_pvo = LIST_NEXT(pvo, pvo_vlink); 1867133166Sgrehan 1868134329Salc pmap = pvo->pvo_pmap; 1869134329Salc PMAP_LOCK(pmap); 1870152180Sgrehan moea_pvo_remove(pvo, -1); 1871134329Salc PMAP_UNLOCK(pmap); 1872110172Sgrehan } 1873238357Salc if ((m->aflags & PGA_WRITEABLE) && moea_query_bit(m, PTE_CHG)) { 1874208847Snwhitehorn moea_attr_clear(m, PTE_CHG); 1875204042Snwhitehorn vm_page_dirty(m); 1876204042Snwhitehorn } 1877225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1878238159Salc rw_wunlock(&pvh_global_lock); 1879110172Sgrehan} 1880110172Sgrehan 1881110172Sgrehan/* 188290643Sbenno * Allocate a physical page of memory directly from the phys_avail map. 1883152180Sgrehan * Can only be called from moea_bootstrap before avail start and end are 188490643Sbenno * calculated. 188583682Smp */ 188690643Sbennostatic vm_offset_t 1887152180Sgrehanmoea_bootstrap_alloc(vm_size_t size, u_int align) 188883682Smp{ 188990643Sbenno vm_offset_t s, e; 189090643Sbenno int i, j; 189183682Smp 189290643Sbenno size = round_page(size); 189390643Sbenno for (i = 0; phys_avail[i + 1] != 0; i += 2) { 189490643Sbenno if (align != 0) 189590643Sbenno s = (phys_avail[i] + align - 1) & ~(align - 1); 189690643Sbenno else 189790643Sbenno s = phys_avail[i]; 189890643Sbenno e = s + size; 189990643Sbenno 190090643Sbenno if (s < phys_avail[i] || e > phys_avail[i + 1]) 190190643Sbenno continue; 190290643Sbenno 190390643Sbenno if (s == phys_avail[i]) { 190490643Sbenno phys_avail[i] += size; 190590643Sbenno } else if (e == phys_avail[i + 1]) { 190690643Sbenno phys_avail[i + 1] -= size; 190790643Sbenno } else { 190890643Sbenno for (j = phys_avail_count * 2; j > i; j -= 2) { 190990643Sbenno phys_avail[j] = phys_avail[j - 2]; 191090643Sbenno phys_avail[j + 1] = phys_avail[j - 1]; 191190643Sbenno } 191290643Sbenno 191390643Sbenno phys_avail[i + 3] = phys_avail[i + 1]; 191490643Sbenno phys_avail[i + 1] = s; 191590643Sbenno phys_avail[i + 2] = e; 191690643Sbenno phys_avail_count++; 191790643Sbenno } 191890643Sbenno 191990643Sbenno return (s); 192083682Smp } 1921152180Sgrehan panic("moea_bootstrap_alloc: could not allocate memory"); 192283682Smp} 192383682Smp 192490643Sbennostatic void 1925152180Sgrehanmoea_syncicache(vm_offset_t pa, vm_size_t len) 192677957Sbenno{ 192790643Sbenno __syncicache((void *)pa, len); 192890643Sbenno} 192977957Sbenno 193090643Sbennostatic int 1931152180Sgrehanmoea_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, 193290643Sbenno vm_offset_t va, vm_offset_t pa, u_int pte_lo, int flags) 193377957Sbenno{ 193490643Sbenno struct pvo_entry *pvo; 193590643Sbenno u_int sr; 193690643Sbenno int first; 193790643Sbenno u_int ptegidx; 193890643Sbenno int i; 1939103604Sgrehan int bootstrap; 194077957Sbenno 1941152180Sgrehan moea_pvo_enter_calls++; 194296250Sbenno first = 0; 1943103604Sgrehan bootstrap = 0; 194490643Sbenno 194590643Sbenno /* 194690643Sbenno * Compute the PTE Group index. 194790643Sbenno */ 194890643Sbenno va &= ~ADDR_POFF; 194990643Sbenno sr = va_to_sr(pm->pm_sr, va); 195090643Sbenno ptegidx = va_to_pteg(sr, va); 195190643Sbenno 195290643Sbenno /* 195390643Sbenno * Remove any existing mapping for this page. Reuse the pvo entry if 195490643Sbenno * there is a mapping. 195590643Sbenno */ 1956152180Sgrehan mtx_lock(&moea_table_mutex); 1957152180Sgrehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 195890643Sbenno if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1959183290Snwhitehorn if ((pvo->pvo_pte.pte.pte_lo & PTE_RPGN) == pa && 1960183290Snwhitehorn (pvo->pvo_pte.pte.pte_lo & PTE_PP) == 196196334Sbenno (pte_lo & PTE_PP)) { 1962152180Sgrehan mtx_unlock(&moea_table_mutex); 196392521Sbenno return (0); 196496334Sbenno } 1965152180Sgrehan moea_pvo_remove(pvo, -1); 196690643Sbenno break; 196790643Sbenno } 196890643Sbenno } 196990643Sbenno 197090643Sbenno /* 197190643Sbenno * If we aren't overwriting a mapping, try to allocate. 197290643Sbenno */ 1973152180Sgrehan if (moea_initialized) { 197492847Sjeff pvo = uma_zalloc(zone, M_NOWAIT); 197592521Sbenno } else { 1976152180Sgrehan if (moea_bpvo_pool_index >= BPVO_POOL_SIZE) { 1977152180Sgrehan panic("moea_enter: bpvo pool exhausted, %d, %d, %d", 1978152180Sgrehan moea_bpvo_pool_index, BPVO_POOL_SIZE, 197999037Sbenno BPVO_POOL_SIZE * sizeof(struct pvo_entry)); 198092521Sbenno } 1981152180Sgrehan pvo = &moea_bpvo_pool[moea_bpvo_pool_index]; 1982152180Sgrehan moea_bpvo_pool_index++; 1983103604Sgrehan bootstrap = 1; 198492521Sbenno } 198590643Sbenno 198690643Sbenno if (pvo == NULL) { 1987152180Sgrehan mtx_unlock(&moea_table_mutex); 198890643Sbenno return (ENOMEM); 198990643Sbenno } 199090643Sbenno 1991152180Sgrehan moea_pvo_entries++; 199290643Sbenno pvo->pvo_vaddr = va; 199390643Sbenno pvo->pvo_pmap = pm; 1994152180Sgrehan LIST_INSERT_HEAD(&moea_pvo_table[ptegidx], pvo, pvo_olink); 199590643Sbenno pvo->pvo_vaddr &= ~ADDR_POFF; 199690643Sbenno if (flags & VM_PROT_EXECUTE) 199790643Sbenno pvo->pvo_vaddr |= PVO_EXECUTABLE; 199890643Sbenno if (flags & PVO_WIRED) 199990643Sbenno pvo->pvo_vaddr |= PVO_WIRED; 2000152180Sgrehan if (pvo_head != &moea_pvo_kunmanaged) 200190643Sbenno pvo->pvo_vaddr |= PVO_MANAGED; 2002103604Sgrehan if (bootstrap) 2003103604Sgrehan pvo->pvo_vaddr |= PVO_BOOTSTRAP; 2004142416Sgrehan 2005183290Snwhitehorn moea_pte_create(&pvo->pvo_pte.pte, sr, va, pa | pte_lo); 200690643Sbenno 200790643Sbenno /* 2008228412Snwhitehorn * Add to pmap list 2009228412Snwhitehorn */ 2010235689Snwhitehorn RB_INSERT(pvo_tree, &pm->pmap_pvo, pvo); 2011228412Snwhitehorn 2012228412Snwhitehorn /* 201390643Sbenno * Remember if the list was empty and therefore will be the first 201490643Sbenno * item. 201590643Sbenno */ 201696250Sbenno if (LIST_FIRST(pvo_head) == NULL) 201796250Sbenno first = 1; 2018142416Sgrehan LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 201990643Sbenno 2020183290Snwhitehorn if (pvo->pvo_pte.pte.pte_lo & PVO_WIRED) 2021134453Salc pm->pm_stats.wired_count++; 2022134453Salc pm->pm_stats.resident_count++; 202390643Sbenno 2024183290Snwhitehorn i = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); 2025253976Sjhibbits KASSERT(i < 8, ("Invalid PTE index")); 202690643Sbenno if (i >= 0) { 202790643Sbenno PVO_PTEGIDX_SET(pvo, i); 202890643Sbenno } else { 2029152180Sgrehan panic("moea_pvo_enter: overflow"); 2030152180Sgrehan moea_pte_overflow++; 203190643Sbenno } 2032152180Sgrehan mtx_unlock(&moea_table_mutex); 203390643Sbenno 203490643Sbenno return (first ? ENOENT : 0); 203577957Sbenno} 203677957Sbenno 203790643Sbennostatic void 2038152180Sgrehanmoea_pvo_remove(struct pvo_entry *pvo, int pteidx) 203977957Sbenno{ 204090643Sbenno struct pte *pt; 204177957Sbenno 204290643Sbenno /* 204390643Sbenno * If there is an active pte entry, we need to deactivate it (and 204490643Sbenno * save the ref & cfg bits). 204590643Sbenno */ 2046152180Sgrehan pt = moea_pvo_to_pte(pvo, pteidx); 204790643Sbenno if (pt != NULL) { 2048183290Snwhitehorn moea_pte_unset(pt, &pvo->pvo_pte.pte, pvo->pvo_vaddr); 2049159928Salc mtx_unlock(&moea_table_mutex); 205090643Sbenno PVO_PTEGIDX_CLR(pvo); 205190643Sbenno } else { 2052152180Sgrehan moea_pte_overflow--; 2053142416Sgrehan } 205490643Sbenno 205590643Sbenno /* 205690643Sbenno * Update our statistics. 205790643Sbenno */ 205890643Sbenno pvo->pvo_pmap->pm_stats.resident_count--; 2059183290Snwhitehorn if (pvo->pvo_pte.pte.pte_lo & PVO_WIRED) 206090643Sbenno pvo->pvo_pmap->pm_stats.wired_count--; 206190643Sbenno 206290643Sbenno /* 206390643Sbenno * Save the REF/CHG bits into their cache if the page is managed. 206490643Sbenno */ 2065224746Skib if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED) { 206690643Sbenno struct vm_page *pg; 206790643Sbenno 2068183290Snwhitehorn pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN); 206990643Sbenno if (pg != NULL) { 2070183290Snwhitehorn moea_attr_save(pg, pvo->pvo_pte.pte.pte_lo & 207190643Sbenno (PTE_REF | PTE_CHG)); 207290643Sbenno } 207390643Sbenno } 207490643Sbenno 207590643Sbenno /* 2076228412Snwhitehorn * Remove this PVO from the PV and pmap lists. 207790643Sbenno */ 207890643Sbenno LIST_REMOVE(pvo, pvo_vlink); 2079235689Snwhitehorn RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo); 208090643Sbenno 208190643Sbenno /* 208290643Sbenno * Remove this from the overflow list and return it to the pool 208390643Sbenno * if we aren't going to reuse it. 208490643Sbenno */ 208590643Sbenno LIST_REMOVE(pvo, pvo_olink); 208692521Sbenno if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) 2087152180Sgrehan uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea_mpvo_zone : 2088152180Sgrehan moea_upvo_zone, pvo); 2089152180Sgrehan moea_pvo_entries--; 2090152180Sgrehan moea_pvo_remove_calls++; 209177957Sbenno} 209277957Sbenno 209390643Sbennostatic __inline int 2094152180Sgrehanmoea_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 209577957Sbenno{ 209690643Sbenno int pteidx; 209777957Sbenno 209890643Sbenno /* 209990643Sbenno * We can find the actual pte entry without searching by grabbing 210090643Sbenno * the PTEG index from 3 unused bits in pte_lo[11:9] and by 210190643Sbenno * noticing the HID bit. 210290643Sbenno */ 210390643Sbenno pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 2104183290Snwhitehorn if (pvo->pvo_pte.pte.pte_hi & PTE_HID) 2105152180Sgrehan pteidx ^= moea_pteg_mask * 8; 210690643Sbenno 210790643Sbenno return (pteidx); 210877957Sbenno} 210977957Sbenno 211090643Sbennostatic struct pvo_entry * 2111152180Sgrehanmoea_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p) 211277957Sbenno{ 211390643Sbenno struct pvo_entry *pvo; 211490643Sbenno int ptegidx; 211590643Sbenno u_int sr; 211677957Sbenno 211790643Sbenno va &= ~ADDR_POFF; 211890643Sbenno sr = va_to_sr(pm->pm_sr, va); 211990643Sbenno ptegidx = va_to_pteg(sr, va); 212090643Sbenno 2121152180Sgrehan mtx_lock(&moea_table_mutex); 2122152180Sgrehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 212390643Sbenno if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 212490643Sbenno if (pteidx_p) 2125152180Sgrehan *pteidx_p = moea_pvo_pte_index(pvo, ptegidx); 2126134535Salc break; 212790643Sbenno } 212890643Sbenno } 2129152180Sgrehan mtx_unlock(&moea_table_mutex); 213090643Sbenno 2131134535Salc return (pvo); 213277957Sbenno} 213377957Sbenno 213490643Sbennostatic struct pte * 2135152180Sgrehanmoea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 213677957Sbenno{ 213790643Sbenno struct pte *pt; 213877957Sbenno 213990643Sbenno /* 214090643Sbenno * If we haven't been supplied the ptegidx, calculate it. 214190643Sbenno */ 214290643Sbenno if (pteidx == -1) { 214390643Sbenno int ptegidx; 214490643Sbenno u_int sr; 214577957Sbenno 214690643Sbenno sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr); 214790643Sbenno ptegidx = va_to_pteg(sr, pvo->pvo_vaddr); 2148152180Sgrehan pteidx = moea_pvo_pte_index(pvo, ptegidx); 214990643Sbenno } 215090643Sbenno 2151152180Sgrehan pt = &moea_pteg_table[pteidx >> 3].pt[pteidx & 7]; 2152159928Salc mtx_lock(&moea_table_mutex); 215390643Sbenno 2154183290Snwhitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 2155152180Sgrehan panic("moea_pvo_to_pte: pvo %p has valid pte in pvo but no " 215690643Sbenno "valid pte index", pvo); 215790643Sbenno } 215890643Sbenno 2159183290Snwhitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 2160152180Sgrehan panic("moea_pvo_to_pte: pvo %p has valid pte index in pvo " 216190643Sbenno "pvo but no valid pte", pvo); 216290643Sbenno } 216390643Sbenno 2164183290Snwhitehorn if ((pt->pte_hi ^ (pvo->pvo_pte.pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 2165183290Snwhitehorn if ((pvo->pvo_pte.pte.pte_hi & PTE_VALID) == 0) { 2166152180Sgrehan panic("moea_pvo_to_pte: pvo %p has valid pte in " 2167152180Sgrehan "moea_pteg_table %p but invalid in pvo", pvo, pt); 216877957Sbenno } 216990643Sbenno 2170183290Snwhitehorn if (((pt->pte_lo ^ pvo->pvo_pte.pte.pte_lo) & ~(PTE_CHG|PTE_REF)) 217190643Sbenno != 0) { 2172152180Sgrehan panic("moea_pvo_to_pte: pvo %p pte does not match " 2173152180Sgrehan "pte %p in moea_pteg_table", pvo, pt); 217490643Sbenno } 217590643Sbenno 2176159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 217790643Sbenno return (pt); 217877957Sbenno } 217977957Sbenno 2180183290Snwhitehorn if (pvo->pvo_pte.pte.pte_hi & PTE_VALID) { 2181152180Sgrehan panic("moea_pvo_to_pte: pvo %p has invalid pte %p in " 2182253976Sjhibbits "moea_pteg_table but valid in pvo: %8x, %8x", pvo, pt, pvo->pvo_pte.pte.pte_hi, pt->pte_hi); 218390643Sbenno } 218477957Sbenno 2185159928Salc mtx_unlock(&moea_table_mutex); 218690643Sbenno return (NULL); 218777957Sbenno} 218878880Sbenno 218978880Sbenno/* 219090643Sbenno * XXX: THIS STUFF SHOULD BE IN pte.c? 219178880Sbenno */ 219290643Sbennoint 2193152180Sgrehanmoea_pte_spill(vm_offset_t addr) 219478880Sbenno{ 219590643Sbenno struct pvo_entry *source_pvo, *victim_pvo; 219690643Sbenno struct pvo_entry *pvo; 219790643Sbenno int ptegidx, i, j; 219890643Sbenno u_int sr; 219990643Sbenno struct pteg *pteg; 220090643Sbenno struct pte *pt; 220178880Sbenno 2202152180Sgrehan moea_pte_spills++; 220390643Sbenno 220494836Sbenno sr = mfsrin(addr); 220590643Sbenno ptegidx = va_to_pteg(sr, addr); 220690643Sbenno 220778880Sbenno /* 220890643Sbenno * Have to substitute some entry. Use the primary hash for this. 220990643Sbenno * Use low bits of timebase as random generator. 221078880Sbenno */ 2211152180Sgrehan pteg = &moea_pteg_table[ptegidx]; 2212152180Sgrehan mtx_lock(&moea_table_mutex); 221390643Sbenno __asm __volatile("mftb %0" : "=r"(i)); 221490643Sbenno i &= 7; 221590643Sbenno pt = &pteg->pt[i]; 221678880Sbenno 221790643Sbenno source_pvo = NULL; 221890643Sbenno victim_pvo = NULL; 2219152180Sgrehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx], pvo_olink) { 222078880Sbenno /* 222190643Sbenno * We need to find a pvo entry for this address. 222278880Sbenno */ 222390643Sbenno if (source_pvo == NULL && 2224183290Snwhitehorn moea_pte_match(&pvo->pvo_pte.pte, sr, addr, 2225183290Snwhitehorn pvo->pvo_pte.pte.pte_hi & PTE_HID)) { 222690643Sbenno /* 222790643Sbenno * Now found an entry to be spilled into the pteg. 222890643Sbenno * The PTE is now valid, so we know it's active. 222990643Sbenno */ 2230183290Snwhitehorn j = moea_pte_insert(ptegidx, &pvo->pvo_pte.pte); 223178880Sbenno 223290643Sbenno if (j >= 0) { 223390643Sbenno PVO_PTEGIDX_SET(pvo, j); 2234152180Sgrehan moea_pte_overflow--; 2235152180Sgrehan mtx_unlock(&moea_table_mutex); 223690643Sbenno return (1); 223790643Sbenno } 223890643Sbenno 223990643Sbenno source_pvo = pvo; 224090643Sbenno 224190643Sbenno if (victim_pvo != NULL) 224290643Sbenno break; 224390643Sbenno } 224490643Sbenno 224578880Sbenno /* 224690643Sbenno * We also need the pvo entry of the victim we are replacing 224790643Sbenno * so save the R & C bits of the PTE. 224878880Sbenno */ 224990643Sbenno if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 2250183290Snwhitehorn moea_pte_compare(pt, &pvo->pvo_pte.pte)) { 225190643Sbenno victim_pvo = pvo; 225290643Sbenno if (source_pvo != NULL) 225390643Sbenno break; 225490643Sbenno } 225590643Sbenno } 225678880Sbenno 2257134535Salc if (source_pvo == NULL) { 2258152180Sgrehan mtx_unlock(&moea_table_mutex); 225990643Sbenno return (0); 2260134535Salc } 226190643Sbenno 226290643Sbenno if (victim_pvo == NULL) { 226390643Sbenno if ((pt->pte_hi & PTE_HID) == 0) 2264152180Sgrehan panic("moea_pte_spill: victim p-pte (%p) has no pvo" 226590643Sbenno "entry", pt); 226690643Sbenno 226778880Sbenno /* 226890643Sbenno * If this is a secondary PTE, we need to search it's primary 226990643Sbenno * pvo bucket for the matching PVO. 227078880Sbenno */ 2271152180Sgrehan LIST_FOREACH(pvo, &moea_pvo_table[ptegidx ^ moea_pteg_mask], 227290643Sbenno pvo_olink) { 227390643Sbenno /* 227490643Sbenno * We also need the pvo entry of the victim we are 227590643Sbenno * replacing so save the R & C bits of the PTE. 227690643Sbenno */ 2277183290Snwhitehorn if (moea_pte_compare(pt, &pvo->pvo_pte.pte)) { 227890643Sbenno victim_pvo = pvo; 227990643Sbenno break; 228090643Sbenno } 228190643Sbenno } 228278880Sbenno 228390643Sbenno if (victim_pvo == NULL) 2284152180Sgrehan panic("moea_pte_spill: victim s-pte (%p) has no pvo" 228590643Sbenno "entry", pt); 228690643Sbenno } 228778880Sbenno 228890643Sbenno /* 228990643Sbenno * We are invalidating the TLB entry for the EA we are replacing even 229090643Sbenno * though it's valid. If we don't, we lose any ref/chg bit changes 229190643Sbenno * contained in the TLB entry. 229290643Sbenno */ 2293183290Snwhitehorn source_pvo->pvo_pte.pte.pte_hi &= ~PTE_HID; 229478880Sbenno 2295183290Snwhitehorn moea_pte_unset(pt, &victim_pvo->pvo_pte.pte, victim_pvo->pvo_vaddr); 2296183290Snwhitehorn moea_pte_set(pt, &source_pvo->pvo_pte.pte); 229790643Sbenno 229890643Sbenno PVO_PTEGIDX_CLR(victim_pvo); 229990643Sbenno PVO_PTEGIDX_SET(source_pvo, i); 2300152180Sgrehan moea_pte_replacements++; 230190643Sbenno 2302152180Sgrehan mtx_unlock(&moea_table_mutex); 230390643Sbenno return (1); 230490643Sbenno} 230590643Sbenno 2306253976Sjhibbitsstatic __inline struct pvo_entry * 2307253976Sjhibbitsmoea_pte_spillable_ident(u_int ptegidx) 2308253976Sjhibbits{ 2309253976Sjhibbits struct pte *pt; 2310253976Sjhibbits struct pvo_entry *pvo_walk, *pvo = NULL; 2311253976Sjhibbits 2312253976Sjhibbits LIST_FOREACH(pvo_walk, &moea_pvo_table[ptegidx], pvo_olink) { 2313253976Sjhibbits if (pvo_walk->pvo_vaddr & PVO_WIRED) 2314253976Sjhibbits continue; 2315253976Sjhibbits 2316253976Sjhibbits if (!(pvo_walk->pvo_pte.pte.pte_hi & PTE_VALID)) 2317253976Sjhibbits continue; 2318253976Sjhibbits 2319253976Sjhibbits pt = moea_pvo_to_pte(pvo_walk, -1); 2320253976Sjhibbits 2321253976Sjhibbits if (pt == NULL) 2322253976Sjhibbits continue; 2323253976Sjhibbits 2324253976Sjhibbits pvo = pvo_walk; 2325253976Sjhibbits 2326253976Sjhibbits mtx_unlock(&moea_table_mutex); 2327253976Sjhibbits if (!(pt->pte_lo & PTE_REF)) 2328253976Sjhibbits return (pvo_walk); 2329253976Sjhibbits } 2330253976Sjhibbits 2331253976Sjhibbits return (pvo); 2332253976Sjhibbits} 2333253976Sjhibbits 233490643Sbennostatic int 2335152180Sgrehanmoea_pte_insert(u_int ptegidx, struct pte *pvo_pt) 233690643Sbenno{ 233790643Sbenno struct pte *pt; 2338253976Sjhibbits struct pvo_entry *victim_pvo; 233990643Sbenno int i; 2340253976Sjhibbits int victim_idx; 2341253976Sjhibbits u_int pteg_bkpidx = ptegidx; 234290643Sbenno 2343159928Salc mtx_assert(&moea_table_mutex, MA_OWNED); 2344159928Salc 234590643Sbenno /* 234690643Sbenno * First try primary hash. 234790643Sbenno */ 2348152180Sgrehan for (pt = moea_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 234990643Sbenno if ((pt->pte_hi & PTE_VALID) == 0) { 235090643Sbenno pvo_pt->pte_hi &= ~PTE_HID; 2351152180Sgrehan moea_pte_set(pt, pvo_pt); 235290643Sbenno return (i); 235378880Sbenno } 235490643Sbenno } 235578880Sbenno 235690643Sbenno /* 235790643Sbenno * Now try secondary hash. 235890643Sbenno */ 2359152180Sgrehan ptegidx ^= moea_pteg_mask; 2360165362Sgrehan 2361152180Sgrehan for (pt = moea_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 236290643Sbenno if ((pt->pte_hi & PTE_VALID) == 0) { 236390643Sbenno pvo_pt->pte_hi |= PTE_HID; 2364152180Sgrehan moea_pte_set(pt, pvo_pt); 236590643Sbenno return (i); 236690643Sbenno } 236790643Sbenno } 236878880Sbenno 2369253976Sjhibbits /* Try again, but this time try to force a PTE out. */ 2370253976Sjhibbits ptegidx = pteg_bkpidx; 2371253976Sjhibbits 2372253976Sjhibbits victim_pvo = moea_pte_spillable_ident(ptegidx); 2373253976Sjhibbits if (victim_pvo == NULL) { 2374253976Sjhibbits ptegidx ^= moea_pteg_mask; 2375253976Sjhibbits victim_pvo = moea_pte_spillable_ident(ptegidx); 2376253976Sjhibbits } 2377253976Sjhibbits 2378253976Sjhibbits if (victim_pvo == NULL) { 2379253976Sjhibbits panic("moea_pte_insert: overflow"); 2380253976Sjhibbits return (-1); 2381253976Sjhibbits } 2382253976Sjhibbits 2383253976Sjhibbits victim_idx = moea_pvo_pte_index(victim_pvo, ptegidx); 2384253976Sjhibbits 2385253976Sjhibbits if (pteg_bkpidx == ptegidx) 2386253976Sjhibbits pvo_pt->pte_hi &= ~PTE_HID; 2387253976Sjhibbits else 2388253976Sjhibbits pvo_pt->pte_hi |= PTE_HID; 2389253976Sjhibbits 2390253976Sjhibbits /* 2391253976Sjhibbits * Synchronize the sacrifice PTE with its PVO, then mark both 2392253976Sjhibbits * invalid. The PVO will be reused when/if the VM system comes 2393253976Sjhibbits * here after a fault. 2394253976Sjhibbits */ 2395253976Sjhibbits pt = &moea_pteg_table[victim_idx >> 3].pt[victim_idx & 7]; 2396253976Sjhibbits 2397253976Sjhibbits if (pt->pte_hi != victim_pvo->pvo_pte.pte.pte_hi) 2398253976Sjhibbits panic("Victim PVO doesn't match PTE! PVO: %8x, PTE: %8x", victim_pvo->pvo_pte.pte.pte_hi, pt->pte_hi); 2399253976Sjhibbits 2400253976Sjhibbits /* 2401253976Sjhibbits * Set the new PTE. 2402253976Sjhibbits */ 2403253976Sjhibbits moea_pte_unset(pt, &victim_pvo->pvo_pte.pte, victim_pvo->pvo_vaddr); 2404253976Sjhibbits PVO_PTEGIDX_CLR(victim_pvo); 2405253976Sjhibbits moea_pte_overflow++; 2406253976Sjhibbits moea_pte_set(pt, pvo_pt); 2407253976Sjhibbits 2408253976Sjhibbits return (victim_idx & 7); 240978880Sbenno} 241084921Sbenno 241190643Sbennostatic boolean_t 2412152180Sgrehanmoea_query_bit(vm_page_t m, int ptebit) 241384921Sbenno{ 241490643Sbenno struct pvo_entry *pvo; 241590643Sbenno struct pte *pt; 241684921Sbenno 2417238357Salc rw_assert(&pvh_global_lock, RA_WLOCKED); 2418152180Sgrehan if (moea_attr_fetch(m) & ptebit) 241990643Sbenno return (TRUE); 242084921Sbenno 242190643Sbenno LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 242284921Sbenno 242390643Sbenno /* 242490643Sbenno * See if we saved the bit off. If so, cache it and return 242590643Sbenno * success. 242690643Sbenno */ 2427183290Snwhitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 2428152180Sgrehan moea_attr_save(m, ptebit); 242990643Sbenno return (TRUE); 243090643Sbenno } 243190643Sbenno } 243284921Sbenno 243390643Sbenno /* 243490643Sbenno * No luck, now go through the hard part of looking at the PTEs 243590643Sbenno * themselves. Sync so that any pending REF/CHG bits are flushed to 243690643Sbenno * the PTEs. 243790643Sbenno */ 2438183094Smarcel powerpc_sync(); 243990643Sbenno LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 244090643Sbenno 244190643Sbenno /* 244290643Sbenno * See if this pvo has a valid PTE. if so, fetch the 244390643Sbenno * REF/CHG bits from the valid PTE. If the appropriate 244490643Sbenno * ptebit is set, cache it and return success. 244590643Sbenno */ 2446152180Sgrehan pt = moea_pvo_to_pte(pvo, -1); 244790643Sbenno if (pt != NULL) { 2448183290Snwhitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 2449159928Salc mtx_unlock(&moea_table_mutex); 2450183290Snwhitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 2451152180Sgrehan moea_attr_save(m, ptebit); 245290643Sbenno return (TRUE); 245390643Sbenno } 245490643Sbenno } 245584921Sbenno } 245684921Sbenno 2457123354Sgallatin return (FALSE); 245884921Sbenno} 245990643Sbenno 2460110172Sgrehanstatic u_int 2461208990Salcmoea_clear_bit(vm_page_t m, int ptebit) 246290643Sbenno{ 2463110172Sgrehan u_int count; 246490643Sbenno struct pvo_entry *pvo; 246590643Sbenno struct pte *pt; 246690643Sbenno 2467238357Salc rw_assert(&pvh_global_lock, RA_WLOCKED); 2468208990Salc 246990643Sbenno /* 247090643Sbenno * Clear the cached value. 247190643Sbenno */ 2472152180Sgrehan moea_attr_clear(m, ptebit); 247390643Sbenno 247490643Sbenno /* 247590643Sbenno * Sync so that any pending REF/CHG bits are flushed to the PTEs (so 247690643Sbenno * we can reset the right ones). note that since the pvo entries and 247790643Sbenno * list heads are accessed via BAT0 and are never placed in the page 247890643Sbenno * table, we don't have to worry about further accesses setting the 247990643Sbenno * REF/CHG bits. 248090643Sbenno */ 2481183094Smarcel powerpc_sync(); 248290643Sbenno 248390643Sbenno /* 248490643Sbenno * For each pvo entry, clear the pvo's ptebit. If this pvo has a 248590643Sbenno * valid pte clear the ptebit from the valid pte. 248690643Sbenno */ 2487110172Sgrehan count = 0; 248890643Sbenno LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2489152180Sgrehan pt = moea_pvo_to_pte(pvo, -1); 249090643Sbenno if (pt != NULL) { 2491183290Snwhitehorn moea_pte_synch(pt, &pvo->pvo_pte.pte); 2492183290Snwhitehorn if (pvo->pvo_pte.pte.pte_lo & ptebit) { 2493110172Sgrehan count++; 2494152180Sgrehan moea_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2495110172Sgrehan } 2496159928Salc mtx_unlock(&moea_table_mutex); 249790643Sbenno } 2498183290Snwhitehorn pvo->pvo_pte.pte.pte_lo &= ~ptebit; 249990643Sbenno } 250090643Sbenno 2501110172Sgrehan return (count); 250290643Sbenno} 250399038Sbenno 250499038Sbenno/* 2505103604Sgrehan * Return true if the physical range is encompassed by the battable[idx] 2506103604Sgrehan */ 2507103604Sgrehanstatic int 2508152180Sgrehanmoea_bat_mapped(int idx, vm_offset_t pa, vm_size_t size) 2509103604Sgrehan{ 2510103604Sgrehan u_int prot; 2511103604Sgrehan u_int32_t start; 2512103604Sgrehan u_int32_t end; 2513103604Sgrehan u_int32_t bat_ble; 2514103604Sgrehan 2515103604Sgrehan /* 2516103604Sgrehan * Return immediately if not a valid mapping 2517103604Sgrehan */ 2518214601Snwhitehorn if (!(battable[idx].batu & BAT_Vs)) 2519103604Sgrehan return (EINVAL); 2520103604Sgrehan 2521103604Sgrehan /* 2522103604Sgrehan * The BAT entry must be cache-inhibited, guarded, and r/w 2523103604Sgrehan * so it can function as an i/o page 2524103604Sgrehan */ 2525103604Sgrehan prot = battable[idx].batl & (BAT_I|BAT_G|BAT_PP_RW); 2526103604Sgrehan if (prot != (BAT_I|BAT_G|BAT_PP_RW)) 2527103604Sgrehan return (EPERM); 2528103604Sgrehan 2529103604Sgrehan /* 2530103604Sgrehan * The address should be within the BAT range. Assume that the 2531103604Sgrehan * start address in the BAT has the correct alignment (thus 2532103604Sgrehan * not requiring masking) 2533103604Sgrehan */ 2534103604Sgrehan start = battable[idx].batl & BAT_PBS; 2535103604Sgrehan bat_ble = (battable[idx].batu & ~(BAT_EBS)) | 0x03; 2536103604Sgrehan end = start | (bat_ble << 15) | 0x7fff; 2537103604Sgrehan 2538103604Sgrehan if ((pa < start) || ((pa + size) > end)) 2539103604Sgrehan return (ERANGE); 2540103604Sgrehan 2541103604Sgrehan return (0); 2542103604Sgrehan} 2543103604Sgrehan 2544152180Sgrehanboolean_t 2545235936Srajmoea_dev_direct_mapped(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 2546133855Sssouhlal{ 2547133855Sssouhlal int i; 2548103604Sgrehan 2549133855Sssouhlal /* 2550133855Sssouhlal * This currently does not work for entries that 2551133855Sssouhlal * overlap 256M BAT segments. 2552133855Sssouhlal */ 2553133855Sssouhlal 2554133855Sssouhlal for(i = 0; i < 16; i++) 2555152180Sgrehan if (moea_bat_mapped(i, pa, size) == 0) 2556133855Sssouhlal return (0); 2557133855Sssouhlal 2558133855Sssouhlal return (EFAULT); 2559133855Sssouhlal} 2560133855Sssouhlal 2561103604Sgrehan/* 256299038Sbenno * Map a set of physical memory pages into the kernel virtual 256399038Sbenno * address space. Return a pointer to where it is mapped. This 256499038Sbenno * routine is intended to be used for mapping device memory, 256599038Sbenno * NOT real memory. 256699038Sbenno */ 256799038Sbennovoid * 2568235936Srajmoea_mapdev(mmu_t mmu, vm_paddr_t pa, vm_size_t size) 256999038Sbenno{ 2570213307Snwhitehorn 2571213307Snwhitehorn return (moea_mapdev_attr(mmu, pa, size, VM_MEMATTR_DEFAULT)); 2572213307Snwhitehorn} 2573213307Snwhitehorn 2574213307Snwhitehornvoid * 2575213307Snwhitehornmoea_mapdev_attr(mmu_t mmu, vm_offset_t pa, vm_size_t size, vm_memattr_t ma) 2576213307Snwhitehorn{ 2577103604Sgrehan vm_offset_t va, tmpva, ppa, offset; 2578103604Sgrehan int i; 2579103604Sgrehan 2580103604Sgrehan ppa = trunc_page(pa); 258199038Sbenno offset = pa & PAGE_MASK; 258299038Sbenno size = roundup(offset + size, PAGE_SIZE); 258399038Sbenno 2584103604Sgrehan /* 2585103604Sgrehan * If the physical address lies within a valid BAT table entry, 2586103604Sgrehan * return the 1:1 mapping. This currently doesn't work 2587103604Sgrehan * for regions that overlap 256M BAT segments. 2588103604Sgrehan */ 2589103604Sgrehan for (i = 0; i < 16; i++) { 2590152180Sgrehan if (moea_bat_mapped(i, pa, size) == 0) 2591103604Sgrehan return ((void *) pa); 2592103604Sgrehan } 2593103604Sgrehan 2594254025Sjeff va = kva_alloc(size); 259599038Sbenno if (!va) 2596152180Sgrehan panic("moea_mapdev: Couldn't alloc kernel virtual memory"); 259799038Sbenno 259899038Sbenno for (tmpva = va; size > 0;) { 2599213307Snwhitehorn moea_kenter_attr(mmu, tmpva, ppa, ma); 2600183094Smarcel tlbie(tmpva); 260199038Sbenno size -= PAGE_SIZE; 260299038Sbenno tmpva += PAGE_SIZE; 2603103604Sgrehan ppa += PAGE_SIZE; 260499038Sbenno } 260599038Sbenno 260699038Sbenno return ((void *)(va + offset)); 260799038Sbenno} 260899038Sbenno 260999038Sbennovoid 2610152180Sgrehanmoea_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) 261199038Sbenno{ 261299038Sbenno vm_offset_t base, offset; 261399038Sbenno 2614103604Sgrehan /* 2615103604Sgrehan * If this is outside kernel virtual space, then it's a 2616103604Sgrehan * battable entry and doesn't require unmapping 2617103604Sgrehan */ 2618204128Snwhitehorn if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= virtual_end)) { 2619103604Sgrehan base = trunc_page(va); 2620103604Sgrehan offset = va & PAGE_MASK; 2621103604Sgrehan size = roundup(offset + size, PAGE_SIZE); 2622254025Sjeff kva_free(base, size); 2623103604Sgrehan } 262499038Sbenno} 2625198341Smarcel 2626198341Smarcelstatic void 2627198341Smarcelmoea_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) 2628198341Smarcel{ 2629198341Smarcel struct pvo_entry *pvo; 2630198341Smarcel vm_offset_t lim; 2631198341Smarcel vm_paddr_t pa; 2632198341Smarcel vm_size_t len; 2633198341Smarcel 2634198341Smarcel PMAP_LOCK(pm); 2635198341Smarcel while (sz > 0) { 2636198341Smarcel lim = round_page(va); 2637198341Smarcel len = MIN(lim - va, sz); 2638198341Smarcel pvo = moea_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 2639198341Smarcel if (pvo != NULL) { 2640198341Smarcel pa = (pvo->pvo_pte.pte.pte_lo & PTE_RPGN) | 2641198341Smarcel (va & ADDR_POFF); 2642198341Smarcel moea_syncicache(pa, len); 2643198341Smarcel } 2644198341Smarcel va += len; 2645198341Smarcel sz -= len; 2646198341Smarcel } 2647198341Smarcel PMAP_UNLOCK(pm); 2648198341Smarcel} 2649249864Sjhibbits 2650249864Sjhibbitsvm_offset_t 2651249864Sjhibbitsmoea_dumpsys_map(mmu_t mmu, struct pmap_md *md, vm_size_t ofs, 2652249864Sjhibbits vm_size_t *sz) 2653249864Sjhibbits{ 2654249864Sjhibbits if (md->md_vaddr == ~0UL) 2655249864Sjhibbits return (md->md_paddr + ofs); 2656249864Sjhibbits else 2657249864Sjhibbits return (md->md_vaddr + ofs); 2658249864Sjhibbits} 2659249864Sjhibbits 2660249864Sjhibbitsstruct pmap_md * 2661249864Sjhibbitsmoea_scan_md(mmu_t mmu, struct pmap_md *prev) 2662249864Sjhibbits{ 2663249864Sjhibbits static struct pmap_md md; 2664249864Sjhibbits struct pvo_entry *pvo; 2665249864Sjhibbits vm_offset_t va; 2666249864Sjhibbits 2667249864Sjhibbits if (dumpsys_minidump) { 2668249864Sjhibbits md.md_paddr = ~0UL; /* Minidumps use virtual addresses. */ 2669249864Sjhibbits if (prev == NULL) { 2670249864Sjhibbits /* 1st: kernel .data and .bss. */ 2671249864Sjhibbits md.md_index = 1; 2672249864Sjhibbits md.md_vaddr = trunc_page((uintptr_t)_etext); 2673249864Sjhibbits md.md_size = round_page((uintptr_t)_end) - md.md_vaddr; 2674249864Sjhibbits return (&md); 2675249864Sjhibbits } 2676249864Sjhibbits switch (prev->md_index) { 2677249864Sjhibbits case 1: 2678249864Sjhibbits /* 2nd: msgbuf and tables (see pmap_bootstrap()). */ 2679249864Sjhibbits md.md_index = 2; 2680249864Sjhibbits md.md_vaddr = (vm_offset_t)msgbufp->msg_ptr; 2681249864Sjhibbits md.md_size = round_page(msgbufp->msg_size); 2682249864Sjhibbits break; 2683249864Sjhibbits case 2: 2684249864Sjhibbits /* 3rd: kernel VM. */ 2685249864Sjhibbits va = prev->md_vaddr + prev->md_size; 2686249864Sjhibbits /* Find start of next chunk (from va). */ 2687249864Sjhibbits while (va < virtual_end) { 2688249864Sjhibbits /* Don't dump the buffer cache. */ 2689249864Sjhibbits if (va >= kmi.buffer_sva && 2690249864Sjhibbits va < kmi.buffer_eva) { 2691249864Sjhibbits va = kmi.buffer_eva; 2692249864Sjhibbits continue; 2693249864Sjhibbits } 2694249864Sjhibbits pvo = moea_pvo_find_va(kernel_pmap, 2695249864Sjhibbits va & ~ADDR_POFF, NULL); 2696249864Sjhibbits if (pvo != NULL && 2697249864Sjhibbits (pvo->pvo_pte.pte.pte_hi & PTE_VALID)) 2698249864Sjhibbits break; 2699249864Sjhibbits va += PAGE_SIZE; 2700249864Sjhibbits } 2701249864Sjhibbits if (va < virtual_end) { 2702249864Sjhibbits md.md_vaddr = va; 2703249864Sjhibbits va += PAGE_SIZE; 2704249864Sjhibbits /* Find last page in chunk. */ 2705249864Sjhibbits while (va < virtual_end) { 2706249864Sjhibbits /* Don't run into the buffer cache. */ 2707249864Sjhibbits if (va == kmi.buffer_sva) 2708249864Sjhibbits break; 2709249864Sjhibbits pvo = moea_pvo_find_va(kernel_pmap, 2710249864Sjhibbits va & ~ADDR_POFF, NULL); 2711249864Sjhibbits if (pvo == NULL || 2712249864Sjhibbits !(pvo->pvo_pte.pte.pte_hi & PTE_VALID)) 2713249864Sjhibbits break; 2714249864Sjhibbits va += PAGE_SIZE; 2715249864Sjhibbits } 2716249864Sjhibbits md.md_size = va - md.md_vaddr; 2717249864Sjhibbits break; 2718249864Sjhibbits } 2719249864Sjhibbits md.md_index = 3; 2720249864Sjhibbits /* FALLTHROUGH */ 2721249864Sjhibbits default: 2722249864Sjhibbits return (NULL); 2723249864Sjhibbits } 2724249864Sjhibbits } else { /* minidumps */ 2725249864Sjhibbits mem_regions(&pregions, &pregions_sz, 2726249864Sjhibbits ®ions, ®ions_sz); 2727249864Sjhibbits 2728249864Sjhibbits if (prev == NULL) { 2729249864Sjhibbits /* first physical chunk. */ 2730249864Sjhibbits md.md_paddr = pregions[0].mr_start; 2731249864Sjhibbits md.md_size = pregions[0].mr_size; 2732249864Sjhibbits md.md_vaddr = ~0UL; 2733249864Sjhibbits md.md_index = 1; 2734249864Sjhibbits } else if (md.md_index < pregions_sz) { 2735249864Sjhibbits md.md_paddr = pregions[md.md_index].mr_start; 2736249864Sjhibbits md.md_size = pregions[md.md_index].mr_size; 2737249864Sjhibbits md.md_vaddr = ~0UL; 2738249864Sjhibbits md.md_index++; 2739249864Sjhibbits } else { 2740249864Sjhibbits /* There's no next physical chunk. */ 2741249864Sjhibbits return (NULL); 2742249864Sjhibbits } 2743249864Sjhibbits } 2744249864Sjhibbits 2745249864Sjhibbits return (&md); 2746249864Sjhibbits} 2747