1a2,36
> * Copyright (c) 2001 The NetBSD Foundation, Inc.
> * All rights reserved.
> *
> * This code is derived from software contributed to The NetBSD Foundation
> * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
> *
> * Redistribution and use in source and binary forms, with or without
> * modification, are permitted provided that the following conditions
> * are met:
> * 1. Redistributions of source code must retain the above copyright
> * notice, this list of conditions and the following disclaimer.
> * 2. Redistributions in binary form must reproduce the above copyright
> * notice, this list of conditions and the following disclaimer in the
> * documentation and/or other materials provided with the distribution.
> * 3. All advertising materials mentioning features or use of this software
> * must display the following acknowledgement:
> * This product includes software developed by the NetBSD
> * Foundation, Inc. and its contributors.
> * 4. Neither the name of The NetBSD Foundation nor the names of its
> * contributors may be used to endorse or promote products derived
> * from this software without specific prior written permission.
> *
> * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
> * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
> * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
> * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
> * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
> * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
> * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
> * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
> * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
> * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> * POSSIBILITY OF SUCH DAMAGE.
> */
> /*
60c95
< "$FreeBSD: head/sys/powerpc/aim/mmu_oea.c 85201 2001-10-19 22:45:46Z mp $";
---
> "$FreeBSD: head/sys/powerpc/aim/mmu_oea.c 90643 2002-02-14 01:39:11Z benno $";
62a98,118
> /*
> * Manages physical address maps.
> *
> * In addition to hardware address maps, this module is called upon to
> * provide software-use-only maps which may or may not be stored in the
> * same form as hardware maps. These pseudo-maps are used to store
> * intermediate results from copy operations to and from address spaces.
> *
> * Since the information managed by this module is also stored by the
> * logical address mapping module, this module may throw away valid virtual
> * to physical mappings at almost any time. However, invalidations of
> * mappings must be done as requested.
> *
> * In order to cope with hardware architectures which make virtual to
> * physical map invalidates expensive, this module may delay invalidate
> * reduced protection operations until such time as they are actually
> * necessary. This module is given full information as to which processors
> * are currently using which maps, and to when physical maps must be made
> * correct.
> */
>
64d119
< #include <sys/systm.h>
66,71c121
< #include <sys/proc.h>
< #include <sys/malloc.h>
< #include <sys/msgbuf.h>
< #include <sys/vmmeter.h>
< #include <sys/mman.h>
< #include <sys/queue.h>
---
> #include <sys/ktr.h>
72a123
> #include <sys/msgbuf.h>
73a125,128
> #include <sys/proc.h>
> #include <sys/sysctl.h>
> #include <sys/systm.h>
> #include <sys/vmmeter.h>
75c130,132
< #include <vm/vm.h>
---
> #include <dev/ofw/openfirm.h>
>
> #include <vm/vm.h>
86,87d142
< #include <sys/user.h>
<
89,90c144,146
< #include <machine/pcb.h>
< #include <machine/powerpc.h>
---
> #include <machine/frame.h>
> #include <machine/md_var.h>
> #include <machine/psl.h>
91a148
> #include <machine/sr.h>
93,96c150
< pte_t *ptable;
< int ptab_cnt;
< u_int ptab_mask;
< #define HTABSIZE (ptab_cnt * 64)
---
> #define PMAP_DEBUG
98c152
< #define MINPV 2048
---
> #define TODO panic("%s: not implemented", __func__);
100,102c154,184
< struct pte_ovfl {
< LIST_ENTRY(pte_ovfl) po_list; /* Linked list of overflow entries */
< struct pte po_pte; /* PTE for this mapping */
---
> #define PMAP_LOCK(pm)
> #define PMAP_UNLOCK(pm)
>
> #define TLBIE(va) __asm __volatile("tlbie %0" :: "r"(va))
> #define TLBSYNC() __asm __volatile("tlbsync");
> #define SYNC() __asm __volatile("sync");
> #define EIEIO() __asm __volatile("eieio");
>
> #define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
> #define VSID_TO_SR(vsid) ((vsid) & 0xf)
> #define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
>
> #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */
> #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */
> #define PVO_WIRED 0x0010 /* PVO entry is wired */
> #define PVO_MANAGED 0x0020 /* PVO entry is managed */
> #define PVO_EXECUTABLE 0x0040 /* PVO entry is executable */
> #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
> #define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE)
> #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
> #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
> #define PVO_PTEGIDX_CLR(pvo) \
> ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
> #define PVO_PTEGIDX_SET(pvo, i) \
> ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
>
> #define PMAP_PVO_CHECK(pvo)
>
> struct mem_region {
> vm_offset_t mr_start;
> vm_offset_t mr_size;
105c187,192
< LIST_HEAD(pte_ovtab, pte_ovfl) *potable; /* Overflow entries for ptable */
---
> struct ofw_map {
> vm_offset_t om_va;
> vm_size_t om_len;
> vm_offset_t om_pa;
> u_int om_mode;
> };
107,108c194
< static struct pmap kernel_pmap_store;
< pmap_t kernel_pmap;
---
> int pmap_bootstrapped = 0;
110,111c196,200
< static int npgs;
< static u_int nextavail;
---
> /*
> * Virtual and physical address of message buffer.
> */
> struct msgbuf *msgbufp;
> vm_offset_t msgbuf_phys;
113,115c202,206
< #ifndef MSGBUFADDR
< extern vm_offset_t msgbuf_paddr;
< #endif
---
> /*
> * Physical addresses of first and last available physical page.
> */
> vm_offset_t avail_start;
> vm_offset_t avail_end;
117c208,215
< static struct mem_region *mem, *avail;
---
> /*
> * Map of physical memory regions.
> */
> vm_offset_t phys_avail[128];
> u_int phys_avail_count;
> static struct mem_region regions[128];
> static struct ofw_map translations[128];
> static int translations_size;
119,122c217,222
< vm_offset_t avail_start;
< vm_offset_t avail_end;
< vm_offset_t virtual_avail;
< vm_offset_t virtual_end;
---
> /*
> * First and last available kernel virtual addresses.
> */
> vm_offset_t virtual_avail;
> vm_offset_t virtual_end;
> vm_offset_t kernel_vm_end;
124c224,228
< vm_offset_t kernel_vm_end;
---
> /*
> * Kernel pmap.
> */
> struct pmap kernel_pmap_store;
> extern struct pmap ofw_pmap;
126c230,235
< static int pmap_pagedaemon_waken = 0;
---
> /*
> * PTEG data.
> */
> static struct pteg *pmap_pteg_table;
> u_int pmap_pteg_count;
> u_int pmap_pteg_mask;
128c237,244
< extern unsigned int Maxmem;
---
> /*
> * PVO data.
> */
> struct pvo_head *pmap_pvo_table; /* pvo entries by pteg index */
> struct pvo_head pmap_pvo_kunmanaged =
> LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */
> struct pvo_head pmap_pvo_unmanaged =
> LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */
130c246,251
< #define ATTRSHFT 4
---
> vm_zone_t pmap_upvo_zone; /* zone for pvo entries for unmanaged pages */
> vm_zone_t pmap_mpvo_zone; /* zone for pvo entries for managed pages */
> struct vm_zone pmap_upvo_zone_store;
> struct vm_zone pmap_mpvo_zone_store;
> struct vm_object pmap_upvo_zone_obj;
> struct vm_object pmap_mpvo_zone_obj;
132c253,254
< struct pv_entry *pv_table;
---
> #define PMAP_PVO_SIZE 1024
> struct pvo_entry pmap_upvo_pool[PMAP_PVO_SIZE];
134,138c256,257
< static vm_zone_t pvzone;
< static struct vm_zone pvzone_store;
< static struct vm_object pvzone_obj;
< static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0;
< static struct pv_entry *pvinit;
---
> #define VSID_NBPW (sizeof(u_int32_t) * 8)
> static u_int pmap_vsid_bitmap[NPMAPS / VSID_NBPW];
140,142c259
< #if !defined(PMAP_SHPGPERPROC)
< #define PMAP_SHPGPERPROC 200
< #endif
---
> static boolean_t pmap_initialized = FALSE;
144,159c261,284
< struct pv_page;
< struct pv_page_info {
< LIST_ENTRY(pv_page) pgi_list;
< struct pv_entry *pgi_freelist;
< int pgi_nfree;
< };
< #define NPVPPG ((PAGE_SIZE - sizeof(struct pv_page_info)) / sizeof(struct pv_entry))
< struct pv_page {
< struct pv_page_info pvp_pgi;
< struct pv_entry pvp_pv[NPVPPG];
< };
< LIST_HEAD(pv_page_list, pv_page) pv_page_freelist;
< int pv_nfree;
< int pv_pcnt;
< static struct pv_entry *pmap_alloc_pv(void);
< static void pmap_free_pv(struct pv_entry *);
---
> /*
> * Statistics.
> */
> u_int pmap_pte_valid = 0;
> u_int pmap_pte_overflow = 0;
> u_int pmap_pte_replacements = 0;
> u_int pmap_pvo_entries = 0;
> u_int pmap_pvo_enter_calls = 0;
> u_int pmap_pvo_remove_calls = 0;
> u_int pmap_pte_spills = 0;
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_valid, CTLFLAG_RD, &pmap_pte_valid,
> 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_overflow, CTLFLAG_RD,
> &pmap_pte_overflow, 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_replacements, CTLFLAG_RD,
> &pmap_pte_replacements, 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_entries, CTLFLAG_RD, &pmap_pvo_entries,
> 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_enter_calls, CTLFLAG_RD,
> &pmap_pvo_enter_calls, 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_remove_calls, CTLFLAG_RD,
> &pmap_pvo_remove_calls, 0, "");
> SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_spills, CTLFLAG_RD,
> &pmap_pte_spills, 0, "");
161,177c286
< struct po_page;
< struct po_page_info {
< LIST_ENTRY(po_page) pgi_list;
< vm_page_t pgi_page;
< LIST_HEAD(po_freelist, pte_ovfl) pgi_freelist;
< int pgi_nfree;
< };
< #define NPOPPG ((PAGE_SIZE - sizeof(struct po_page_info)) / sizeof(struct pte_ovfl))
< struct po_page {
< struct po_page_info pop_pgi;
< struct pte_ovfl pop_po[NPOPPG];
< };
< LIST_HEAD(po_page_list, po_page) po_page_freelist;
< int po_nfree;
< int po_pcnt;
< static struct pte_ovfl *poalloc(void);
< static void pofree(struct pte_ovfl *, int);
---
> struct pvo_entry *pmap_pvo_zeropage;
179c288,289
< static u_int usedsr[NPMAPS / sizeof(u_int) / 8];
---
> vm_offset_t pmap_rkva_start = VM_MIN_KERNEL_ADDRESS;
> u_int pmap_rkva_count = 4;
181c291,294
< static int pmap_initialized;
---
> /*
> * Allocate physical memory for use in pmap_bootstrap.
> */
> static vm_offset_t pmap_bootstrap_alloc(vm_size_t, u_int);
183c296,299
< int pte_spill(vm_offset_t);
---
> /*
> * PTE calls.
> */
> static int pmap_pte_insert(u_int, struct pte *);
186,187c302
< * These small routines may have to be replaced,
< * if/when we support processors other that the 604.
---
> * PVO calls.
189,191c304,308
< static __inline void
< tlbie(vm_offset_t ea)
< {
---
> static int pmap_pvo_enter(pmap_t, vm_zone_t, struct pvo_head *,
> vm_offset_t, vm_offset_t, u_int, int);
> static void pmap_pvo_remove(struct pvo_entry *, int);
> static struct pvo_entry *pmap_pvo_find_va(pmap_t, vm_offset_t, int *);
> static struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int);
193c310,325
< __asm __volatile ("tlbie %0" :: "r"(ea));
---
> /*
> * Utility routines.
> */
> static struct pvo_entry *pmap_rkva_alloc(void);
> static void pmap_pa_map(struct pvo_entry *, vm_offset_t,
> struct pte *, int *);
> static void pmap_pa_unmap(struct pvo_entry *, struct pte *, int *);
> static void pmap_syncicache(vm_offset_t, vm_size_t);
> static boolean_t pmap_query_bit(vm_page_t, int);
> static boolean_t pmap_clear_bit(vm_page_t, int);
> static void tlbia(void);
>
> static __inline int
> va_to_sr(u_int *sr, vm_offset_t va)
> {
> return (sr[(uintptr_t)va >> ADDR_SR_SHFT]);
196,197c328,329
< static __inline void
< tlbsync(void)
---
> static __inline u_int
> va_to_pteg(u_int sr, vm_offset_t addr)
198a331
> u_int hash;
200c333,335
< __asm __volatile ("sync; tlbsync; sync");
---
> hash = (sr & SR_VSID_MASK) ^ (((u_int)addr & ADDR_PIDX) >>
> ADDR_PIDX_SHFT);
> return (hash & pmap_pteg_mask);
203,204c338,339
< static __inline void
< tlbia(void)
---
> static __inline struct pvo_head *
> pa_to_pvoh(vm_offset_t pa)
206,212c341,348
< vm_offset_t i;
<
< __asm __volatile ("sync");
< for (i = 0; i < (vm_offset_t)0x00040000; i += 0x00001000) {
< tlbie(i);
< }
< tlbsync();
---
> struct vm_page *pg;
>
> pg = PHYS_TO_VM_PAGE(pa);
>
> if (pg == NULL)
> return (&pmap_pvo_unmanaged);
>
> return (&pg->md.mdpg_pvoh);
215,216c351,352
< static __inline int
< ptesr(sr_t *sr, vm_offset_t addr)
---
> static __inline struct pvo_head *
> vm_page_to_pvoh(vm_page_t m)
219c355
< return sr[(u_int)addr >> ADDR_SR_SHFT];
---
> return (&m->md.mdpg_pvoh);
222,223c358,359
< static __inline int
< pteidx(sr_t sr, vm_offset_t addr)
---
> static __inline void
> pmap_attr_clear(vm_page_t m, int ptebit)
225,228c361,362
< int hash;
<
< hash = (sr & SR_VSID) ^ (((u_int)addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);
< return hash & ptab_mask;
---
>
> m->md.mdpg_attrs &= ~ptebit;
232c366
< ptematch(pte_t *ptp, sr_t sr, vm_offset_t va, int which)
---
> pmap_attr_fetch(vm_page_t m)
235,236c369
< return ptp->pte_hi == (((sr & SR_VSID) << PTE_VSID_SHFT) |
< (((u_int)va >> ADDR_API_SHFT) & PTE_API) | which);
---
> return (m->md.mdpg_attrs);
239,240c372,373
< static __inline struct pv_entry *
< pa_to_pv(vm_offset_t pa)
---
> static __inline void
> pmap_attr_save(vm_page_t m, int ptebit)
242,243d374
< #if 0 /* XXX */
< int bank, pg;
245,250c376
< bank = vm_physseg_find(atop(pa), &pg);
< if (bank == -1)
< return NULL;
< return &vm_physmem[bank].pmseg.pvent[pg];
< #endif
< return (NULL);
---
> m->md.mdpg_attrs |= ptebit;
253,254c379,380
< static __inline char *
< pa_to_attr(vm_offset_t pa)
---
> static __inline int
> pmap_pte_compare(const struct pte *pt, const struct pte *pvo_pt)
256,257c382,383
< #if 0 /* XXX */
< int bank, pg;
---
> if (pt->pte_hi == pvo_pt->pte_hi)
> return (1);
259,264c385
< bank = vm_physseg_find(atop(pa), &pg);
< if (bank == -1)
< return NULL;
< return &vm_physmem[bank].pmseg.attrs[pg];
< #endif
< return (NULL);
---
> return (0);
267,274c388,389
< /*
< * Try to insert page table entry *pt into the ptable at idx.
< *
< * Note: *pt mustn't have PTE_VALID set.
< * This is done here as required by Book III, 4.12.
< */
< static int
< pte_insert(int idx, pte_t *pt)
---
> static __inline int
> pmap_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which)
276,277c391,394
< pte_t *ptp;
< int i;
---
> return (pt->pte_hi & ~PTE_VALID) ==
> (((sr & SR_VSID_MASK) << PTE_VSID_SHFT) |
> ((va >> ADDR_API_SHFT) & PTE_API) | which);
> }
278a396,398
> static __inline void
> pmap_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo)
> {
280c400,403
< * First try primary hash.
---
> * Construct a PTE. Default to IMB initially. Valid bit only gets
> * set when the real pte is set in memory.
> *
> * Note: Don't set the valid bit for correct operation of tlb update.
282,290c405,408
< for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++) {
< if (!(ptp->pte_hi & PTE_VALID)) {
< *ptp = *pt;
< ptp->pte_hi &= ~PTE_HID;
< __asm __volatile ("sync");
< ptp->pte_hi |= PTE_VALID;
< return 1;
< }
< }
---
> pt->pte_hi = ((sr & SR_VSID_MASK) << PTE_VSID_SHFT) |
> (((va & ADDR_PIDX) >> ADDR_API_SHFT) & PTE_API);
> pt->pte_lo = pte_lo;
> }
292,294c410,412
< /*
< * Then try secondary hash.
< */
---
> static __inline void
> pmap_pte_synch(struct pte *pt, struct pte *pvo_pt)
> {
296c414,415
< idx ^= ptab_mask;
---
> pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG);
> }
298,306c417,419
< for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++) {
< if (!(ptp->pte_hi & PTE_VALID)) {
< *ptp = *pt;
< ptp->pte_hi |= PTE_HID;
< __asm __volatile ("sync");
< ptp->pte_hi |= PTE_VALID;
< return 1;
< }
< }
---
> static __inline void
> pmap_pte_clear(struct pte *pt, vm_offset_t va, int ptebit)
> {
308c421,428
< return 0;
---
> /*
> * As shown in Section 7.6.3.2.3
> */
> pt->pte_lo &= ~ptebit;
> TLBIE(va);
> EIEIO();
> TLBSYNC();
> SYNC();
311,319c431,432
< /*
< * Spill handler.
< *
< * Tries to spill a page table entry from the overflow area.
< * Note that this routine runs in real mode on a separate stack,
< * with interrupts disabled.
< */
< int
< pte_spill(vm_offset_t addr)
---
> static __inline void
> pmap_pte_set(struct pte *pt, struct pte *pvo_pt)
321,325d433
< int idx, i;
< sr_t sr;
< struct pte_ovfl *po;
< pte_t ps;
< pte_t *pt;
327,370c435
< __asm ("mfsrin %0,%1" : "=r"(sr) : "r"(addr));
< idx = pteidx(sr, addr);
< for (po = potable[idx].lh_first; po; po = po->po_list.le_next) {
< if (ptematch(&po->po_pte, sr, addr, 0)) {
< /*
< * Now found an entry to be spilled into the real
< * ptable.
< */
< if (pte_insert(idx, &po->po_pte)) {
< LIST_REMOVE(po, po_list);
< pofree(po, 0);
< return 1;
< }
< /*
< * Have to substitute some entry. Use the primary
< * hash for this.
< *
< * Use low bits of timebase as random generator
< */
< __asm ("mftb %0" : "=r"(i));
< pt = ptable + idx * 8 + (i & 7);
< pt->pte_hi &= ~PTE_VALID;
< ps = *pt;
< __asm __volatile ("sync");
< tlbie(addr);
< tlbsync();
< *pt = po->po_pte;
< __asm __volatile ("sync");
< pt->pte_hi |= PTE_VALID;
< po->po_pte = ps;
< if (ps.pte_hi & PTE_HID) {
< /*
< * We took an entry that was on the alternate
< * hash chain, so move it to it's original
< * chain.
< */
< po->po_pte.pte_hi &= ~PTE_HID;
< LIST_REMOVE(po, po_list);
< LIST_INSERT_HEAD(potable + (idx ^ ptab_mask),
< po, po_list);
< }
< return 1;
< }
< }
---
> pvo_pt->pte_hi |= PTE_VALID;
372c437,446
< return 0;
---
> /*
> * Update the PTE as defined in section 7.6.3.1.
> * Note that the REF/CHG bits are from pvo_pt and thus should havce
> * been saved so this routine can restore them (if desired).
> */
> pt->pte_lo = pvo_pt->pte_lo;
> EIEIO();
> pt->pte_hi = pvo_pt->pte_hi;
> SYNC();
> pmap_pte_valid++;
375,379c449,450
< /*
< * This is called during powerpc_init, before the system is really initialized.
< */
< void
< pmap_setavailmem(u_int kernelstart, u_int kernelend)
---
> static __inline void
> pmap_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va)
381,383d451
< struct mem_region *mp, *mp1;
< int cnt, i;
< u_int s, e, sz;
384a453,454
> pvo_pt->pte_hi &= ~PTE_VALID;
>
386c456
< * Get memory.
---
> * Force the reg & chg bits back into the PTEs.
388,390c458
< mem_regions(&mem, &avail);
< for (mp = mem; mp->size; mp++)
< Maxmem += btoc(mp->size);
---
> SYNC();
393c461
< * Count the number of available entries.
---
> * Invalidate the pte.
395,397c463
< for (cnt = 0, mp = avail; mp->size; mp++) {
< cnt++;
< }
---
> pt->pte_hi &= ~PTE_VALID;
398a465,470
> SYNC();
> TLBIE(va);
> EIEIO();
> TLBSYNC();
> SYNC();
>
400,402c472
< * Page align all regions.
< * Non-page aligned memory isn't very interesting to us.
< * Also, sort the entries for ascending addresses.
---
> * Save the reg & chg bits.
404,452c474,476
< kernelstart &= ~PAGE_MASK;
< kernelend = (kernelend + PAGE_MASK) & ~PAGE_MASK;
< for (mp = avail; mp->size; mp++) {
< s = mp->start;
< e = mp->start + mp->size;
< /*
< * Check whether this region holds all of the kernel.
< */
< if (s < kernelstart && e > kernelend) {
< avail[cnt].start = kernelend;
< avail[cnt++].size = e - kernelend;
< e = kernelstart;
< }
< /*
< * Look whether this regions starts within the kernel.
< */
< if (s >= kernelstart && s < kernelend) {
< if (e <= kernelend)
< goto empty;
< s = kernelend;
< }
< /*
< * Now look whether this region ends within the kernel.
< */
< if (e > kernelstart && e <= kernelend) {
< if (s >= kernelstart)
< goto empty;
< e = kernelstart;
< }
< /*
< * Now page align the start and size of the region.
< */
< s = round_page(s);
< e = trunc_page(e);
< if (e < s) {
< e = s;
< }
< sz = e - s;
< /*
< * Check whether some memory is left here.
< */
< if (sz == 0) {
< empty:
< bcopy(mp + 1, mp,
< (cnt - (mp - avail)) * sizeof *mp);
< cnt--;
< mp--;
< continue;
< }
---
> pmap_pte_synch(pt, pvo_pt);
> pmap_pte_valid--;
> }
454,457c478,480
< /*
< * Do an insertion sort.
< */
< npgs += btoc(sz);
---
> static __inline void
> pmap_pte_change(struct pte *pt, struct pte *pvo_pt, vm_offset_t va)
> {
459,489d481
< for (mp1 = avail; mp1 < mp; mp1++) {
< if (s < mp1->start) {
< break;
< }
< }
<
< if (mp1 < mp) {
< bcopy(mp1, mp1 + 1, (char *)mp - (char *)mp1);
< mp1->start = s;
< mp1->size = sz;
< } else {
< mp->start = s;
< mp->size = sz;
< }
< }
<
< #ifdef HTABENTS
< ptab_cnt = HTABENTS;
< #else
< ptab_cnt = (Maxmem + 1) / 2;
<
< /* The minimum is 1024 PTEGs. */
< if (ptab_cnt < 1024) {
< ptab_cnt = 1024;
< }
<
< /* Round up to power of 2. */
< __asm ("cntlzw %0,%1" : "=r"(i) : "r"(ptab_cnt - 1));
< ptab_cnt = 1 << (32 - i);
< #endif
<
491c483
< * Find suitably aligned memory for HTAB.
---
> * Invalidate the PTE
493,494c485,487
< for (mp = avail; mp->size; mp++) {
< s = roundup(mp->start, HTABSIZE) - mp->start;
---
> pmap_pte_unset(pt, pvo_pt, va);
> pmap_pte_set(pt, pvo_pt);
> }
496,498c489,493
< if (mp->size < s + HTABSIZE) {
< continue;
< }
---
> /*
> * Quick sort callout for comparing memory regions.
> */
> static int mr_cmp(const void *a, const void *b);
> static int om_cmp(const void *a, const void *b);
500c495,499
< ptable = (pte_t *)(mp->start + s);
---
> static int
> mr_cmp(const void *a, const void *b)
> {
> const struct mem_region *regiona;
> const struct mem_region *regionb;
502,511c501,509
< if (mp->size == s + HTABSIZE) {
< if (s)
< mp->size = s;
< else {
< bcopy(mp + 1, mp,
< (cnt - (mp - avail)) * sizeof *mp);
< mp = avail;
< }
< break;
< }
---
> regiona = a;
> regionb = b;
> if (regiona->mr_start < regionb->mr_start)
> return (-1);
> else if (regiona->mr_start > regionb->mr_start)
> return (1);
> else
> return (0);
> }
513,518c511,515
< if (s != 0) {
< bcopy(mp, mp + 1,
< (cnt - (mp - avail)) * sizeof *mp);
< mp++->size = s;
< cnt++;
< }
---
> static int
> om_cmp(const void *a, const void *b)
> {
> const struct ofw_map *mapa;
> const struct ofw_map *mapb;
520,523c517,525
< mp->start += s + HTABSIZE;
< mp->size -= s + HTABSIZE;
< break;
< }
---
> mapa = a;
> mapb = b;
> if (mapa->om_pa < mapb->om_pa)
> return (-1);
> else if (mapa->om_pa > mapb->om_pa)
> return (1);
> else
> return (0);
> }
525,527c527,536
< if (!mp->size) {
< panic("not enough memory?");
< }
---
> void
> pmap_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
> {
> ihandle_t pmem, mmui;
> phandle_t chosen, mmu;
> int sz;
> int i, j;
> vm_size_t size;
> vm_offset_t pa, va, off;
> u_int batl, batu;
529,531c538,551
< npgs -= btoc(HTABSIZE);
< bzero((void *)ptable, HTABSIZE);
< ptab_mask = ptab_cnt - 1;
---
> /*
> * Use an IBAT and a DBAT to map the bottom segment of memory
> * where we are.
> */
> batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs);
> batl = BATL(0x00000000, BAT_M, BAT_PP_RW);
> __asm ("mtibatu 0,%0; mtibatl 0,%1; mtdbatu 0,%0; mtdbatl 0,%1"
> :: "r"(batu), "r"(batl));
> #if 0
> batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs);
> batl = BATL(0x80000000, BAT_M, BAT_PP_RW);
> __asm ("mtibatu 1,%0; mtibatl 1,%1; mtdbatu 1,%0; mtdbatl 1,%1"
> :: "r"(batu), "r"(batl));
> #endif
534,535c554
< * We cannot do pmap_steal_memory here,
< * since we don't run with translation enabled yet.
---
> * Set the start and end of kva.
537,538c556,557
< s = sizeof(struct pte_ovtab) * ptab_cnt;
< sz = round_page(s);
---
> virtual_avail = VM_MIN_KERNEL_ADDRESS;
> virtual_end = VM_MAX_KERNEL_ADDRESS;
540,543c559,580
< for (mp = avail; mp->size; mp++) {
< if (mp->size >= sz) {
< break;
< }
---
> if ((pmem = OF_finddevice("/memory")) == -1)
> panic("pmap_bootstrap: can't locate memory device");
> if ((sz = OF_getproplen(pmem, "available")) == -1)
> panic("pmap_bootstrap: can't get length of available memory");
> if (sizeof(phys_avail) < sz)
> panic("pmap_bootstrap: phys_avail too small");
> if (sizeof(regions) < sz)
> panic("pmap_bootstrap: regions too small");
> bzero(regions, sz);
> if (OF_getprop(pmem, "available", regions, sz) == -1)
> panic("pmap_bootstrap: can't get available memory");
> sz /= sizeof(*regions);
> CTR0(KTR_PMAP, "pmap_bootstrap: physical memory");
> qsort(regions, sz, sizeof(*regions), mr_cmp);
> phys_avail_count = 0;
> for (i = 0, j = 0; i < sz; i++, j += 2) {
> CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start,
> regions[i].mr_start + regions[i].mr_size,
> regions[i].mr_size);
> phys_avail[j] = regions[i].mr_start;
> phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
> phys_avail_count++;
546,548c583,589
< if (!mp->size) {
< panic("not enough memory?");
< }
---
> /*
> * Allocate PTEG table.
> */
> #ifdef PTEGCOUNT
> pmap_pteg_count = PTEGCOUNT;
> #else
> pmap_pteg_count = 0x1000;
550,553c591,592
< npgs -= btoc(sz);
< potable = (struct pte_ovtab *)mp->start;
< mp->size -= sz;
< mp->start += sz;
---
> while (pmap_pteg_count < physmem)
> pmap_pteg_count <<= 1;
555,557c594,595
< if (mp->size <= 0) {
< bcopy(mp + 1, mp, (cnt - (mp - avail)) * sizeof *mp);
< }
---
> pmap_pteg_count >>= 1;
> #endif /* PTEGCOUNT */
559,561c597,603
< for (i = 0; i < ptab_cnt; i++) {
< LIST_INIT(potable + i);
< }
---
> size = pmap_pteg_count * sizeof(struct pteg);
> CTR2(KTR_PMAP, "pmap_bootstrap: %d PTEGs, %d bytes", pmap_pteg_count,
> size);
> pmap_pteg_table = (struct pteg *)pmap_bootstrap_alloc(size, size);
> CTR1(KTR_PMAP, "pmap_bootstrap: PTEG table at %p", pmap_pteg_table);
> bzero((void *)pmap_pteg_table, pmap_pteg_count * sizeof(struct pteg));
> pmap_pteg_mask = pmap_pteg_count - 1;
563d604
< #ifndef MSGBUFADDR
565c606
< * allow for msgbuf
---
> * Allocate PTE overflow lists.
567,568c608,613
< sz = round_page(MSGBUFSIZE);
< mp = NULL;
---
> size = sizeof(struct pvo_head) * pmap_pteg_count;
> pmap_pvo_table = (struct pvo_head *)pmap_bootstrap_alloc(size,
> PAGE_SIZE);
> CTR1(KTR_PMAP, "pmap_bootstrap: PVO table at %p", pmap_pvo_table);
> for (i = 0; i < pmap_pteg_count; i++)
> LIST_INIT(&pmap_pvo_table[i]);
570,574c615,618
< for (mp1 = avail; mp1->size; mp1++) {
< if (mp1->size >= sz) {
< mp = mp1;
< }
< }
---
> /*
> * Allocate the message buffer.
> */
> msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE, 0);
576,578c620,625
< if (mp == NULL) {
< panic("not enough memory?");
< }
---
> /*
> * Initialise the unmanaged pvo pool.
> */
> pmap_upvo_zone = &pmap_upvo_zone_store;
> zbootinit(pmap_upvo_zone, "unmanaged pvo", sizeof (struct pvo_entry),
> pmap_upvo_pool, PMAP_PVO_SIZE);
580,582c627,632
< npgs -= btoc(sz);
< msgbuf_paddr = mp->start + mp->size - sz;
< mp->size -= sz;
---
> /*
> * Make sure kernel vsid is allocated as well as VSID 0.
> */
> pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW]
> |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
> pmap_vsid_bitmap[0] |= 1;
584,605d633
< if (mp->size <= 0) {
< bcopy(mp + 1, mp, (cnt - (mp - avail)) * sizeof *mp);
< }
< #endif
<
< nextavail = avail->start;
< avail_start = avail->start;
< for (mp = avail, i = 0; mp->size; mp++) {
< avail_end = mp->start + mp->size;
< phys_avail[i++] = mp->start;
< phys_avail[i++] = mp->start + mp->size;
< }
<
<
< }
<
< void
< pmap_bootstrap()
< {
< int i;
< u_int32_t batl, batu;
<
607c635
< * Initialize kernel pmap and hardware.
---
> * Set up the OpenFirmware pmap and add it's mappings.
609c637,656
< kernel_pmap = &kernel_pmap_store;
---
> pmap_pinit(&ofw_pmap);
> ofw_pmap.pm_sr[KERNEL_SR] = KERNEL_SEGMENT;
> if ((chosen = OF_finddevice("/chosen")) == -1)
> panic("pmap_bootstrap: can't find /chosen");
> OF_getprop(chosen, "mmu", &mmui, 4);
> if ((mmu = OF_instance_to_package(mmui)) == -1)
> panic("pmap_bootstrap: can't get mmu package");
> if ((sz = OF_getproplen(mmu, "translations")) == -1)
> panic("pmap_bootstrap: can't get ofw translation count");
> if (sizeof(translations) < sz)
> panic("pmap_bootstrap: translations too small");
> bzero(translations, sz);
> if (OF_getprop(mmu, "translations", translations, sz) == -1)
> panic("pmap_bootstrap: can't get ofw translations");
> CTR0(KTR_PMAP, "pmap_bootstrap: translations");
> qsort(translations, sz, sizeof (*translations), om_cmp);
> for (i = 0; i < sz; i++) {
> CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x",
> translations[i].om_pa, translations[i].om_va,
> translations[i].om_len);
611,613c658
< batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs);
< batl = BATL(0x80000000, BAT_M, BAT_PP_RW);
< __asm ("mtdbatu 1,%0; mtdbatl 1,%1" :: "r" (batu), "r" (batl));
---
> /* Drop stuff below something? */
615,617c660,670
< #if NPMAPS >= KERNEL_SEGMENT / 16
< usedsr[KERNEL_SEGMENT / 16 / (sizeof usedsr[0] * 8)]
< |= 1 << ((KERNEL_SEGMENT / 16) % (sizeof usedsr[0] * 8));
---
> /* Enter the pages? */
> for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
> struct vm_page m;
>
> m.phys_addr = translations[i].om_pa + off;
> pmap_enter(&ofw_pmap, translations[i].om_va + off, &m,
> VM_PROT_ALL, 1);
> }
> }
> #ifdef SMP
> TLBSYNC();
620c673,675
< #if 0 /* XXX */
---
> /*
> * Initialize the kernel pmap (which is statically allocated).
> */
623,624d677
< __asm __volatile ("mtsrin %0,%1"
< :: "r"(EMPTY_SEGMENT), "r"(i << ADDR_SR_SHFT));
626c679,681
< #endif
---
> kernel_pmap->pm_sr[KERNEL_SR] = KERNEL_SEGMENT;
> kernel_pmap->pm_active = ~0;
> kernel_pmap->pm_count = 1;
628,629c683,698
< for (i = 0; i < 16; i++) {
< int j;
---
> /*
> * Allocate a kernel stack with a guard page for thread0 and map it
> * into the kernel page map.
> */
> pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, 0);
> kstack0_phys = pa;
> kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE);
> CTR2(KTR_PMAP, "pmap_bootstrap: kstack0 at %#x (%#x)", kstack0_phys,
> kstack0);
> virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE;
> for (i = 0; i < KSTACK_PAGES; i++) {
> pa = kstack0_phys + i * PAGE_SIZE;
> va = kstack0 + i * PAGE_SIZE;
> pmap_kenter(va, pa);
> TLBIE(va);
> }
631,633c700,707
< __asm __volatile ("mfsrin %0,%1"
< : "=r" (j)
< : "r" (i << ADDR_SR_SHFT));
---
> /*
> * Calculate the first and last available physical addresses.
> */
> avail_start = phys_avail[0];
> for (i = 0; phys_avail[i + 2] != 0; i += 2)
> ;
> avail_end = phys_avail[i + 1];
> Maxmem = powerpc_btop(avail_end);
635,636c709,713
< kernel_pmap->pm_sr[i] = j;
< }
---
> /*
> * Allocate virtual address space for the message buffer.
> */
> msgbufp = (struct msgbuf *)virtual_avail;
> virtual_avail += round_page(MSGBUF_SIZE);
638c715,721
< kernel_pmap->pm_sr[KERNEL_SR] = KERNEL_SEGMENT;
---
> /*
> * Initialize hardware.
> */
> for (i = 0; i < 16; i++) {
> __asm __volatile("mtsrin %0,%1"
> :: "r"(EMPTY_SEGMENT), "r"(i << ADDR_SR_SHFT));
> }
640,641c723
< :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT));
<
---
> :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT));
643,644c725
< :: "r"((u_int)ptable | (ptab_mask >> 10)));
<
---
> :: "r"((u_int)pmap_pteg_table | (pmap_pteg_mask >> 10)));
647,648c728
< virtual_avail = VM_MIN_KERNEL_ADDRESS;
< virtual_end = VM_MAX_KERNEL_ADDRESS;
---
> pmap_bootstrapped++;
652,653c732,733
< * Initialize anything else for pmap handling.
< * Called during vm_init().
---
> * Activate a user pmap. The pmap must be activated before it's address
> * space can be accessed in any way.
656c736
< pmap_init(vm_offset_t phys_start, vm_offset_t phys_end)
---
> pmap_activate(struct thread *td)
658c738,739
< int initial_pvs;
---
> pmap_t pm;
> int i;
661c742,743
< * init the pv free list
---
> * Load all the data we need up front to encourasge the compiler to
> * not issue any loads while we have interrupts disabled below.
663,671c745
< initial_pvs = vm_page_array_size;
< if (initial_pvs < MINPV) {
< initial_pvs = MINPV;
< }
< pvzone = &pvzone_store;
< pvinit = (struct pv_entry *) kmem_alloc(kernel_map,
< initial_pvs * sizeof(struct pv_entry));
< zbootinit(pvzone, "PV ENTRY", sizeof(struct pv_entry), pvinit,
< vm_page_array_size);
---
> pm = &td->td_proc->p_vmspace->vm_pmap;
673,674c747
< pmap_initialized = TRUE;
< }
---
> KASSERT(pm->pm_active == 0, ("pmap_activate: pmap already active?"));
676,682c749
< /*
< * Initialize a preallocated and zeroed pmap structure.
< */
< void
< pmap_pinit(struct pmap *pm)
< {
< int i, j;
---
> pm->pm_active |= PCPU_GET(cpumask);
685c752
< * Allocate some segment registers for this pmap.
---
> * XXX: Address this again later?
687,697c754,757
< pm->pm_refs = 1;
< for (i = 0; i < sizeof usedsr / sizeof usedsr[0]; i++) {
< if (usedsr[i] != 0xffffffff) {
< j = ffs(~usedsr[i]) - 1;
< usedsr[i] |= 1 << j;
< pm->pm_sr[0] = (i * sizeof usedsr[0] * 8 + j) * 16;
< for (i = 1; i < 16; i++) {
< pm->pm_sr[i] = pm->pm_sr[i - 1] + 1;
< }
< return;
< }
---
> critical_enter();
>
> for (i = 0; i < 16; i++) {
> __asm __volatile("mtsr %0,%1" :: "r"(i), "r"(pm->pm_sr[i]));
699c759,761
< panic("out of segments");
---
> __asm __volatile("sync; isync");
>
> critical_exit();
702,703c764,765
< void
< pmap_pinit2(pmap_t pmap)
---
> vm_offset_t
> pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size)
705,709c767,768
<
< /*
< * Nothing to be done.
< */
< return;
---
> TODO;
> return (0);
712,714d770
< /*
< * Add a reference to the given pmap.
< */
716c772
< pmap_reference(struct pmap *pm)
---
> pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
718,719c774
<
< pm->pm_refs++;
---
> TODO;
722,725d776
< /*
< * Retire the given pmap from service.
< * Should only be called if the map contains no valid mappings.
< */
727c778
< pmap_destroy(struct pmap *pm)
---
> pmap_clear_modify(vm_page_t m)
730,733c781,783
< if (--pm->pm_refs == 0) {
< pmap_release(pm);
< free((caddr_t)pm, M_VMPGDATA);
< }
---
> if (m->flags * PG_FICTITIOUS)
> return;
> pmap_clear_bit(m, PTE_CHG);
736,739d785
< /*
< * Release any resources held by the given physical map.
< * Called when a pmap initialized by pmap_pinit is being released.
< */
741c787
< pmap_release(struct pmap *pm)
---
> pmap_collect(void)
743,751c789
< int i, j;
<
< if (!pm->pm_sr[0]) {
< panic("pmap_release");
< }
< i = pm->pm_sr[0] / 16;
< j = i % (sizeof usedsr[0] * 8);
< i /= sizeof usedsr[0] * 8;
< usedsr[i] &= ~(1 << j);
---
> TODO;
754,760d791
< /*
< * Copy the range specified by src_addr/len
< * from the source map to the range dst_addr/len
< * in the destination map.
< *
< * This routine is only advisory and need not do anything.
< */
762,763c793,794
< pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vm_offset_t dst_addr,
< vm_size_t len, vm_offset_t src_addr)
---
> pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
> vm_size_t len, vm_offset_t src_addr)
765,766c796
<
< return;
---
> TODO;
769,776d798
< /*
< * Garbage collects the physical map system for
< * pages which are no longer used.
< * Success need not be guaranteed -- that is, there
< * may well be pages which are not referenced, but
< * others may be collected.
< * Called by the pageout daemon when pages are scarce.
< */
778c800
< pmap_collect(void)
---
> pmap_copy_page(vm_offset_t src, vm_offset_t dst)
780,781c802
<
< return;
---
> TODO;
785c806
< * Fill the given physical page with zeroes.
---
> * Zero a page of physical memory by temporarily mapping it into the tlb.
790,792c811
< #if 0
< bzero((caddr_t)pa, PAGE_SIZE);
< #else
---
> caddr_t va;
795,797c814,822
< for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) {
< __asm __volatile ("dcbz 0,%0" :: "r"(pa));
< pa += CACHELINESIZE;
---
> if (pa < SEGMENT_LENGTH) {
> va = (caddr_t) pa;
> } else if (pmap_initialized) {
> if (pmap_pvo_zeropage == NULL)
> pmap_pvo_zeropage = pmap_rkva_alloc();
> pmap_pa_map(pmap_pvo_zeropage, pa, NULL, NULL);
> va = (caddr_t)PVO_VADDR(pmap_pvo_zeropage);
> } else {
> panic("pmap_zero_page: can't zero pa %#x", pa);
799c824,833
< #endif
---
>
> bzero(va, PAGE_SIZE);
>
> for (i = PAGE_SIZE / CACHELINESIZE; i > 0; i--) {
> __asm __volatile("dcbz 0,%0" :: "r"(va));
> va += CACHELINESIZE;
> }
>
> if (pa >= SEGMENT_LENGTH)
> pmap_pa_unmap(pmap_pvo_zeropage, NULL, NULL);
805,806c839
<
< bzero((caddr_t)pa + off, size);
---
> TODO;
810c843,845
< * Copy the given physical source page to its destination.
---
> * Map the given physical page at the specified virtual address in the
> * target pmap with the protection requested. If specified the page
> * will be wired down.
813c848,849
< pmap_copy_page(vm_offset_t src, vm_offset_t dst)
---
> pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
> boolean_t wired)
814a851,854
> struct pvo_head *pvo_head;
> vm_zone_t zone;
> u_int pte_lo, pvo_flags;
> int error;
816,817c856,864
< bcopy((caddr_t)src, (caddr_t)dst, PAGE_SIZE);
< }
---
> if (!pmap_initialized) {
> pvo_head = &pmap_pvo_kunmanaged;
> zone = pmap_upvo_zone;
> pvo_flags = 0;
> } else {
> pvo_head = pa_to_pvoh(m->phys_addr);
> zone = pmap_mpvo_zone;
> pvo_flags = PVO_MANAGED;
> }
819,822c866
< static struct pv_entry *
< pmap_alloc_pv()
< {
< pv_entry_count++;
---
> pte_lo = PTE_I | PTE_G;
824,829c868,871
< if (pv_entry_high_water &&
< (pv_entry_count > pv_entry_high_water) &&
< (pmap_pagedaemon_waken == 0)) {
< pmap_pagedaemon_waken = 1;
< wakeup(&vm_pages_needed);
< }
---
> if (prot & VM_PROT_WRITE)
> pte_lo |= PTE_BW;
> else
> pte_lo |= PTE_BR;
831,832c873,874
< return zalloc(pvzone);
< }
---
> if (prot & VM_PROT_EXECUTE)
> pvo_flags |= PVO_EXECUTABLE;
834,836c876,877
< static void
< pmap_free_pv(struct pv_entry *pv)
< {
---
> if (wired)
> pvo_flags |= PVO_WIRED;
838,840c879
< pv_entry_count--;
< zfree(pvzone, pv);
< }
---
> critical_enter();
842,860c881,886
< /*
< * We really hope that we don't need overflow entries
< * before the VM system is initialized!
< *
< * XXX: Should really be switched over to the zone allocator.
< */
< static struct pte_ovfl *
< poalloc()
< {
< struct po_page *pop;
< struct pte_ovfl *po;
< vm_page_t mem;
< int i;
<
< if (!pmap_initialized) {
< panic("poalloc");
< }
<
< if (po_nfree == 0) {
---
> error = pmap_pvo_enter(pmap, zone, pvo_head, va, m->phys_addr, pte_lo,
> pvo_flags);
>
> critical_exit();
>
> if (error == ENOENT) {
862,863c888
< * Since we cannot use maps for potable allocation,
< * we have to steal some memory from the VM system. XXX
---
> * Flush the real memory from the cache.
865,872c890,891
< mem = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM);
< po_pcnt++;
< pop = (struct po_page *)VM_PAGE_TO_PHYS(mem);
< pop->pop_pgi.pgi_page = mem;
< LIST_INIT(&pop->pop_pgi.pgi_freelist);
< for (i = NPOPPG - 1, po = pop->pop_po + 1; --i >= 0; po++) {
< LIST_INSERT_HEAD(&pop->pop_pgi.pgi_freelist, po,
< po_list);
---
> if ((pvo_flags & PVO_EXECUTABLE) && (pte_lo & PTE_I) == 0) {
> pmap_syncicache(m->phys_addr, PAGE_SIZE);
874,884d892
< po_nfree += pop->pop_pgi.pgi_nfree = NPOPPG - 1;
< LIST_INSERT_HEAD(&po_page_freelist, pop, pop_pgi.pgi_list);
< po = pop->pop_po;
< } else {
< po_nfree--;
< pop = po_page_freelist.lh_first;
< if (--pop->pop_pgi.pgi_nfree <= 0) {
< LIST_REMOVE(pop, pop_pgi.pgi_list);
< }
< po = pop->pop_pgi.pgi_freelist.lh_first;
< LIST_REMOVE(po, po_list);
886,887d893
<
< return po;
890,891c896,897
< static void
< pofree(struct pte_ovfl *po, int freepage)
---
> vm_offset_t
> pmap_extract(pmap_t pmap, vm_offset_t va)
893,912c899,900
< struct po_page *pop;
<
< pop = (struct po_page *)trunc_page((vm_offset_t)po);
< switch (++pop->pop_pgi.pgi_nfree) {
< case NPOPPG:
< if (!freepage) {
< break;
< }
< po_nfree -= NPOPPG - 1;
< po_pcnt--;
< LIST_REMOVE(pop, pop_pgi.pgi_list);
< vm_page_free(pop->pop_pgi.pgi_page);
< return;
< case 1:
< LIST_INSERT_HEAD(&po_page_freelist, pop, pop_pgi.pgi_list);
< default:
< break;
< }
< LIST_INSERT_HEAD(&pop->pop_pgi.pgi_freelist, po, po_list);
< po_nfree++;
---
> TODO;
> return (0);
916c904
< * This returns whether this is the first mapping of a page.
---
> * Grow the number of kernel page table entries. Unneeded.
918,919c906,907
< static int
< pmap_enter_pv(int pteidx, vm_offset_t va, vm_offset_t pa)
---
> void
> pmap_growkernel(vm_offset_t addr)
921,926c909
< struct pv_entry *pv, *npv;
< int s, first;
<
< if (!pmap_initialized) {
< return 0;
< }
---
> }
928c911,913
< s = splimp();
---
> void
> pmap_init(vm_offset_t phys_start, vm_offset_t phys_end)
> {
930,951c915
< pv = pa_to_pv(pa);
< first = pv->pv_idx;
< if (pv->pv_idx == -1) {
< /*
< * No entries yet, use header as the first entry.
< */
< pv->pv_va = va;
< pv->pv_idx = pteidx;
< pv->pv_next = NULL;
< } else {
< /*
< * There is at least one other VA mapping this page.
< * Place this entry after the header.
< */
< npv = pmap_alloc_pv();
< npv->pv_va = va;
< npv->pv_idx = pteidx;
< npv->pv_next = pv->pv_next;
< pv->pv_next = npv;
< }
< splx(s);
< return first;
---
> CTR(KTR_PMAP, "pmap_init");
954,955c918,919
< static void
< pmap_remove_pv(int pteidx, vm_offset_t va, vm_offset_t pa, struct pte *pte)
---
> void
> pmap_init2(void)
957,958d920
< struct pv_entry *pv, *npv;
< char *attr;
960,1003c922,927
< /*
< * First transfer reference/change bits to cache.
< */
< attr = pa_to_attr(pa);
< if (attr == NULL) {
< return;
< }
< *attr |= (pte->pte_lo & (PTE_REF | PTE_CHG)) >> ATTRSHFT;
<
< /*
< * Remove from the PV table.
< */
< pv = pa_to_pv(pa);
<
< /*
< * If it is the first entry on the list, it is actually
< * in the header and we must copy the following entry up
< * to the header. Otherwise we must search the list for
< * the entry. In either case we free the now unused entry.
< */
< if (pteidx == pv->pv_idx && va == pv->pv_va) {
< npv = pv->pv_next;
< if (npv) {
< *pv = *npv;
< pmap_free_pv(npv);
< } else {
< pv->pv_idx = -1;
< }
< } else {
< for (; (npv = pv->pv_next); pv = npv) {
< if (pteidx == npv->pv_idx && va == npv->pv_va) {
< break;
< }
< }
< if (npv) {
< pv->pv_next = npv->pv_next;
< pmap_free_pv(npv);
< }
< #ifdef DIAGNOSTIC
< else {
< panic("pmap_remove_pv: not on list\n");
< }
< #endif
< }
---
> CTR(KTR_PMAP, "pmap_init2");
> zinitna(pmap_upvo_zone, &pmap_upvo_zone_obj, NULL, 0, PMAP_PVO_SIZE,
> ZONE_INTERRUPT, 1);
> pmap_mpvo_zone = zinit("managed pvo", sizeof(struct pvo_entry),
> PMAP_PVO_SIZE, ZONE_INTERRUPT, 1);
> pmap_initialized = TRUE;
1005a930,949
> boolean_t
> pmap_is_modified(vm_page_t m)
> {
> TODO;
> return (0);
> }
>
> void
> pmap_clear_reference(vm_page_t m)
> {
> TODO;
> }
>
> int
> pmap_ts_referenced(vm_page_t m)
> {
> TODO;
> return (0);
> }
>
1007c951
< * Insert physical page at pa into the given pmap at virtual address va.
---
> * Map a wired page into kernel virtual address space.
1010,1011c954
< pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t pg, vm_prot_t prot,
< boolean_t wired)
---
> pmap_kenter(vm_offset_t va, vm_offset_t pa)
1013,1018c956,958
< sr_t sr;
< int idx, s;
< pte_t pte;
< struct pte_ovfl *po;
< struct mem_region *mp;
< vm_offset_t pa;
---
> u_int pte_lo;
> int error;
> int i;
1020c960,964
< pa = VM_PAGE_TO_PHYS(pg) & ~PAGE_MASK;
---
> #if 0
> if (va < VM_MIN_KERNEL_ADDRESS)
> panic("pmap_kenter: attempt to enter non-kernel address %#x",
> va);
> #endif
1022,1042c966,969
< /*
< * Have to remove any existing mapping first.
< */
< pmap_remove(pm, va, va + PAGE_SIZE);
<
< /*
< * Compute the HTAB index.
< */
< idx = pteidx(sr = ptesr(pm->pm_sr, va), va);
< /*
< * Construct the PTE.
< *
< * Note: Don't set the valid bit for correct operation of tlb update.
< */
< pte.pte_hi = ((sr & SR_VSID) << PTE_VSID_SHFT)
< | ((va & ADDR_PIDX) >> ADDR_API_SHFT);
< pte.pte_lo = (pa & PTE_RPGN) | PTE_M | PTE_I | PTE_G;
<
< for (mp = mem; mp->size; mp++) {
< if (pa >= mp->start && pa < mp->start + mp->size) {
< pte.pte_lo &= ~(PTE_I | PTE_G);
---
> pte_lo = PTE_I | PTE_G | PTE_BW;
> for (i = 0; phys_avail[i + 2] != 0; i += 2) {
> if (pa >= phys_avail[i] && pa < phys_avail[i + 1]) {
> pte_lo &= ~(PTE_I | PTE_G);
1046,1050d972
< if (prot & VM_PROT_WRITE) {
< pte.pte_lo |= PTE_RW;
< } else {
< pte.pte_lo |= PTE_RO;
< }
1052,1062c974
< /*
< * Now record mapping for later back-translation.
< */
< if (pmap_initialized && (pg->flags & PG_FICTITIOUS) == 0) {
< if (pmap_enter_pv(idx, va, pa)) {
< /*
< * Flush the real memory from the cache.
< */
< __syncicache((void *)pa, PAGE_SIZE);
< }
< }
---
> critical_enter();
1064,1072c976,977
< s = splimp();
< pm->pm_stats.resident_count++;
< /*
< * Try to insert directly into HTAB.
< */
< if (pte_insert(idx, &pte)) {
< splx(s);
< return;
< }
---
> error = pmap_pvo_enter(kernel_pmap, pmap_upvo_zone,
> &pmap_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED);
1073a979,984
> critical_exit();
>
> if (error != 0 && error != ENOENT)
> panic("pmap_kenter: failed to enter va %#x pa %#x: %d", va,
> pa, error);
>
1075,1077c986
< * Have to allocate overflow entry.
< *
< * Note, that we must use real addresses for these.
---
> * Flush the real memory from the instruction cache.
1079,1082c988,990
< po = poalloc();
< po->po_pte = pte;
< LIST_INSERT_HEAD(potable + idx, po, po_list);
< splx(s);
---
> if ((pte_lo & (PTE_I | PTE_G)) == 0) {
> pmap_syncicache(pa, PAGE_SIZE);
> }
1085,1086c993,994
< void
< pmap_kenter(vm_offset_t va, vm_offset_t pa)
---
> vm_offset_t
> pmap_kextract(vm_offset_t va)
1088,1091c996,997
< struct vm_page pg;
<
< pg.phys_addr = pa;
< pmap_enter(kernel_pmap, va, &pg, VM_PROT_READ|VM_PROT_WRITE, TRUE);
---
> TODO;
> return (0);
1097c1003
< pmap_remove(kernel_pmap, va, va + PAGE_SIZE);
---
> TODO;
1101c1007,1013
< * Remove the given range of mapping entries.
---
> * Map a range of physical addresses into kernel virtual address space.
> *
> * The value passed in *virt is a suggested virtual address for the mapping.
> * Architectures which can support a direct-mapped physical to virtual region
> * can return the appropriate address within that region, leaving '*virt'
> * unchanged. We cannot and therefore do not; *virt is updated with the
> * first usable address after the mapped region.
1103,1104c1015,1016
< void
< pmap_remove(struct pmap *pm, vm_offset_t va, vm_offset_t endva)
---
> vm_offset_t
> pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot)
1106,1109c1018
< int idx, i, s;
< sr_t sr;
< pte_t *ptp;
< struct pte_ovfl *po, *npo;
---
> vm_offset_t sva, va;
1111,1147c1020,1025
< s = splimp();
< while (va < endva) {
< idx = pteidx(sr = ptesr(pm->pm_sr, va), va);
< for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++) {
< if (ptematch(ptp, sr, va, PTE_VALID)) {
< pmap_remove_pv(idx, va, ptp->pte_lo, ptp);
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(va);
< tlbsync();
< pm->pm_stats.resident_count--;
< }
< }
< for (ptp = ptable + (idx ^ ptab_mask) * 8, i = 8; --i >= 0;
< ptp++) {
< if (ptematch(ptp, sr, va, PTE_VALID | PTE_HID)) {
< pmap_remove_pv(idx, va, ptp->pte_lo, ptp);
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(va);
< tlbsync();
< pm->pm_stats.resident_count--;
< }
< }
< for (po = potable[idx].lh_first; po; po = npo) {
< npo = po->po_list.le_next;
< if (ptematch(&po->po_pte, sr, va, 0)) {
< pmap_remove_pv(idx, va, po->po_pte.pte_lo,
< &po->po_pte);
< LIST_REMOVE(po, po_list);
< pofree(po, 1);
< pm->pm_stats.resident_count--;
< }
< }
< va += PAGE_SIZE;
< }
< splx(s);
---
> sva = *virt;
> va = sva;
> for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE)
> pmap_kenter(va, pa_start);
> *virt = va;
> return (sva);
1150,1151c1028,1029
< static pte_t *
< pte_find(struct pmap *pm, vm_offset_t va)
---
> int
> pmap_mincore(pmap_t pmap, vm_offset_t addr)
1153,1174c1031,1032
< int idx, i;
< sr_t sr;
< pte_t *ptp;
< struct pte_ovfl *po;
<
< idx = pteidx(sr = ptesr(pm->pm_sr, va), va);
< for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++) {
< if (ptematch(ptp, sr, va, PTE_VALID)) {
< return ptp;
< }
< }
< for (ptp = ptable + (idx ^ ptab_mask) * 8, i = 8; --i >= 0; ptp++) {
< if (ptematch(ptp, sr, va, PTE_VALID | PTE_HID)) {
< return ptp;
< }
< }
< for (po = potable[idx].lh_first; po; po = po->po_list.le_next) {
< if (ptematch(&po->po_pte, sr, va, 0)) {
< return &po->po_pte;
< }
< }
< return 0;
---
> TODO;
> return (0);
1177,1178c1035,1037
< /*
< * Get the physical page address for the given pmap/virtual address.
---
> /*
> * Create the uarea for a new process.
> * This routine directly affects the fork perf for a process.
1180,1201d1038
< vm_offset_t
< pmap_extract(pmap_t pm, vm_offset_t va)
< {
< pte_t *ptp;
< int s;
<
< s = splimp();
<
< if (!(ptp = pte_find(pm, va))) {
< splx(s);
< return (0);
< }
< splx(s);
< return ((ptp->pte_lo & PTE_RPGN) | (va & ADDR_POFF));
< }
<
< /*
< * Lower the protection on the specified range of this pmap.
< *
< * There are only two cases: either the protection is going to 0,
< * or it is going to read-only.
< */
1203c1040
< pmap_protect(struct pmap *pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
---
> pmap_new_proc(struct proc *p)
1205,1229c1042,1045
< pte_t *ptp;
< int valid, s;
<
< if (prot & VM_PROT_READ) {
< s = splimp();
< while (sva < eva) {
< ptp = pte_find(pm, sva);
< if (ptp) {
< valid = ptp->pte_hi & PTE_VALID;
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(sva);
< tlbsync();
< ptp->pte_lo &= ~PTE_PP;
< ptp->pte_lo |= PTE_RO;
< __asm __volatile ("sync");
< ptp->pte_hi |= valid;
< }
< sva += PAGE_SIZE;
< }
< splx(s);
< return;
< }
< pmap_remove(pm, sva, eva);
< }
---
> vm_object_t upobj;
> vm_offset_t up;
> vm_page_t m;
> u_int i;
1231,1243d1046
< boolean_t
< ptemodify(vm_page_t pg, u_int mask, u_int val)
< {
< vm_offset_t pa;
< struct pv_entry *pv;
< pte_t *ptp;
< struct pte_ovfl *po;
< int i, s;
< char *attr;
< int rv;
<
< pa = VM_PAGE_TO_PHYS(pg);
<
1245c1048
< * First modify bits in cache.
---
> * Allocate the object for the upages.
1247,1249c1050,1053
< attr = pa_to_attr(pa);
< if (attr == NULL) {
< return FALSE;
---
> upobj = p->p_upages_obj;
> if (upobj == NULL) {
> upobj = vm_object_allocate(OBJT_DEFAULT, UAREA_PAGES);
> p->p_upages_obj = upobj;
1252,1257c1056,1064
< *attr &= ~mask >> ATTRSHFT;
< *attr |= val >> ATTRSHFT;
<
< pv = pa_to_pv(pa);
< if (pv->pv_idx < 0) {
< return FALSE;
---
> /*
> * Get a kernel virtual address for the uarea for this process.
> */
> up = (vm_offset_t)p->p_uarea;
> if (up == 0) {
> up = kmem_alloc_nofault(kernel_map, UAREA_PAGES * PAGE_SIZE);
> if (up == 0)
> panic("pmap_new_proc: upage allocation failed");
> p->p_uarea = (struct user *)up;
1260,1303c1067,1071
< rv = FALSE;
< s = splimp();
< for (; pv; pv = pv->pv_next) {
< for (ptp = ptable + pv->pv_idx * 8, i = 8; --i >= 0; ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(pv->pv_va);
< tlbsync();
< rv |= ptp->pte_lo & mask;
< ptp->pte_lo &= ~mask;
< ptp->pte_lo |= val;
< __asm __volatile ("sync");
< ptp->pte_hi |= PTE_VALID;
< }
< }
< for (ptp = ptable + (pv->pv_idx ^ ptab_mask) * 8, i = 8;
< --i >= 0; ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(pv->pv_va);
< tlbsync();
< rv |= ptp->pte_lo & mask;
< ptp->pte_lo &= ~mask;
< ptp->pte_lo |= val;
< __asm __volatile ("sync");
< ptp->pte_hi |= PTE_VALID;
< }
< }
< for (po = potable[pv->pv_idx].lh_first; po;
< po = po->po_list.le_next) {
< if ((po->po_pte.pte_lo & PTE_RPGN) == pa) {
< rv |= ptp->pte_lo & mask;
< po->po_pte.pte_lo &= ~mask;
< po->po_pte.pte_lo |= val;
< }
< }
< }
< splx(s);
< return rv != 0;
< }
---
> for (i = 0; i < UAREA_PAGES; i++) {
> /*
> * Get a uarea page.
> */
> m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
1305,1313c1073,1076
< int
< ptebits(vm_page_t pg, int bit)
< {
< struct pv_entry *pv;
< pte_t *ptp;
< struct pte_ovfl *po;
< int i, s, bits;
< char *attr;
< vm_offset_t pa;
---
> /*
> * Wire the page.
> */
> m->wire_count++;
1315,1316c1078,1081
< bits = 0;
< pa = VM_PAGE_TO_PHYS(pg);
---
> /*
> * Enter the page into the kernel address space.
> */
> pmap_kenter(up + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m));
1318,1323c1083,1086
< /*
< * First try the cache.
< */
< attr = pa_to_attr(pa);
< if (attr == NULL) {
< return 0;
---
> vm_page_wakeup(m);
> vm_page_flag_clear(m, PG_ZERO);
> vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
> m->valid = VM_PAGE_BITS_ALL;
1325,1328c1088
< bits |= (*attr << ATTRSHFT) & bit;
< if (bits == bit) {
< return bits;
< }
---
> }
1330,1370c1090,1094
< pv = pa_to_pv(pa);
< if (pv->pv_idx < 0) {
< return 0;
< }
<
< s = splimp();
< for (; pv; pv = pv->pv_next) {
< for (ptp = ptable + pv->pv_idx * 8, i = 8; --i >= 0; ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< bits |= ptp->pte_lo & bit;
< if (bits == bit) {
< splx(s);
< return bits;
< }
< }
< }
< for (ptp = ptable + (pv->pv_idx ^ ptab_mask) * 8, i = 8;
< --i >= 0; ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< bits |= ptp->pte_lo & bit;
< if (bits == bit) {
< splx(s);
< return bits;
< }
< }
< }
< for (po = potable[pv->pv_idx].lh_first; po;
< po = po->po_list.le_next) {
< if ((po->po_pte.pte_lo & PTE_RPGN) == pa) {
< bits |= po->po_pte.pte_lo & bit;
< if (bits == bit) {
< splx(s);
< return bits;
< }
< }
< }
< }
< splx(s);
< return bits;
---
> void
> pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
> vm_pindex_t pindex, vm_size_t size, int limit)
> {
> TODO;
1374,1377c1098
< * Lower the protection on the specified physical page.
< *
< * There are only two cases: either the protection is going to 0,
< * or it is going to read-only.
---
> * Lower the permission for all mappings to a given page.
1382,1387c1103,1105
< vm_offset_t pa;
< vm_offset_t va;
< pte_t *ptp;
< struct pte_ovfl *po, *npo;
< int i, s, idx;
< struct pv_entry *pv;
---
> struct pvo_head *pvo_head;
> struct pvo_entry *pvo, *next_pvo;
> struct pte *pt;
1389,1393c1107,1113
< pa = VM_PAGE_TO_PHYS(m);
<
< pa &= ~ADDR_POFF;
< if (prot & VM_PROT_READ) {
< ptemodify(m, PTE_PP, PTE_RO);
---
> /*
> * Since the routine only downgrades protection, if the
> * maximal protection is desired, there isn't any change
> * to be made.
> */
> if ((prot & (VM_PROT_READ|VM_PROT_WRITE)) ==
> (VM_PROT_READ|VM_PROT_WRITE))
1395d1114
< }
1397,1400c1116
< pv = pa_to_pv(pa);
< if (pv == NULL) {
< return;
< }
---
> critical_enter();
1402,1415c1118,1128
< s = splimp();
< while (pv->pv_idx >= 0) {
< idx = pv->pv_idx;
< va = pv->pv_va;
< for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< pmap_remove_pv(idx, va, pa, ptp);
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(va);
< tlbsync();
< goto next;
< }
---
> pvo_head = vm_page_to_pvoh(m);
> for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
> next_pvo = LIST_NEXT(pvo, pvo_vlink);
> PMAP_PVO_CHECK(pvo); /* sanity check */
>
> /*
> * Downgrading to no mapping at all, we just remove the entry.
> */
> if ((prot & VM_PROT_READ) == 0) {
> pmap_pvo_remove(pvo, -1);
> continue;
1417,1427c1130,1144
< for (ptp = ptable + (idx ^ ptab_mask) * 8, i = 8; --i >= 0;
< ptp++) {
< if ((ptp->pte_hi & PTE_VALID)
< && (ptp->pte_lo & PTE_RPGN) == pa) {
< pmap_remove_pv(idx, va, pa, ptp);
< ptp->pte_hi &= ~PTE_VALID;
< __asm __volatile ("sync");
< tlbie(va);
< tlbsync();
< goto next;
< }
---
>
> /*
> * If EXEC permission is being revoked, just clear the flag
> * in the PVO.
> */
> if ((prot & VM_PROT_EXECUTE) == 0)
> pvo->pvo_vaddr &= ~PVO_EXECUTABLE;
>
> /*
> * If this entry is already RO, don't diddle with the page
> * table.
> */
> if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) {
> PMAP_PVO_CHECK(pvo);
> continue;
1429,1438c1146,1156
< for (po = potable[idx].lh_first; po; po = npo) {
< npo = po->po_list.le_next;
< if ((po->po_pte.pte_lo & PTE_RPGN) == pa) {
< pmap_remove_pv(idx, va, pa, &po->po_pte);
< LIST_REMOVE(po, po_list);
< pofree(po, 1);
< goto next;
< }
< }
< next:
---
>
> /*
> * Grab the PTE before we diddle the bits so pvo_to_pte can
> * verify the pte contents are as expected.
> */
> pt = pmap_pvo_to_pte(pvo, -1);
> pvo->pvo_pte.pte_lo &= ~PTE_PP;
> pvo->pvo_pte.pte_lo |= PTE_BR;
> if (pt != NULL)
> pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
> PMAP_PVO_CHECK(pvo); /* sanity check */
1440c1158,1159
< splx(s);
---
>
> critical_exit();
1444,1445c1163
< * Activate the address space for the specified process. If the process
< * is the current process, load the new MMU context.
---
> * Make the specified page pageable (or not). Unneeded.
1448c1166,1167
< pmap_activate(struct thread *td)
---
> pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
> boolean_t pageable)
1450,1453c1169
< struct pcb *pcb;
< pmap_t pmap;
< pmap_t rpm;
< int psl, i, ksr, seg;
---
> }
1455,1456c1171,1176
< pcb = td->td_pcb;
< pmap = vmspace_pmap(td->td_proc->p_vmspace);
---
> boolean_t
> pmap_page_exists(pmap_t pmap, vm_page_t m)
> {
> TODO;
> return (0);
> }
1458,1465c1178
< /*
< * XXX Normally performed in cpu_fork().
< */
< if (pcb->pcb_pm != pmap) {
< pcb->pcb_pm = pmap;
< (vm_offset_t) pcb->pcb_pmreal = pmap_extract(kernel_pmap,
< (vm_offset_t)pcb->pcb_pm);
< }
---
> static u_int pmap_vsidcontext;
1467,1470c1180,1184
< if (td == curthread) {
< /* Disable interrupts while switching. */
< psl = mfmsr();
< mtmsr(psl & ~PSL_EE);
---
> void
> pmap_pinit(pmap_t pmap)
> {
> int i, mask;
> u_int entropy;
1472,1475c1186,1187
< #if 0 /* XXX */
< /* Store pointer to new current pmap. */
< curpm = pcb->pcb_pmreal;
< #endif
---
> entropy = 0;
> __asm __volatile("mftb %0" : "=r"(entropy));
1477,1478c1189,1194
< /* Save kernel SR. */
< __asm __volatile("mfsr %0,14" : "=r"(ksr) :);
---
> /*
> * Allocate some segment registers for this pmap.
> */
> pmap->pm_count = 1;
> for (i = 0; i < NPMAPS; i += VSID_NBPW) {
> u_int hash, n;
1481,1482c1197,1201
< * Set new segment registers. We use the pmap's real
< * address to avoid accessibility problems.
---
> * Create a new value by mutiplying by a prime and adding in
> * entropy from the timebase register. This is to make the
> * VSID more random so that the PT hash function collides
> * less often. (Note that the prime casues gcc to do shifts
> * instead of a multiply.)
1484,1488c1203,1219
< rpm = pcb->pcb_pmreal;
< for (i = 0; i < 16; i++) {
< seg = rpm->pm_sr[i];
< __asm __volatile("mtsrin %0,%1"
< :: "r"(seg), "r"(i << ADDR_SR_SHFT));
---
> pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy;
> hash = pmap_vsidcontext & (NPMAPS - 1);
> if (hash == 0) /* 0 is special, avoid it */
> continue;
> n = hash >> 5;
> mask = 1 << (hash & (VSID_NBPW - 1));
> hash = (pmap_vsidcontext & 0xfffff);
> if (pmap_vsid_bitmap[n] & mask) { /* collision? */
> /* anything free in this bucket? */
> if (pmap_vsid_bitmap[n] == 0xffffffff) {
> entropy = (pmap_vsidcontext >> 20);
> continue;
> }
> i = ffs(~pmap_vsid_bitmap[i]) - 1;
> mask = 1 << i;
> hash &= 0xfffff & ~(VSID_NBPW - 1);
> hash |= i;
1490,1495c1221,1224
<
< /* Restore kernel SR. */
< __asm __volatile("mtsr 14,%0" :: "r"(ksr));
<
< /* Interrupts are OK again. */
< mtmsr(psl);
---
> pmap_vsid_bitmap[n] |= mask;
> for (i = 0; i < 16; i++)
> pmap->pm_sr[i] = VSID_MAKE(i, hash);
> return;
1496a1226,1227
>
> panic("pmap_pinit: out of segments");
1500,1505c1231
< * Add a list of wired pages to the kva
< * this routine is only used for temporary
< * kernel mappings that do not need to have
< * page modification or references recorded.
< * Note that old mappings are simply written
< * over. The page *must* be wired.
---
> * Initialize the pmap associated with process 0.
1508c1234
< pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
---
> pmap_pinit0(pmap_t pm)
1510d1235
< int i;
1512,1515c1237,1238
< for (i = 0; i < count; i++) {
< vm_offset_t tva = va + i * PAGE_SIZE;
< pmap_kenter(tva, VM_PAGE_TO_PHYS(m[i]));
< }
---
> pmap_pinit(pm);
> bzero(&pm->pm_stats, sizeof(pm->pm_stats));
1518,1521d1240
< /*
< * this routine jerks page mappings from the
< * kernel -- it is meant only for temporary mappings.
< */
1523c1242
< pmap_qremove(vm_offset_t va, int count)
---
> pmap_pinit2(pmap_t pmap)
1525c1244,1245
< vm_offset_t end_va;
---
> /* XXX: Remove this stub when no longer called */
> }
1527,1536c1247,1250
< end_va = va + count*PAGE_SIZE;
<
< while (va < end_va) {
< unsigned *pte;
<
< pte = (unsigned *)vtopte(va);
< *pte = 0;
< tlbie(va);
< va += PAGE_SIZE;
< }
---
> void
> pmap_prefault(pmap_t pmap, vm_offset_t va, vm_map_entry_t entry)
> {
> TODO;
1539,1545c1253,1254
< /*
< * pmap_ts_referenced:
< *
< * Return the count of reference bits for a page, clearing all of them.
< */
< int
< pmap_ts_referenced(vm_page_t m)
---
> void
> pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
1546a1256,1257
> TODO;
> }
1548c1259,1262
< /* XXX: coming soon... */
---
> vm_offset_t
> pmap_phys_address(int ppn)
> {
> TODO;
1552,1557c1266,1267
< /*
< * this routine returns true if a physical page resides
< * in the given pmap.
< */
< boolean_t
< pmap_page_exists(pmap_t pmap, vm_page_t m)
---
> void
> pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
1559,1561c1269
< #if 0 /* XXX: This must go! */
< register pv_entry_t pv;
< int s;
---
> int i;
1563,1564c1271,1273
< if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
< return FALSE;
---
> for (i = 0; i < count; i++, va += PAGE_SIZE)
> pmap_kenter(va, VM_PAGE_TO_PHYS(m[i]));
> }
1566,1579c1275,1278
< s = splvm();
<
< /*
< * Not found, check current mappings returning immediately if found.
< */
< for (pv = pv_table; pv; pv = pv->pv_next) {
< if (pv->pv_pmap == pmap) {
< splx(s);
< return TRUE;
< }
< }
< splx(s);
< #endif
< return (FALSE);
---
> void
> pmap_qremove(vm_offset_t va, int count)
> {
> TODO;
1583,1587c1282
< * Used to map a range of physical addresses into kernel
< * virtual address space.
< *
< * For now, VM is already on, we only need to map the
< * specified memory.
---
> * Add a reference to the specified pmap.
1589,1590c1284,1285
< vm_offset_t
< pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot)
---
> void
> pmap_reference(pmap_t pm)
1592d1286
< vm_offset_t sva, va;
1594,1604c1288,1289
< sva = *virt;
< va = sva;
<
< while (start < end) {
< pmap_kenter(va, start);
< va += PAGE_SIZE;
< start += PAGE_SIZE;
< }
<
< *virt = va;
< return (sva);
---
> if (pm != NULL)
> pm->pm_count++;
1607,1608c1292,1293
< vm_offset_t
< pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
---
> void
> pmap_release(pmap_t pmap)
1610,1611c1295
<
< return (addr);
---
> TODO;
1614,1615c1298,1299
< int
< pmap_mincore(pmap_t pmap, vm_offset_t addr)
---
> void
> pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1617,1619c1301
<
< /* XXX: coming soon... */
< return (0);
---
> TODO;
1623,1624c1305
< pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
< vm_pindex_t pindex, vm_size_t size, int limit)
---
> pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1626,1628c1307
<
< /* XXX: coming soon... */
< return;
---
> TODO;
1632c1311
< pmap_growkernel(vm_offset_t addr)
---
> pmap_swapin_proc(struct proc *p)
1633a1313,1314
> TODO;
> }
1635,1636c1316,1319
< /* XXX: coming soon... */
< return;
---
> void
> pmap_swapout_proc(struct proc *p)
> {
> TODO;
1640,1642c1323,1325
< * Initialize the address space (zone) for the pv_entries. Set a
< * high water mark so that the system can recover from excessive
< * numbers of pv entries.
---
> * Create the kernel stack and pcb for a new thread.
> * This routine directly affects the fork perf for a process and
> * create performance for a thread.
1645c1328
< pmap_init2()
---
> pmap_new_thread(struct thread *td)
1647c1330,1333
< int shpgperproc = PMAP_SHPGPERPROC;
---
> vm_object_t ksobj;
> vm_offset_t ks;
> vm_page_t m;
> u_int i;
1649,1652c1335,1378
< TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
< pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
< pv_entry_high_water = 9 * (pv_entry_max / 10);
< zinitna(pvzone, &pvzone_obj, NULL, 0, pv_entry_max, ZONE_INTERRUPT, 1);
---
> /*
> * Allocate object for the kstack.
> */
> ksobj = td->td_kstack_obj;
> if (ksobj == NULL) {
> ksobj = vm_object_allocate(OBJT_DEFAULT, KSTACK_PAGES);
> td->td_kstack_obj = ksobj;
> }
>
> /*
> * Get a kernel virtual address for the kstack for this thread.
> */
> ks = td->td_kstack;
> if (ks == 0) {
> ks = kmem_alloc_nofault(kernel_map,
> (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE);
> if (ks == 0)
> panic("pmap_new_thread: kstack allocation failed");
> TLBIE(ks);
> ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
> td->td_kstack = ks;
> }
>
> for (i = 0; i < KSTACK_PAGES; i++) {
> /*
> * Get a kernel stack page.
> */
> m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
>
> /*
> * Wire the page.
> */
> m->wire_count++;
>
> /*
> * Enter the page into the kernel address space.
> */
> pmap_kenter(ks + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m));
>
> vm_page_wakeup(m);
> vm_page_flag_clear(m, PG_ZERO);
> vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
> m->valid = VM_PAGE_BITS_ALL;
> }
1656c1382
< pmap_swapin_proc(struct proc *p)
---
> pmap_dispose_proc(struct proc *p)
1658,1660c1384
<
< /* XXX: coming soon... */
< return;
---
> TODO;
1664c1388
< pmap_swapout_proc(struct proc *p)
---
> pmap_dispose_thread(struct thread *td)
1666,1668c1390
<
< /* XXX: coming soon... */
< return;
---
> TODO;
1671,1676d1392
<
< /*
< * Create the kernel stack (including pcb for i386) for a new thread.
< * This routine directly affects the fork perf for a process and
< * create performance for a thread.
< */
1678,1679c1394
< pmap_new_thread(td)
< struct thread *td;
---
> pmap_swapin_thread(struct thread *td)
1681,1682c1396
< /* XXX: coming soon... */
< return;
---
> TODO;
1685,1688d1398
< /*
< * Dispose the kernel stack for a thread that has exited.
< * This routine directly impacts the exit perf of a process and thread.
< */
1690,1691c1400
< pmap_dispose_thread(td)
< struct thread *td;
---
> pmap_swapout_thread(struct thread *td)
1693,1694c1402
< /* XXX: coming soon... */
< return;
---
> TODO;
1698c1406,1408
< * Allow the Kernel stack for a thread to be prejudicially paged out.
---
> * Allocate a physical page of memory directly from the phys_avail map.
> * Can only be called from pmap_bootstrap before avail start and end are
> * calculated.
1700,1702c1410,1411
< void
< pmap_swapout_thread(td)
< struct thread *td;
---
> static vm_offset_t
> pmap_bootstrap_alloc(vm_size_t size, u_int align)
1704,1707c1413,1414
< int i;
< vm_object_t ksobj;
< vm_offset_t ks;
< vm_page_t m;
---
> vm_offset_t s, e;
> int i, j;
1709,1717c1416,1443
< ksobj = td->td_kstack_obj;
< ks = td->td_kstack;
< for (i = 0; i < KSTACK_PAGES; i++) {
< m = vm_page_lookup(ksobj, i);
< if (m == NULL)
< panic("pmap_swapout_thread: kstack already missing?");
< vm_page_dirty(m);
< vm_page_unwire(m, 0);
< pmap_kremove(ks + i * PAGE_SIZE);
---
> size = round_page(size);
> for (i = 0; phys_avail[i + 1] != 0; i += 2) {
> if (align != 0)
> s = (phys_avail[i] + align - 1) & ~(align - 1);
> else
> s = phys_avail[i];
> e = s + size;
>
> if (s < phys_avail[i] || e > phys_avail[i + 1])
> continue;
>
> if (s == phys_avail[i]) {
> phys_avail[i] += size;
> } else if (e == phys_avail[i + 1]) {
> phys_avail[i + 1] -= size;
> } else {
> for (j = phys_avail_count * 2; j > i; j -= 2) {
> phys_avail[j] = phys_avail[j - 2];
> phys_avail[j + 1] = phys_avail[j - 1];
> }
>
> phys_avail[i + 3] = phys_avail[i + 1];
> phys_avail[i + 1] = s;
> phys_avail[i + 2] = e;
> phys_avail_count++;
> }
>
> return (s);
1718a1445
> panic("pmap_bootstrap_alloc: could not allocate memory");
1722c1449,1450
< * Bring the kernel stack for a specified thread back in.
---
> * Return an unmapped pvo for a kernel virtual address.
> * Used by pmap functions that operate on physical pages.
1724,1726c1452,1453
< void
< pmap_swapin_thread(td)
< struct thread *td;
---
> static struct pvo_entry *
> pmap_rkva_alloc(void)
1728,1731c1455,1458
< int i, rv;
< vm_object_t ksobj;
< vm_offset_t ks;
< vm_page_t m;
---
> struct pvo_entry *pvo;
> struct pte *pt;
> vm_offset_t kva;
> int pteidx;
1733,1743c1460,1503
< ksobj = td->td_kstack_obj;
< ks = td->td_kstack;
< for (i = 0; i < KSTACK_PAGES; i++) {
< m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
< pmap_kenter(ks + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m));
< if (m->valid != VM_PAGE_BITS_ALL) {
< rv = vm_pager_get_pages(ksobj, &m, 1, 0);
< if (rv != VM_PAGER_OK)
< panic("pmap_swapin_thread: cannot get kstack for proc: %d\n", td->td_proc->p_pid);
< m = vm_page_lookup(ksobj, i);
< m->valid = VM_PAGE_BITS_ALL;
---
> if (pmap_rkva_count == 0)
> panic("pmap_rkva_alloc: no more reserved KVAs");
>
> kva = pmap_rkva_start + (PAGE_SIZE * --pmap_rkva_count);
> pmap_kenter(kva, 0);
>
> pvo = pmap_pvo_find_va(kernel_pmap, kva, &pteidx);
>
> if (pvo == NULL)
> panic("pmap_kva_alloc: pmap_pvo_find_va failed");
>
> pt = pmap_pvo_to_pte(pvo, pteidx);
>
> if (pt == NULL)
> panic("pmap_kva_alloc: pmap_pvo_to_pte failed");
>
> pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
> PVO_PTEGIDX_CLR(pvo);
>
> pmap_pte_overflow++;
>
> return (pvo);
> }
>
> static void
> pmap_pa_map(struct pvo_entry *pvo, vm_offset_t pa, struct pte *saved_pt,
> int *depth_p)
> {
> struct pte *pt;
>
> critical_enter();
>
> /*
> * If this pvo already has a valid pte, we need to save it so it can
> * be restored later. We then just reload the new PTE over the old
> * slot.
> */
> if (saved_pt != NULL) {
> pt = pmap_pvo_to_pte(pvo, -1);
>
> if (pt != NULL) {
> pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
> PVO_PTEGIDX_CLR(pvo);
> pmap_pte_overflow++;
1745,1747c1505,1508
< vm_page_wire(m);
< vm_page_wakeup(m);
< vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
---
>
> *saved_pt = pvo->pvo_pte;
>
> pvo->pvo_pte.pte_lo &= ~PTE_RPGN;
1748a1510,1519
>
> pvo->pvo_pte.pte_lo |= pa;
>
> if (!pmap_pte_spill(pvo->pvo_vaddr))
> panic("pmap_pa_map: could not spill pvo %p", pvo);
>
> if (depth_p != NULL)
> (*depth_p)++;
>
> critical_exit();
1751,1752c1522,1523
< void
< pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, boolean_t pageable)
---
> static void
> pmap_pa_unmap(struct pvo_entry *pvo, struct pte *saved_pt, int *depth_p)
1753a1525
> struct pte *pt;
1755c1527,1552
< return;
---
> critical_enter();
>
> pt = pmap_pvo_to_pte(pvo, -1);
>
> if (pt != NULL) {
> pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
> PVO_PTEGIDX_CLR(pvo);
> pmap_pte_overflow++;
> }
>
> pvo->pvo_pte.pte_lo &= ~PTE_RPGN;
>
> /*
> * If there is a saved PTE and it's valid, restore it and return.
> */
> if (saved_pt != NULL && (saved_pt->pte_lo & PTE_RPGN) != 0) {
> if (depth_p != NULL && --(*depth_p) == 0)
> panic("pmap_pa_unmap: restoring but depth == 0");
>
> pvo->pvo_pte = *saved_pt;
>
> if (!pmap_pte_spill(pvo->pvo_vaddr))
> panic("pmap_pa_unmap: could not spill pvo %p", pvo);
> }
>
> critical_exit();
1758,1759c1555,1556
< void
< pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
---
> static void
> pmap_syncicache(vm_offset_t pa, vm_size_t len)
1760a1558,1559
> __syncicache((void *)pa, len);
> }
1762,1763c1561,1572
< /* XXX: coming soon... */
< return;
---
> static void
> tlbia(void)
> {
> caddr_t i;
>
> SYNC();
> for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) {
> TLBIE(i);
> EIEIO();
> }
> TLBSYNC();
> SYNC();
1766,1767c1575,1577
< void
< pmap_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
---
> static int
> pmap_pvo_enter(pmap_t pm, vm_zone_t zone, struct pvo_head *pvo_head,
> vm_offset_t va, vm_offset_t pa, u_int pte_lo, int flags)
1768a1579,1583
> struct pvo_entry *pvo;
> u_int sr;
> int first;
> u_int ptegidx;
> int i;
1770,1771c1585,1658
< /* XXX: coming soon... */
< return;
---
> pmap_pvo_enter_calls++;
>
> /*
> * Compute the PTE Group index.
> */
> va &= ~ADDR_POFF;
> sr = va_to_sr(pm->pm_sr, va);
> ptegidx = va_to_pteg(sr, va);
>
> critical_enter();
>
> /*
> * Remove any existing mapping for this page. Reuse the pvo entry if
> * there is a mapping.
> */
> LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
> if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
> pmap_pvo_remove(pvo, -1);
> break;
> }
> }
>
> /*
> * If we aren't overwriting a mapping, try to allocate.
> */
> critical_exit();
>
> pvo = zalloc(zone);
>
> critical_enter();
>
> if (pvo == NULL) {
> critical_exit();
> return (ENOMEM);
> }
>
> pmap_pvo_entries++;
> pvo->pvo_vaddr = va;
> pvo->pvo_pmap = pm;
> LIST_INSERT_HEAD(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
> pvo->pvo_vaddr &= ~ADDR_POFF;
> if (flags & VM_PROT_EXECUTE)
> pvo->pvo_vaddr |= PVO_EXECUTABLE;
> if (flags & PVO_WIRED)
> pvo->pvo_vaddr |= PVO_WIRED;
> if (pvo_head != &pmap_pvo_kunmanaged)
> pvo->pvo_vaddr |= PVO_MANAGED;
> pmap_pte_create(&pvo->pvo_pte, sr, va, pa | pte_lo);
>
> /*
> * Remember if the list was empty and therefore will be the first
> * item.
> */
> first = LIST_FIRST(pvo_head) == NULL;
>
> LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
> if (pvo->pvo_pte.pte_lo & PVO_WIRED)
> pvo->pvo_pmap->pm_stats.wired_count++;
> pvo->pvo_pmap->pm_stats.resident_count++;
>
> /*
> * We hope this succeeds but it isn't required.
> */
> i = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
> if (i >= 0) {
> PVO_PTEGIDX_SET(pvo, i);
> } else {
> panic("pmap_pvo_enter: overflow");
> pmap_pte_overflow++;
> }
>
> critical_exit();
>
> return (first ? ENOENT : 0);
1774,1775c1661,1662
< void
< pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
---
> static void
> pmap_pvo_remove(struct pvo_entry *pvo, int pteidx)
1776a1664
> struct pte *pt;
1778,1779c1666,1711
< /* XXX: coming soon... */
< return;
---
> /*
> * If there is an active pte entry, we need to deactivate it (and
> * save the ref & cfg bits).
> */
> pt = pmap_pvo_to_pte(pvo, pteidx);
> if (pt != NULL) {
> pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr);
> PVO_PTEGIDX_CLR(pvo);
> } else {
> pmap_pte_overflow--;
> }
>
> /*
> * Update our statistics.
> */
> pvo->pvo_pmap->pm_stats.resident_count--;
> if (pvo->pvo_pte.pte_lo & PVO_WIRED)
> pvo->pvo_pmap->pm_stats.wired_count--;
>
> /*
> * Save the REF/CHG bits into their cache if the page is managed.
> */
> if (pvo->pvo_vaddr & PVO_MANAGED) {
> struct vm_page *pg;
>
> pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte_lo * PTE_RPGN);
> if (pg != NULL) {
> pmap_attr_save(pg, pvo->pvo_pte.pte_lo &
> (PTE_REF | PTE_CHG));
> }
> }
>
> /*
> * Remove this PVO from the PV list.
> */
> LIST_REMOVE(pvo, pvo_vlink);
>
> /*
> * Remove this from the overflow list and return it to the pool
> * if we aren't going to reuse it.
> */
> LIST_REMOVE(pvo, pvo_olink);
> zfree(pvo->pvo_vaddr & PVO_MANAGED ? pmap_mpvo_zone : pmap_upvo_zone,
> pvo);
> pmap_pvo_entries--;
> pmap_pvo_remove_calls++;
1782,1783c1714,1715
< void
< pmap_pinit0(pmap_t pmap)
---
> static __inline int
> pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
1784a1717
> int pteidx;
1786,1787c1719,1728
< /* XXX: coming soon... */
< return;
---
> /*
> * We can find the actual pte entry without searching by grabbing
> * the PTEG index from 3 unused bits in pte_lo[11:9] and by
> * noticing the HID bit.
> */
> pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
> if (pvo->pvo_pte.pte_hi & PTE_HID)
> pteidx ^= pmap_pteg_mask * 8;
>
> return (pteidx);
1790,1791c1731,1732
< void
< pmap_dispose_proc(struct proc *p)
---
> static struct pvo_entry *
> pmap_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
1792a1734,1736
> struct pvo_entry *pvo;
> int ptegidx;
> u_int sr;
1794,1795c1738,1750
< /* XXX: coming soon... */
< return;
---
> va &= ~ADDR_POFF;
> sr = va_to_sr(pm->pm_sr, va);
> ptegidx = va_to_pteg(sr, va);
>
> LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
> if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
> if (pteidx_p)
> *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx);
> return (pvo);
> }
> }
>
> return (NULL);
1798,1799c1753,1754
< vm_offset_t
< pmap_steal_memory(vm_size_t size)
---
> static struct pte *
> pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
1801,1802c1756
< vm_size_t bank_size;
< vm_offset_t pa;
---
> struct pte *pt;
1804c1758,1763
< size = round_page(size);
---
> /*
> * If we haven't been supplied the ptegidx, calculate it.
> */
> if (pteidx == -1) {
> int ptegidx;
> u_int sr;
1806,1811c1765,1785
< bank_size = phys_avail[1] - phys_avail[0];
< while (size > bank_size) {
< int i;
< for (i = 0; phys_avail[i+2]; i+= 2) {
< phys_avail[i] = phys_avail[i+2];
< phys_avail[i+1] = phys_avail[i+3];
---
> sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr);
> ptegidx = va_to_pteg(sr, pvo->pvo_vaddr);
> pteidx = pmap_pvo_pte_index(pvo, ptegidx);
> }
>
> pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7];
>
> if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) {
> panic("pmap_pvo_to_pte: pvo %p has valid pte in pvo but no "
> "valid pte index", pvo);
> }
>
> if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) {
> panic("pmap_pvo_to_pte: pvo %p has valid pte index in pvo "
> "pvo but no valid pte", pvo);
> }
>
> if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) {
> if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) {
> panic("pmap_pvo_to_pte: pvo %p has valid pte in "
> "pmap_pteg_table %p but invalid in pvo", pvo, pt);
1813,1817c1787,1794
< phys_avail[i] = 0;
< phys_avail[i+1] = 0;
< if (!phys_avail[0])
< panic("pmap_steal_memory: out of memory");
< bank_size = phys_avail[1] - phys_avail[0];
---
>
> if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF))
> != 0) {
> panic("pmap_pvo_to_pte: pvo %p pte does not match "
> "pte %p in pmap_pteg_table", pvo, pt);
> }
>
> return (pt);
1820,1821c1797,1800
< pa = phys_avail[0];
< phys_avail[0] += size;
---
> if (pvo->pvo_pte.pte_hi & PTE_VALID) {
> panic("pmap_pvo_to_pte: pvo %p has invalid pte %p in "
> "pmap_pteg_table but valid in pvo", pvo, pt);
> }
1823,1824c1802
< bzero((caddr_t) pa, size);
< return pa;
---
> return (NULL);
1828,1829c1806
< * Create the UAREA_PAGES for a new process.
< * This routine directly affects the fork perf for a process.
---
> * XXX: THIS STUFF SHOULD BE IN pte.c?
1831,1832c1808,1809
< void
< pmap_new_proc(struct proc *p)
---
> int
> pmap_pte_spill(vm_offset_t addr)
1834,1841c1811,1816
< int i;
< vm_object_t upobj;
< vm_offset_t up;
< vm_page_t m;
< pte_t pte;
< sr_t sr;
< int idx;
< vm_offset_t va;
---
> struct pvo_entry *source_pvo, *victim_pvo;
> struct pvo_entry *pvo;
> int ptegidx, i, j;
> u_int sr;
> struct pteg *pteg;
> struct pte *pt;
1842a1818,1822
> pmap_pte_spills++;
>
> __asm __volatile("mfsrin %0,%1" : "=r"(sr) : "r"(addr));
> ptegidx = va_to_pteg(sr, addr);
>
1844c1824,1825
< * allocate object for the upages
---
> * Have to substitute some entry. Use the primary hash for this.
> * Use low bits of timebase as random generator.
1846,1850c1827,1830
< upobj = p->p_upages_obj;
< if (upobj == NULL) {
< upobj = vm_object_allocate(OBJT_DEFAULT, UAREA_PAGES);
< p->p_upages_obj = upobj;
< }
---
> pteg = &pmap_pteg_table[ptegidx];
> __asm __volatile("mftb %0" : "=r"(i));
> i &= 7;
> pt = &pteg->pt[i];
1852,1861c1832,1834
< /* get a kernel virtual address for the UAREA_PAGES for this proc */
< up = (vm_offset_t)p->p_uarea;
< if (up == 0) {
< up = kmem_alloc_nofault(kernel_map, UAREA_PAGES * PAGE_SIZE);
< if (up == 0)
< panic("pmap_new_proc: upage allocation failed");
< p->p_uarea = (struct user *)up;
< }
<
< for (i = 0; i < UAREA_PAGES; i++) {
---
> source_pvo = NULL;
> victim_pvo = NULL;
> LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
1863c1836
< * Get a kernel stack page
---
> * We need to find a pvo entry for this address.
1865c1838,1846
< m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
---
> PMAP_PVO_CHECK(pvo);
> if (source_pvo == NULL &&
> pmap_pte_match(&pvo->pvo_pte, sr, addr,
> pvo->pvo_pte.pte_hi & PTE_HID)) {
> /*
> * Now found an entry to be spilled into the pteg.
> * The PTE is now valid, so we know it's active.
> */
> j = pmap_pte_insert(ptegidx, &pvo->pvo_pte);
1866a1848,1860
> if (j >= 0) {
> PVO_PTEGIDX_SET(pvo, j);
> pmap_pte_overflow--;
> PMAP_PVO_CHECK(pvo);
> return (1);
> }
>
> source_pvo = pvo;
>
> if (victim_pvo != NULL)
> break;
> }
>
1868c1862,1863
< * Wire the page
---
> * We also need the pvo entry of the victim we are replacing
> * so save the R & C bits of the PTE.
1870,1871c1865,1871
< m->wire_count++;
< cnt.v_wire_count++;
---
> if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL &&
> pmap_pte_compare(pt, &pvo->pvo_pte)) {
> victim_pvo = pvo;
> if (source_pvo != NULL)
> break;
> }
> }
1872a1873,1880
> if (source_pvo == NULL)
> return (0);
>
> if (victim_pvo == NULL) {
> if ((pt->pte_hi & PTE_HID) == 0)
> panic("pmap_pte_spill: victim p-pte (%p) has no pvo"
> "entry", pt);
>
1874c1882,1883
< * Enter the page into the kernel address space.
---
> * If this is a secondary PTE, we need to search it's primary
> * pvo bucket for the matching PVO.
1876,1877c1885,1896
< va = up + i * PAGE_SIZE;
< idx = pteidx(sr = ptesr(kernel_pmap->pm_sr, va), va);
---
> LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx ^ pmap_pteg_mask],
> pvo_olink) {
> PMAP_PVO_CHECK(pvo);
> /*
> * We also need the pvo entry of the victim we are
> * replacing so save the R & C bits of the PTE.
> */
> if (pmap_pte_compare(pt, &pvo->pvo_pte)) {
> victim_pvo = pvo;
> break;
> }
> }
1879,1882c1898,1901
< pte.pte_hi = ((sr & SR_VSID) << PTE_VSID_SHFT) |
< ((va & ADDR_PIDX) >> ADDR_API_SHFT);
< pte.pte_lo = (VM_PAGE_TO_PHYS(m) & PTE_RPGN) | PTE_M | PTE_I |
< PTE_G | PTE_RW;
---
> if (victim_pvo == NULL)
> panic("pmap_pte_spill: victim s-pte (%p) has no pvo"
> "entry", pt);
> }
1884,1885c1903,1908
< if (!pte_insert(idx, &pte)) {
< struct pte_ovfl *po;
---
> /*
> * We are invalidating the TLB entry for the EA we are replacing even
> * though it's valid. If we don't, we lose any ref/chg bit changes
> * contained in the TLB entry.
> */
> source_pvo->pvo_pte.pte_hi &= ~PTE_HID;
1887,1890c1910,1911
< po = poalloc();
< po->po_pte = pte;
< LIST_INSERT_HEAD(potable + idx, po, po_list);
< }
---
> pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr);
> pmap_pte_set(pt, &source_pvo->pvo_pte);
1892c1913,1915
< tlbie(va);
---
> PVO_PTEGIDX_CLR(victim_pvo);
> PVO_PTEGIDX_SET(source_pvo, i);
> pmap_pte_replacements++;
1894,1897c1917,1937
< vm_page_wakeup(m);
< vm_page_flag_clear(m, PG_ZERO);
< vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE);
< m->valid = VM_PAGE_BITS_ALL;
---
> PMAP_PVO_CHECK(victim_pvo);
> PMAP_PVO_CHECK(source_pvo);
>
> return (1);
> }
>
> static int
> pmap_pte_insert(u_int ptegidx, struct pte *pvo_pt)
> {
> struct pte *pt;
> int i;
>
> /*
> * First try primary hash.
> */
> for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
> if ((pt->pte_hi & PTE_VALID) == 0) {
> pvo_pt->pte_hi &= ~PTE_HID;
> pmap_pte_set(pt, pvo_pt);
> return (i);
> }
1898a1939,1954
>
> /*
> * Now try secondary hash.
> */
> ptegidx ^= pmap_pteg_mask;
> ptegidx++;
> for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
> if ((pt->pte_hi & PTE_VALID) == 0) {
> pvo_pt->pte_hi |= PTE_HID;
> pmap_pte_set(pt, pvo_pt);
> return (i);
> }
> }
>
> panic("pmap_pte_insert: overflow");
> return (-1);
1901,1902c1957,1958
< void *
< pmap_mapdev(vm_offset_t pa, vm_size_t len)
---
> static boolean_t
> pmap_query_bit(vm_page_t m, int ptebit)
1904,1906c1960,1961
< vm_offset_t faddr;
< vm_offset_t taddr, va;
< int off;
---
> struct pvo_entry *pvo;
> struct pte *pt;
1908,1910c1963,1964
< faddr = trunc_page(pa);
< off = pa - faddr;
< len = round_page(off + len);
---
> if (pmap_attr_fetch(m) & ptebit)
> return (TRUE);
1912c1966
< GIANT_REQUIRED;
---
> critical_enter();
1914c1968,1969
< va = taddr = kmem_alloc_pageable(kernel_map, len);
---
> LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
> PMAP_PVO_CHECK(pvo); /* sanity check */
1916,1917c1971,1981
< if (va == 0)
< return NULL;
---
> /*
> * See if we saved the bit off. If so, cache it and return
> * success.
> */
> if (pvo->pvo_pte.pte_lo & ptebit) {
> pmap_attr_save(m, ptebit);
> PMAP_PVO_CHECK(pvo); /* sanity check */
> critical_exit();
> return (TRUE);
> }
> }
1919,1922c1983,2006
< for (; len > 0; len -= PAGE_SIZE) {
< pmap_kenter(taddr, faddr);
< faddr += PAGE_SIZE;
< taddr += PAGE_SIZE;
---
> /*
> * No luck, now go through the hard part of looking at the PTEs
> * themselves. Sync so that any pending REF/CHG bits are flushed to
> * the PTEs.
> */
> SYNC();
> LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
> PMAP_PVO_CHECK(pvo); /* sanity check */
>
> /*
> * See if this pvo has a valid PTE. if so, fetch the
> * REF/CHG bits from the valid PTE. If the appropriate
> * ptebit is set, cache it and return success.
> */
> pt = pmap_pvo_to_pte(pvo, -1);
> if (pt != NULL) {
> pmap_pte_synch(pt, &pvo->pvo_pte);
> if (pvo->pvo_pte.pte_lo & ptebit) {
> pmap_attr_save(m, ptebit);
> PMAP_PVO_CHECK(pvo); /* sanity check */
> critical_exit();
> return (TRUE);
> }
> }
1925c2009,2010
< return (void *)(va + off);
---
> critical_exit();
> return (TRUE);
1926a2012,2056
>
> static boolean_t
> pmap_clear_bit(vm_page_t m, int ptebit)
> {
> struct pvo_entry *pvo;
> struct pte *pt;
> int rv;
>
> critical_enter();
>
> /*
> * Clear the cached value.
> */
> rv = pmap_attr_fetch(m);
> pmap_attr_clear(m, ptebit);
>
> /*
> * Sync so that any pending REF/CHG bits are flushed to the PTEs (so
> * we can reset the right ones). note that since the pvo entries and
> * list heads are accessed via BAT0 and are never placed in the page
> * table, we don't have to worry about further accesses setting the
> * REF/CHG bits.
> */
> SYNC();
>
> /*
> * For each pvo entry, clear the pvo's ptebit. If this pvo has a
> * valid pte clear the ptebit from the valid pte.
> */
> LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
> PMAP_PVO_CHECK(pvo); /* sanity check */
> pt = pmap_pvo_to_pte(pvo, -1);
> if (pt != NULL) {
> pmap_pte_synch(pt, &pvo->pvo_pte);
> if (pvo->pvo_pte.pte_lo & ptebit)
> pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit);
> }
> rv |= pvo->pvo_pte.pte_lo;
> pvo->pvo_pte.pte_lo &= ~ptebit;
> PMAP_PVO_CHECK(pvo); /* sanity check */
> }
>
> critical_exit();
> return ((rv & ptebit) != 0);
> }