Deleted Added
sdiff udiff text old ( 191300 ) new ( 191735 )
full compact
pmap.c (191300) pmap.c (191735)
1/*
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *

--- 52 unchanged lines hidden (view full) ---

61 * this module may delay invalidate or reduced protection
62 * operations until such time as they are actually
63 * necessary. This module is given full information as
64 * to which processors are currently using which maps,
65 * and to when physical maps must be made correct.
66 */
67
68#include <sys/cdefs.h>
1/*
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *

--- 52 unchanged lines hidden (view full) ---

61 * this module may delay invalidate or reduced protection
62 * operations until such time as they are actually
63 * necessary. This module is given full information as
64 * to which processors are currently using which maps,
65 * and to when physical maps must be made correct.
66 */
67
68#include <sys/cdefs.h>
69__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 191300 2009-04-20 03:44:54Z alc $");
69__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 191735 2009-05-02 06:12:38Z alc $");
70
71#include "opt_ddb.h"
72#include "opt_msgbuf.h"
73#include <sys/param.h>
74#include <sys/systm.h>
75#include <sys/proc.h>
76#include <sys/msgbuf.h>
77#include <sys/vmmeter.h>

--- 52 unchanged lines hidden (view full) ---

130#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
131
132#define MIPS_SEGSIZE (1L << SEGSHIFT)
133#define mips_segtrunc(va) ((va) & ~(MIPS_SEGSIZE-1))
134#define pmap_TLB_invalidate_all() MIPS_TBIAP()
135#define pmap_va_asid(pmap, va) ((va) | ((pmap)->pm_asid[PCPU_GET(cpuid)].asid << VMTLB_PID_SHIFT))
136#define is_kernel_pmap(x) ((x) == kernel_pmap)
137
70
71#include "opt_ddb.h"
72#include "opt_msgbuf.h"
73#include <sys/param.h>
74#include <sys/systm.h>
75#include <sys/proc.h>
76#include <sys/msgbuf.h>
77#include <sys/vmmeter.h>

--- 52 unchanged lines hidden (view full) ---

130#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
131
132#define MIPS_SEGSIZE (1L << SEGSHIFT)
133#define mips_segtrunc(va) ((va) & ~(MIPS_SEGSIZE-1))
134#define pmap_TLB_invalidate_all() MIPS_TBIAP()
135#define pmap_va_asid(pmap, va) ((va) | ((pmap)->pm_asid[PCPU_GET(cpuid)].asid << VMTLB_PID_SHIFT))
136#define is_kernel_pmap(x) ((x) == kernel_pmap)
137
138static struct pmap kernel_pmap_store;
139pmap_t kernel_pmap;
138struct pmap kernel_pmap_store;
140pd_entry_t *kernel_segmap;
141
139pd_entry_t *kernel_segmap;
140
142vm_offset_t avail_start; /* PA of first available physical page */
143vm_offset_t avail_end; /* PA of last available physical page */
144vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
145vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
146
147static int nkpt;
148unsigned pmap_max_asid; /* max ASID supported by the system */
149
150
151#define PMAP_ASID_RESERVED 0

--- 4 unchanged lines hidden (view full) ---

156static void pmap_asid_alloc(pmap_t pmap);
157
158/*
159 * Data for the pv entry allocation mechanism
160 */
161static uma_zone_t pvzone;
162static struct vm_object pvzone_obj;
163static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
141vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
142vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
143
144static int nkpt;
145unsigned pmap_max_asid; /* max ASID supported by the system */
146
147
148#define PMAP_ASID_RESERVED 0

--- 4 unchanged lines hidden (view full) ---

153static void pmap_asid_alloc(pmap_t pmap);
154
155/*
156 * Data for the pv entry allocation mechanism
157 */
158static uma_zone_t pvzone;
159static struct vm_object pvzone_obj;
160static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
164int pmap_pagedaemon_waken = 0;
165
166struct fpage fpages_shared[FPAGES_SHARED];
167
168struct sysmaps sysmaps_pcpu[MAXCPU];
169
170static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
171static pv_entry_t get_pv_entry(pmap_t locked_pmap);
172static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem);

--- 234 unchanged lines hidden (view full) ---

407 * level page table.
408 */
409 printf("init kernel_segmap va >> = %d nkpt:%d\n",
410 (virtual_avail >> SEGSHIFT),
411 nkpt);
412 for (i = 0, j = (virtual_avail >> SEGSHIFT); i < nkpt; i++, j++)
413 kernel_segmap[j] = (pd_entry_t)(pgtab + (i * NPTEPG));
414
161
162struct fpage fpages_shared[FPAGES_SHARED];
163
164struct sysmaps sysmaps_pcpu[MAXCPU];
165
166static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
167static pv_entry_t get_pv_entry(pmap_t locked_pmap);
168static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem);

--- 234 unchanged lines hidden (view full) ---

403 * level page table.
404 */
405 printf("init kernel_segmap va >> = %d nkpt:%d\n",
406 (virtual_avail >> SEGSHIFT),
407 nkpt);
408 for (i = 0, j = (virtual_avail >> SEGSHIFT); i < nkpt; i++, j++)
409 kernel_segmap[j] = (pd_entry_t)(pgtab + (i * NPTEPG));
410
415 avail_start = phys_avail[0];
416 for (i = 0; phys_avail[i + 2]; i += 2)
417 continue;
411 for (i = 0; phys_avail[i + 2]; i += 2)
412 continue;
418 avail_end = phys_avail[i + 1];
413 printf("avail_start:0x%x avail_end:0x%x\n",
414 phys_avail[0], phys_avail[i + 1]);
419
420 /*
421 * The kernel's pmap is statically allocated so we don't have to use
422 * pmap_create, which is unlikely to work correctly at this part of
423 * the boot sequence (XXX and which no longer exists).
424 */
415
416 /*
417 * The kernel's pmap is statically allocated so we don't have to use
418 * pmap_create, which is unlikely to work correctly at this part of
419 * the boot sequence (XXX and which no longer exists).
420 */
425 kernel_pmap = &kernel_pmap_store;
426
427 PMAP_LOCK_INIT(kernel_pmap);
428 kernel_pmap->pm_segtab = kernel_segmap;
429 kernel_pmap->pm_active = ~0;
430 TAILQ_INIT(&kernel_pmap->pm_pvlist);
421 PMAP_LOCK_INIT(kernel_pmap);
422 kernel_pmap->pm_segtab = kernel_segmap;
423 kernel_pmap->pm_active = ~0;
424 TAILQ_INIT(&kernel_pmap->pm_pvlist);
431 printf("avail_start:0x%x avail_end:0x%x\n",
432 avail_start, avail_end);
433
434 kernel_pmap->pm_asid[PCPU_GET(cpuid)].asid = PMAP_ASID_RESERVED;
435 kernel_pmap->pm_asid[PCPU_GET(cpuid)].gen = 0;
436 pmap_max_asid = VMNUM_PIDS;
437 MachSetPID(0);
438}
439
440/*
441 * Initialize a vm_page's machine-dependent fields.

--- 564 unchanged lines hidden (view full) ---

1006
1007#ifdef VM_ALLOC_WIRED_TLB_PG_POOL
1008 if (need_wired_tlb_page_pool)
1009 req |= VM_ALLOC_WIRED_TLB_PG_POOL;
1010#endif
1011 /*
1012 * allocate the page directory page
1013 */
425 kernel_pmap->pm_asid[PCPU_GET(cpuid)].asid = PMAP_ASID_RESERVED;
426 kernel_pmap->pm_asid[PCPU_GET(cpuid)].gen = 0;
427 pmap_max_asid = VMNUM_PIDS;
428 MachSetPID(0);
429}
430
431/*
432 * Initialize a vm_page's machine-dependent fields.

--- 564 unchanged lines hidden (view full) ---

997
998#ifdef VM_ALLOC_WIRED_TLB_PG_POOL
999 if (need_wired_tlb_page_pool)
1000 req |= VM_ALLOC_WIRED_TLB_PG_POOL;
1001#endif
1002 /*
1003 * allocate the page directory page
1004 */
1014 ptdpg = vm_page_alloc(NULL, NUSERPGTBLS, req);
1005 while ((ptdpg = vm_page_alloc(NULL, NUSERPGTBLS, req)) == NULL)
1006 VM_WAIT;
1015
1007
1016#if 0
1017 /* I think we can just delete these, now that PG_BUSY is gone */
1018 vm_page_lock_queues();
1019 vm_page_flag_clear(ptdpg, PTE_BUSY); /* not usually mapped */
1020#endif
1021 ptdpg->valid = VM_PAGE_BITS_ALL;
1022
1008 ptdpg->valid = VM_PAGE_BITS_ALL;
1009
1023#if 0
1024 vm_page_unlock_queues();
1025#endif
1026
1027 pmap->pm_segtab = (pd_entry_t *)
1028 MIPS_PHYS_TO_CACHED(VM_PAGE_TO_PHYS(ptdpg));
1029 if ((ptdpg->flags & PG_ZERO) == 0)
1030 bzero(pmap->pm_segtab, PAGE_SIZE);
1031
1032 pmap->pm_active = 0;
1033 pmap->pm_ptphint = NULL;
1034 for (i = 0; i < MAXCPU; i++) {

--- 153 unchanged lines hidden (view full) ---

1188{
1189 vm_page_t ptdpg;
1190
1191 KASSERT(pmap->pm_stats.resident_count == 0,
1192 ("pmap_release: pmap resident count %ld != 0",
1193 pmap->pm_stats.resident_count));
1194
1195 ptdpg = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pmap->pm_segtab));
1010 pmap->pm_segtab = (pd_entry_t *)
1011 MIPS_PHYS_TO_CACHED(VM_PAGE_TO_PHYS(ptdpg));
1012 if ((ptdpg->flags & PG_ZERO) == 0)
1013 bzero(pmap->pm_segtab, PAGE_SIZE);
1014
1015 pmap->pm_active = 0;
1016 pmap->pm_ptphint = NULL;
1017 for (i = 0; i < MAXCPU; i++) {

--- 153 unchanged lines hidden (view full) ---

1171{
1172 vm_page_t ptdpg;
1173
1174 KASSERT(pmap->pm_stats.resident_count == 0,
1175 ("pmap_release: pmap resident count %ld != 0",
1176 pmap->pm_stats.resident_count));
1177
1178 ptdpg = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pmap->pm_segtab));
1196
1197 vm_page_lock_queues();
1198 ptdpg->wire_count--;
1199 atomic_subtract_int(&cnt.v_wire_count, 1);
1200 vm_page_free_zero(ptdpg);
1179 ptdpg->wire_count--;
1180 atomic_subtract_int(&cnt.v_wire_count, 1);
1181 vm_page_free_zero(ptdpg);
1201 vm_page_unlock_queues();
1202}
1203
1204/*
1205 * grow the number of kernel page table entries, if needed
1206 */
1207void
1208pmap_growkernel(vm_offset_t addr)
1209{

--- 232 unchanged lines hidden (view full) ---

1442/*
1443 * Create a pv entry for page at pa for
1444 * (pmap, va).
1445 */
1446static void
1447pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m,
1448 boolean_t wired)
1449{
1182}
1183
1184/*
1185 * grow the number of kernel page table entries, if needed
1186 */
1187void
1188pmap_growkernel(vm_offset_t addr)
1189{

--- 232 unchanged lines hidden (view full) ---

1422/*
1423 * Create a pv entry for page at pa for
1424 * (pmap, va).
1425 */
1426static void
1427pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m,
1428 boolean_t wired)
1429{
1450
1451 pv_entry_t pv;
1452
1453 pv = get_pv_entry(pmap);
1454 pv->pv_va = va;
1455 pv->pv_pmap = pmap;
1456 pv->pv_ptem = mpte;
1457 pv->pv_wired = wired;
1458
1459 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1460 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1461 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
1462 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1463 m->md.pv_list_count++;
1430 pv_entry_t pv;
1431
1432 pv = get_pv_entry(pmap);
1433 pv->pv_va = va;
1434 pv->pv_pmap = pmap;
1435 pv->pv_ptem = mpte;
1436 pv->pv_wired = wired;
1437
1438 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1439 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1440 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
1441 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
1442 m->md.pv_list_count++;
1464
1465}
1466
1467/*
1468 * Conditionally create a pv entry.
1469 */
1470static boolean_t
1471pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va,
1472 vm_page_t m)

--- 54 unchanged lines hidden (view full) ---

1527 }
1528 if (m->md.pv_flags & PV_TABLE_REF)
1529 vm_page_flag_set(m, PG_REFERENCED);
1530 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD);
1531
1532 pmap_remove_entry(pmap, m, va);
1533 }
1534 return pmap_unuse_pt(pmap, va, NULL);
1443}
1444
1445/*
1446 * Conditionally create a pv entry.
1447 */
1448static boolean_t
1449pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va,
1450 vm_page_t m)

--- 54 unchanged lines hidden (view full) ---

1505 }
1506 if (m->md.pv_flags & PV_TABLE_REF)
1507 vm_page_flag_set(m, PG_REFERENCED);
1508 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD);
1509
1510 pmap_remove_entry(pmap, m, va);
1511 }
1512 return pmap_unuse_pt(pmap, va, NULL);
1535
1536}
1537
1538/*
1539 * Remove a single page from a process address space
1540 */
1541static void
1542pmap_remove_page(struct pmap *pmap, vm_offset_t va)
1543{

--- 74 unchanged lines hidden (view full) ---

1618 */
1619
1620void
1621pmap_remove_all(vm_page_t m)
1622{
1623 register pv_entry_t pv;
1624 register pt_entry_t *pte, tpte;
1625
1513}
1514
1515/*
1516 * Remove a single page from a process address space
1517 */
1518static void
1519pmap_remove_page(struct pmap *pmap, vm_offset_t va)
1520{

--- 74 unchanged lines hidden (view full) ---

1595 */
1596
1597void
1598pmap_remove_all(vm_page_t m)
1599{
1600 register pv_entry_t pv;
1601 register pt_entry_t *pte, tpte;
1602
1626#if defined(PMAP_DEBUG)
1627 /*
1628 * XXX This makes pmap_remove_all() illegal for non-managed pages!
1629 */
1630 if (m->flags & PG_FICTITIOUS) {
1631 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%x", VM_PAGE_TO_PHYS(m));
1632 }
1633#endif
1603 KASSERT((m->flags & PG_FICTITIOUS) == 0,
1604 ("pmap_remove_all: page %p is fictitious", m));
1634 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1635
1636 if (m->md.pv_flags & PV_TABLE_REF)
1637 vm_page_flag_set(m, PG_REFERENCED);
1638
1639 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1640 PMAP_LOCK(pv->pv_pmap);
1641 pv->pv_pmap->pm_stats.resident_count--;

--- 837 unchanged lines hidden (view full) ---

2479pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
2480{
2481 pv_entry_t pv;
2482 int loops = 0;
2483
2484 if (m->flags & PG_FICTITIOUS)
2485 return FALSE;
2486
1605 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1606
1607 if (m->md.pv_flags & PV_TABLE_REF)
1608 vm_page_flag_set(m, PG_REFERENCED);
1609
1610 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
1611 PMAP_LOCK(pv->pv_pmap);
1612 pv->pv_pmap->pm_stats.resident_count--;

--- 837 unchanged lines hidden (view full) ---

2450pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
2451{
2452 pv_entry_t pv;
2453 int loops = 0;
2454
2455 if (m->flags & PG_FICTITIOUS)
2456 return FALSE;
2457
2487 vm_page_lock_queues();
2488 PMAP_LOCK(pmap);
2489
2490 /*
2491 * Not found, check current mappings returning immediately if found.
2492 */
2458 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
2493 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2494 if (pv->pv_pmap == pmap) {
2459 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
2460 if (pv->pv_pmap == pmap) {
2495 PMAP_UNLOCK(pmap);
2496 vm_page_unlock_queues();
2497 return TRUE;
2498 }
2499 loops++;
2500 if (loops >= 16)
2501 break;
2502 }
2461 return TRUE;
2462 }
2463 loops++;
2464 if (loops >= 16)
2465 break;
2466 }
2503 PMAP_UNLOCK(pmap);
2504 vm_page_unlock_queues();
2505 return (FALSE);
2506}
2507
2467 return (FALSE);
2468}
2469
2508#define PMAP_REMOVE_PAGES_CURPROC_ONLY
2509/*
2510 * Remove all pages from specified address space
2511 * this aids process exit speeds. Also, this code
2512 * is special cased for current process only, but
2513 * can have the more generic (and slightly slower)
2514 * mode enabled. This is much faster than pmap_remove
2515 * in the case of running down an entire address space.
2516 */
2517void
2518pmap_remove_pages(pmap_t pmap)
2519{
2520 pt_entry_t *pte, tpte;
2521 pv_entry_t pv, npv;
2522 vm_page_t m;
2523
2470/*
2471 * Remove all pages from specified address space
2472 * this aids process exit speeds. Also, this code
2473 * is special cased for current process only, but
2474 * can have the more generic (and slightly slower)
2475 * mode enabled. This is much faster than pmap_remove
2476 * in the case of running down an entire address space.
2477 */
2478void
2479pmap_remove_pages(pmap_t pmap)
2480{
2481 pt_entry_t *pte, tpte;
2482 pv_entry_t pv, npv;
2483 vm_page_t m;
2484
2524#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
2525 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
2526 printf("warning: pmap_remove_pages called with non-current pmap\n");
2527 return;
2528 }
2485 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
2486 printf("warning: pmap_remove_pages called with non-current pmap\n");
2487 return;
2488 }
2529#endif
2530
2531 vm_page_lock_queues();
2532 PMAP_LOCK(pmap);
2533 sched_pin();
2534 //XXX need to be TAILQ_FOREACH_SAFE ?
2535 for (pv = TAILQ_FIRST(&pmap->pm_pvlist);
2536 pv;
2537 pv = npv) {
2538

--- 728 unchanged lines hidden (view full) ---

3267#ifdef VM_ALLOC_WIRED_TLB_PG_POOL
3268 else if (need_wired_tlb_page_pool && ((va >= VM_MIN_KERNEL_ADDRESS) &&
3269 (va < (VM_MIN_KERNEL_ADDRESS + VM_KERNEL_ALLOC_OFFSET))))
3270 pa = MIPS_CACHED_TO_PHYS(va);
3271#endif
3272 else if (va >= MIPS_KSEG2_START && va < VM_MAX_KERNEL_ADDRESS) {
3273 pt_entry_t *ptep;
3274
2489 vm_page_lock_queues();
2490 PMAP_LOCK(pmap);
2491 sched_pin();
2492 //XXX need to be TAILQ_FOREACH_SAFE ?
2493 for (pv = TAILQ_FIRST(&pmap->pm_pvlist);
2494 pv;
2495 pv = npv) {
2496

--- 728 unchanged lines hidden (view full) ---

3225#ifdef VM_ALLOC_WIRED_TLB_PG_POOL
3226 else if (need_wired_tlb_page_pool && ((va >= VM_MIN_KERNEL_ADDRESS) &&
3227 (va < (VM_MIN_KERNEL_ADDRESS + VM_KERNEL_ALLOC_OFFSET))))
3228 pa = MIPS_CACHED_TO_PHYS(va);
3229#endif
3230 else if (va >= MIPS_KSEG2_START && va < VM_MAX_KERNEL_ADDRESS) {
3231 pt_entry_t *ptep;
3232
3275 if (kernel_pmap) {
3233 /* Is the kernel pmap initialized? */
3234 if (kernel_pmap->pm_active) {
3276 if (va >= (vm_offset_t)virtual_sys_start) {
3277 /* Its inside the virtual address range */
3278 ptep = pmap_pte(kernel_pmap, va);
3279 if (ptep)
3280 pa = mips_tlbpfn_to_paddr(*ptep) |
3281 (va & PAGE_MASK);
3282 } else {
3283 int i;

--- 24 unchanged lines hidden --- 		3235 if (va >= (vm_offset_t)virtual_sys_start) {
3236 /* Its inside the virtual address range */
3237 ptep = pmap_pte(kernel_pmap, va);
3238 if (ptep)
3239 pa = mips_tlbpfn_to_paddr(*ptep) |
3240 (va & PAGE_MASK);
3241 } else {
3242 int i;

--- 24 unchanged lines hidden ---