234
235 /*
236 * Compute the number of pages of memory that will be available for
237 * use (taking into account the overhead of a page structure per
238 * page).
239 */
240 first_page = phys_avail[0] / PAGE_SIZE;
241 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
242 npages = (total - (page_range * sizeof(struct vm_page)) -
243 (end - new_end)) / PAGE_SIZE;
244 end = new_end;
245
246 /*
247 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
248 */
249 vaddr += PAGE_SIZE;
250
251 /*
252 * Initialize the mem entry structures now, and put them in the free
253 * queue.
254 */
255 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
256 mapped = pmap_map(&vaddr, new_end, end,
257 VM_PROT_READ | VM_PROT_WRITE);
258 vm_page_array = (vm_page_t) mapped;
259 phys_avail[biggestone + 1] = new_end;
260
261 /*
262 * Clear all of the page structures
263 */
264 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
265 vm_page_array_size = page_range;
266
267 /*
268 * Construct the free queue(s) in descending order (by physical
269 * address) so that the first 16MB of physical memory is allocated
270 * last rather than first. On large-memory machines, this avoids
271 * the exhaustion of low physical memory before isa_dma_init has run.
272 */
273 cnt.v_page_count = 0;
274 cnt.v_free_count = 0;
275 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
276 pa = phys_avail[i];
277 last_pa = phys_avail[i + 1];
278 while (pa < last_pa && npages-- > 0) {
279 vm_pageq_add_new_page(pa);
280 pa += PAGE_SIZE;
281 }
282 }
283 return (vaddr);
284}
285
286void
287vm_page_flag_set(vm_page_t m, unsigned short bits)
288{
289
290 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
291 m->flags |= bits;
292}
293
294void
295vm_page_flag_clear(vm_page_t m, unsigned short bits)
296{
297
298 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
299 m->flags &= ~bits;
300}
301
302void
303vm_page_busy(vm_page_t m)
304{
305
306 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
307 KASSERT((m->flags & PG_BUSY) == 0,
308 ("vm_page_busy: page already busy!!!"));
309 vm_page_flag_set(m, PG_BUSY);
310}
311
312/*
313 * vm_page_flash:
314 *
315 * wakeup anyone waiting for the page.
316 */
317void
318vm_page_flash(vm_page_t m)
319{
320
321 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
322 if (m->flags & PG_WANTED) {
323 vm_page_flag_clear(m, PG_WANTED);
324 wakeup(m);
325 }
326}
327
328/*
329 * vm_page_wakeup:
330 *
331 * clear the PG_BUSY flag and wakeup anyone waiting for the
332 * page.
333 *
334 */
335void
336vm_page_wakeup(vm_page_t m)
337{
338
339 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
340 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
341 vm_page_flag_clear(m, PG_BUSY);
342 vm_page_flash(m);
343}
344
345void
346vm_page_io_start(vm_page_t m)
347{
348
349 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
350 m->busy++;
351}
352
353void
354vm_page_io_finish(vm_page_t m)
355{
356
357 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
358 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
359 m->busy--;
360 if (m->busy == 0)
361 vm_page_flash(m);
362}
363
364/*
365 * Keep page from being freed by the page daemon
366 * much of the same effect as wiring, except much lower
367 * overhead and should be used only for *very* temporary
368 * holding ("wiring").
369 */
370void
371vm_page_hold(vm_page_t mem)
372{
373
374 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
375 mem->hold_count++;
376}
377
378void
379vm_page_unhold(vm_page_t mem)
380{
381
382 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
383 --mem->hold_count;
384 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
385 if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
386 vm_page_free_toq(mem);
387}
388
389/*
390 * vm_page_free:
391 *
392 * Free a page
393 *
394 * The clearing of PG_ZERO is a temporary safety until the code can be
395 * reviewed to determine that PG_ZERO is being properly cleared on
396 * write faults or maps. PG_ZERO was previously cleared in
397 * vm_page_alloc().
398 */
399void
400vm_page_free(vm_page_t m)
401{
402 vm_page_flag_clear(m, PG_ZERO);
403 vm_page_free_toq(m);
404 vm_page_zero_idle_wakeup();
405}
406
407/*
408 * vm_page_free_zero:
409 *
410 * Free a page to the zerod-pages queue
411 */
412void
413vm_page_free_zero(vm_page_t m)
414{
415 vm_page_flag_set(m, PG_ZERO);
416 vm_page_free_toq(m);
417}
418
419/*
420 * vm_page_sleep_if_busy:
421 *
422 * Sleep and release the page queues lock if PG_BUSY is set or,
423 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
424 * thread slept and the page queues lock was released.
425 * Otherwise, retains the page queues lock and returns FALSE.
426 */
427int
428vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
429{
430 vm_object_t object;
431
432 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
433 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
434 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
435 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
436 /*
437 * It's possible that while we sleep, the page will get
438 * unbusied and freed. If we are holding the object
439 * lock, we will assume we hold a reference to the object
440 * such that even if m->object changes, we can re-lock
441 * it.
442 */
443 object = m->object;
444 VM_OBJECT_UNLOCK(object);
445 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
446 VM_OBJECT_LOCK(object);
447 return (TRUE);
448 }
449 return (FALSE);
450}
451
452/*
453 * vm_page_dirty:
454 *
455 * make page all dirty
456 */
457void
458vm_page_dirty(vm_page_t m)
459{
460 KASSERT(m->queue - m->pc != PQ_CACHE,
461 ("vm_page_dirty: page in cache!"));
462 KASSERT(m->queue - m->pc != PQ_FREE,
463 ("vm_page_dirty: page is free!"));
464 m->dirty = VM_PAGE_BITS_ALL;
465}
466
467/*
468 * vm_page_splay:
469 *
470 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
471 * the vm_page containing the given pindex. If, however, that
472 * pindex is not found in the vm_object, returns a vm_page that is
473 * adjacent to the pindex, coming before or after it.
474 */
475vm_page_t
476vm_page_splay(vm_pindex_t pindex, vm_page_t root)
477{
478 struct vm_page dummy;
479 vm_page_t lefttreemax, righttreemin, y;
480
481 if (root == NULL)
482 return (root);
483 lefttreemax = righttreemin = &dummy;
484 for (;; root = y) {
485 if (pindex < root->pindex) {
486 if ((y = root->left) == NULL)
487 break;
488 if (pindex < y->pindex) {
489 /* Rotate right. */
490 root->left = y->right;
491 y->right = root;
492 root = y;
493 if ((y = root->left) == NULL)
494 break;
495 }
496 /* Link into the new root's right tree. */
497 righttreemin->left = root;
498 righttreemin = root;
499 } else if (pindex > root->pindex) {
500 if ((y = root->right) == NULL)
501 break;
502 if (pindex > y->pindex) {
503 /* Rotate left. */
504 root->right = y->left;
505 y->left = root;
506 root = y;
507 if ((y = root->right) == NULL)
508 break;
509 }
510 /* Link into the new root's left tree. */
511 lefttreemax->right = root;
512 lefttreemax = root;
513 } else
514 break;
515 }
516 /* Assemble the new root. */
517 lefttreemax->right = root->left;
518 righttreemin->left = root->right;
519 root->left = dummy.right;
520 root->right = dummy.left;
521 return (root);
522}
523
524/*
525 * vm_page_insert: [ internal use only ]
526 *
527 * Inserts the given mem entry into the object and object list.
528 *
529 * The pagetables are not updated but will presumably fault the page
530 * in if necessary, or if a kernel page the caller will at some point
531 * enter the page into the kernel's pmap. We are not allowed to block
532 * here so we *can't* do this anyway.
533 *
534 * The object and page must be locked.
535 * This routine may not block.
536 */
537void
538vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
539{
540 vm_page_t root;
541
542 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
543 if (m->object != NULL)
544 panic("vm_page_insert: page already inserted");
545
546 /*
547 * Record the object/offset pair in this page
548 */
549 m->object = object;
550 m->pindex = pindex;
551
552 /*
553 * Now link into the object's ordered list of backed pages.
554 */
555 root = object->root;
556 if (root == NULL) {
557 m->left = NULL;
558 m->right = NULL;
559 TAILQ_INSERT_TAIL(&object->memq, m, listq);
560 } else {
561 root = vm_page_splay(pindex, root);
562 if (pindex < root->pindex) {
563 m->left = root->left;
564 m->right = root;
565 root->left = NULL;
566 TAILQ_INSERT_BEFORE(root, m, listq);
567 } else if (pindex == root->pindex)
568 panic("vm_page_insert: offset already allocated");
569 else {
570 m->right = root->right;
571 m->left = root;
572 root->right = NULL;
573 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
574 }
575 }
576 object->root = m;
577 object->generation++;
578
579 /*
580 * show that the object has one more resident page.
581 */
582 object->resident_page_count++;
583 /*
584 * Hold the vnode until the last page is released.
585 */
586 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
587 vhold((struct vnode *)object->handle);
588
589 /*
590 * Since we are inserting a new and possibly dirty page,
591 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
592 */
593 if (m->flags & PG_WRITEABLE)
594 vm_object_set_writeable_dirty(object);
595}
596
597/*
598 * vm_page_remove:
599 * NOTE: used by device pager as well -wfj
600 *
601 * Removes the given mem entry from the object/offset-page
602 * table and the object page list, but do not invalidate/terminate
603 * the backing store.
604 *
605 * The object and page must be locked.
606 * The underlying pmap entry (if any) is NOT removed here.
607 * This routine may not block.
608 */
609void
610vm_page_remove(vm_page_t m)
611{
612 vm_object_t object;
613 vm_page_t root;
614
615 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
616 if ((object = m->object) == NULL)
617 return;
618 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
619 if (m->flags & PG_BUSY) {
620 vm_page_flag_clear(m, PG_BUSY);
621 vm_page_flash(m);
622 }
623
624 /*
625 * Now remove from the object's list of backed pages.
626 */
627 if (m != object->root)
628 vm_page_splay(m->pindex, object->root);
629 if (m->left == NULL)
630 root = m->right;
631 else {
632 root = vm_page_splay(m->pindex, m->left);
633 root->right = m->right;
634 }
635 object->root = root;
636 TAILQ_REMOVE(&object->memq, m, listq);
637
638 /*
639 * And show that the object has one fewer resident page.
640 */
641 object->resident_page_count--;
642 object->generation++;
643 /*
644 * The vnode may now be recycled.
645 */
646 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
647 vdrop((struct vnode *)object->handle);
648
649 m->object = NULL;
650}
651
652/*
653 * vm_page_lookup:
654 *
655 * Returns the page associated with the object/offset
656 * pair specified; if none is found, NULL is returned.
657 *
658 * The object must be locked.
659 * This routine may not block.
660 * This is a critical path routine
661 */
662vm_page_t
663vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
664{
665 vm_page_t m;
666
667 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
668 if ((m = object->root) != NULL && m->pindex != pindex) {
669 m = vm_page_splay(pindex, m);
670 if ((object->root = m)->pindex != pindex)
671 m = NULL;
672 }
673 return (m);
674}
675
676/*
677 * vm_page_rename:
678 *
679 * Move the given memory entry from its
680 * current object to the specified target object/offset.
681 *
682 * The object must be locked.
683 * This routine may not block.
684 *
685 * Note: swap associated with the page must be invalidated by the move. We
686 * have to do this for several reasons: (1) we aren't freeing the
687 * page, (2) we are dirtying the page, (3) the VM system is probably
688 * moving the page from object A to B, and will then later move
689 * the backing store from A to B and we can't have a conflict.
690 *
691 * Note: we *always* dirty the page. It is necessary both for the
692 * fact that we moved it, and because we may be invalidating
693 * swap. If the page is on the cache, we have to deactivate it
694 * or vm_page_dirty() will panic. Dirty pages are not allowed
695 * on the cache.
696 */
697void
698vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
699{
700
701 vm_page_remove(m);
702 vm_page_insert(m, new_object, new_pindex);
703 if (m->queue - m->pc == PQ_CACHE)
704 vm_page_deactivate(m);
705 vm_page_dirty(m);
706}
707
708/*
709 * vm_page_select_cache:
710 *
711 * Move a page of the given color from the cache queue to the free
712 * queue. As pages might be found, but are not applicable, they are
713 * deactivated.
714 *
715 * This routine may not block.
716 */
717vm_page_t
718vm_page_select_cache(int color)
719{
720 vm_object_t object;
721 vm_page_t m;
722 boolean_t was_trylocked;
723
724 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
725 while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
726 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
727 KASSERT(!pmap_page_is_mapped(m),
728 ("Found mapped cache page %p", m));
729 KASSERT((m->flags & PG_UNMANAGED) == 0,
730 ("Found unmanaged cache page %p", m));
731 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
732 if (m->hold_count == 0 && (object = m->object,
733 (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
734 VM_OBJECT_LOCKED(object))) {
735 KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
736 ("Found busy cache page %p", m));
737 vm_page_free(m);
738 if (was_trylocked)
739 VM_OBJECT_UNLOCK(object);
740 break;
741 }
742 vm_page_deactivate(m);
743 }
744 return (m);
745}
746
747/*
748 * vm_page_alloc:
749 *
750 * Allocate and return a memory cell associated
751 * with this VM object/offset pair.
752 *
753 * page_req classes:
754 * VM_ALLOC_NORMAL normal process request
755 * VM_ALLOC_SYSTEM system *really* needs a page
756 * VM_ALLOC_INTERRUPT interrupt time request
757 * VM_ALLOC_ZERO zero page
758 *
759 * This routine may not block.
760 *
761 * Additional special handling is required when called from an
762 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with
763 * the page cache in this case.
764 */
765vm_page_t
766vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
767{
768 vm_page_t m = NULL;
769 int color, flags, page_req;
770
771 page_req = req & VM_ALLOC_CLASS_MASK;
772 KASSERT(curthread->td_intr_nesting_level == 0 ||
773 page_req == VM_ALLOC_INTERRUPT,
774 ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
775
776 if ((req & VM_ALLOC_NOOBJ) == 0) {
777 KASSERT(object != NULL,
778 ("vm_page_alloc: NULL object."));
779 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
780 color = (pindex + object->pg_color) & PQ_L2_MASK;
781 } else
782 color = pindex & PQ_L2_MASK;
783
784 /*
785 * The pager is allowed to eat deeper into the free page list.
786 */
787 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
788 page_req = VM_ALLOC_SYSTEM;
789 };
790
791loop:
792 mtx_lock_spin(&vm_page_queue_free_mtx);
793 if (cnt.v_free_count > cnt.v_free_reserved ||
794 (page_req == VM_ALLOC_SYSTEM &&
795 cnt.v_cache_count == 0 &&
796 cnt.v_free_count > cnt.v_interrupt_free_min) ||
797 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
798 /*
799 * Allocate from the free queue if the number of free pages
800 * exceeds the minimum for the request class.
801 */
802 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
803 } else if (page_req != VM_ALLOC_INTERRUPT) {
804 mtx_unlock_spin(&vm_page_queue_free_mtx);
805 /*
806 * Allocatable from cache (non-interrupt only). On success,
807 * we must free the page and try again, thus ensuring that
808 * cnt.v_*_free_min counters are replenished.
809 */
810 vm_page_lock_queues();
811 if ((m = vm_page_select_cache(color)) == NULL) {
812#if defined(DIAGNOSTIC)
813 if (cnt.v_cache_count > 0)
814 printf("vm_page_alloc(NORMAL): missing pages on cache queue: %d\n", cnt.v_cache_count);
815#endif
816 vm_page_unlock_queues();
817 atomic_add_int(&vm_pageout_deficit, 1);
818 pagedaemon_wakeup();
819 return (NULL);
820 }
821 vm_page_unlock_queues();
822 goto loop;
823 } else {
824 /*
825 * Not allocatable from cache from interrupt, give up.
826 */
827 mtx_unlock_spin(&vm_page_queue_free_mtx);
828 atomic_add_int(&vm_pageout_deficit, 1);
829 pagedaemon_wakeup();
830 return (NULL);
831 }
832
833 /*
834 * At this point we had better have found a good page.
835 */
836
837 KASSERT(
838 m != NULL,
839 ("vm_page_alloc(): missing page on free queue")
840 );
841
842 /*
843 * Remove from free queue
844 */
845 vm_pageq_remove_nowakeup(m);
846
847 /*
848 * Initialize structure. Only the PG_ZERO flag is inherited.
849 */
850 flags = PG_BUSY;
851 if (m->flags & PG_ZERO) {
852 vm_page_zero_count--;
853 if (req & VM_ALLOC_ZERO)
854 flags = PG_ZERO | PG_BUSY;
855 }
856 if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
857 flags &= ~PG_BUSY;
858 m->flags = flags;
859 if (req & VM_ALLOC_WIRED) {
860 atomic_add_int(&cnt.v_wire_count, 1);
861 m->wire_count = 1;
862 } else
863 m->wire_count = 0;
864 m->hold_count = 0;
865 m->act_count = 0;
866 m->busy = 0;
867 m->valid = 0;
868 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
869 mtx_unlock_spin(&vm_page_queue_free_mtx);
870
871 if ((req & VM_ALLOC_NOOBJ) == 0)
872 vm_page_insert(m, object, pindex);
873 else
874 m->pindex = pindex;
875
876 /*
877 * Don't wakeup too often - wakeup the pageout daemon when
878 * we would be nearly out of memory.
879 */
880 if (vm_paging_needed())
881 pagedaemon_wakeup();
882
883 return (m);
884}
885
886/*
887 * vm_wait: (also see VM_WAIT macro)
888 *
889 * Block until free pages are available for allocation
890 * - Called in various places before memory allocations.
891 */
892void
893vm_wait(void)
894{
895
896 vm_page_lock_queues();
897 if (curproc == pageproc) {
898 vm_pageout_pages_needed = 1;
899 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
900 PDROP | PSWP, "VMWait", 0);
901 } else {
902 if (!vm_pages_needed) {
903 vm_pages_needed = 1;
904 wakeup(&vm_pages_needed);
905 }
906 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
907 "vmwait", 0);
908 }
909}
910
911/*
912 * vm_waitpfault: (also see VM_WAITPFAULT macro)
913 *
914 * Block until free pages are available for allocation
915 * - Called only in vm_fault so that processes page faulting
916 * can be easily tracked.
917 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
918 * processes will be able to grab memory first. Do not change
919 * this balance without careful testing first.
920 */
921void
922vm_waitpfault(void)
923{
924
925 vm_page_lock_queues();
926 if (!vm_pages_needed) {
927 vm_pages_needed = 1;
928 wakeup(&vm_pages_needed);
929 }
930 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
931 "pfault", 0);
932}
933
934/*
935 * vm_page_activate:
936 *
937 * Put the specified page on the active list (if appropriate).
938 * Ensure that act_count is at least ACT_INIT but do not otherwise
939 * mess with it.
940 *
941 * The page queues must be locked.
942 * This routine may not block.
943 */
944void
945vm_page_activate(vm_page_t m)
946{
947
948 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
949 if (m->queue != PQ_ACTIVE) {
950 if ((m->queue - m->pc) == PQ_CACHE)
951 cnt.v_reactivated++;
952 vm_pageq_remove(m);
953 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
954 if (m->act_count < ACT_INIT)
955 m->act_count = ACT_INIT;
956 vm_pageq_enqueue(PQ_ACTIVE, m);
957 }
958 } else {
959 if (m->act_count < ACT_INIT)
960 m->act_count = ACT_INIT;
961 }
962}
963
964/*
965 * vm_page_free_wakeup:
966 *
967 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
968 * routine is called when a page has been added to the cache or free
969 * queues.
970 *
971 * The page queues must be locked.
972 * This routine may not block.
973 */
974static __inline void
975vm_page_free_wakeup(void)
976{
977
978 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
979 /*
980 * if pageout daemon needs pages, then tell it that there are
981 * some free.
982 */
983 if (vm_pageout_pages_needed &&
984 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
985 wakeup(&vm_pageout_pages_needed);
986 vm_pageout_pages_needed = 0;
987 }
988 /*
989 * wakeup processes that are waiting on memory if we hit a
990 * high water mark. And wakeup scheduler process if we have
991 * lots of memory. this process will swapin processes.
992 */
993 if (vm_pages_needed && !vm_page_count_min()) {
994 vm_pages_needed = 0;
995 wakeup(&cnt.v_free_count);
996 }
997}
998
999/*
1000 * vm_page_free_toq:
1001 *
1002 * Returns the given page to the PQ_FREE list,
1003 * disassociating it with any VM object.
1004 *
1005 * Object and page must be locked prior to entry.
1006 * This routine may not block.
1007 */
1008
1009void
1010vm_page_free_toq(vm_page_t m)
1011{
1012 struct vpgqueues *pq;
1013
1014 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1015 cnt.v_tfree++;
1016
1017 if (m->busy || ((m->queue - m->pc) == PQ_FREE)) {
1018 printf(
1019 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
1020 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
1021 m->hold_count);
1022 if ((m->queue - m->pc) == PQ_FREE)
1023 panic("vm_page_free: freeing free page");
1024 else
1025 panic("vm_page_free: freeing busy page");
1026 }
1027
1028 /*
1029 * unqueue, then remove page. Note that we cannot destroy
1030 * the page here because we do not want to call the pager's
1031 * callback routine until after we've put the page on the
1032 * appropriate free queue.
1033 */
1034 vm_pageq_remove_nowakeup(m);
1035 vm_page_remove(m);
1036
1037 /*
1038 * If fictitious remove object association and
1039 * return, otherwise delay object association removal.
1040 */
1041 if ((m->flags & PG_FICTITIOUS) != 0) {
1042 return;
1043 }
1044
1045 m->valid = 0;
1046 vm_page_undirty(m);
1047
1048 if (m->wire_count != 0) {
1049 if (m->wire_count > 1) {
1050 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1051 m->wire_count, (long)m->pindex);
1052 }
1053 panic("vm_page_free: freeing wired page");
1054 }
1055
1056 /*
1057 * Clear the UNMANAGED flag when freeing an unmanaged page.
1058 */
1059 if (m->flags & PG_UNMANAGED) {
1060 m->flags &= ~PG_UNMANAGED;
1061 }
1062
1063 if (m->hold_count != 0) {
1064 m->flags &= ~PG_ZERO;
1065 m->queue = PQ_HOLD;
1066 } else
1067 m->queue = PQ_FREE + m->pc;
1068 pq = &vm_page_queues[m->queue];
1069 mtx_lock_spin(&vm_page_queue_free_mtx);
1070 pq->lcnt++;
1071 ++(*pq->cnt);
1072
1073 /*
1074 * Put zero'd pages on the end ( where we look for zero'd pages
1075 * first ) and non-zerod pages at the head.
1076 */
1077 if (m->flags & PG_ZERO) {
1078 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
1079 ++vm_page_zero_count;
1080 } else {
1081 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
1082 }
1083 mtx_unlock_spin(&vm_page_queue_free_mtx);
1084 vm_page_free_wakeup();
1085}
1086
1087/*
1088 * vm_page_unmanage:
1089 *
1090 * Prevent PV management from being done on the page. The page is
1091 * removed from the paging queues as if it were wired, and as a
1092 * consequence of no longer being managed the pageout daemon will not
1093 * touch it (since there is no way to locate the pte mappings for the
1094 * page). madvise() calls that mess with the pmap will also no longer
1095 * operate on the page.
1096 *
1097 * Beyond that the page is still reasonably 'normal'. Freeing the page
1098 * will clear the flag.
1099 *
1100 * This routine is used by OBJT_PHYS objects - objects using unswappable
1101 * physical memory as backing store rather then swap-backed memory and
1102 * will eventually be extended to support 4MB unmanaged physical
1103 * mappings.
1104 */
1105void
1106vm_page_unmanage(vm_page_t m)
1107{
1108
1109 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1110 if ((m->flags & PG_UNMANAGED) == 0) {
1111 if (m->wire_count == 0)
1112 vm_pageq_remove(m);
1113 }
1114 vm_page_flag_set(m, PG_UNMANAGED);
1115}
1116
1117/*
1118 * vm_page_wire:
1119 *
1120 * Mark this page as wired down by yet
1121 * another map, removing it from paging queues
1122 * as necessary.
1123 *
1124 * The page queues must be locked.
1125 * This routine may not block.
1126 */
1127void
1128vm_page_wire(vm_page_t m)
1129{
1130
1131 /*
1132 * Only bump the wire statistics if the page is not already wired,
1133 * and only unqueue the page if it is on some queue (if it is unmanaged
1134 * it is already off the queues).
1135 */
1136 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1137 if (m->flags & PG_FICTITIOUS)
1138 return;
1139 if (m->wire_count == 0) {
1140 if ((m->flags & PG_UNMANAGED) == 0)
1141 vm_pageq_remove(m);
1142 atomic_add_int(&cnt.v_wire_count, 1);
1143 }
1144 m->wire_count++;
1145 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1146}
1147
1148/*
1149 * vm_page_unwire:
1150 *
1151 * Release one wiring of this page, potentially
1152 * enabling it to be paged again.
1153 *
1154 * Many pages placed on the inactive queue should actually go
1155 * into the cache, but it is difficult to figure out which. What
1156 * we do instead, if the inactive target is well met, is to put
1157 * clean pages at the head of the inactive queue instead of the tail.
1158 * This will cause them to be moved to the cache more quickly and
1159 * if not actively re-referenced, freed more quickly. If we just
1160 * stick these pages at the end of the inactive queue, heavy filesystem
1161 * meta-data accesses can cause an unnecessary paging load on memory bound
1162 * processes. This optimization causes one-time-use metadata to be
1163 * reused more quickly.
1164 *
1165 * BUT, if we are in a low-memory situation we have no choice but to
1166 * put clean pages on the cache queue.
1167 *
1168 * A number of routines use vm_page_unwire() to guarantee that the page
1169 * will go into either the inactive or active queues, and will NEVER
1170 * be placed in the cache - for example, just after dirtying a page.
1171 * dirty pages in the cache are not allowed.
1172 *
1173 * The page queues must be locked.
1174 * This routine may not block.
1175 */
1176void
1177vm_page_unwire(vm_page_t m, int activate)
1178{
1179
1180 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1181 if (m->flags & PG_FICTITIOUS)
1182 return;
1183 if (m->wire_count > 0) {
1184 m->wire_count--;
1185 if (m->wire_count == 0) {
1186 atomic_subtract_int(&cnt.v_wire_count, 1);
1187 if (m->flags & PG_UNMANAGED) {
1188 ;
1189 } else if (activate)
1190 vm_pageq_enqueue(PQ_ACTIVE, m);
1191 else {
1192 vm_page_flag_clear(m, PG_WINATCFLS);
1193 vm_pageq_enqueue(PQ_INACTIVE, m);
1194 }
1195 }
1196 } else {
1197 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1198 }
1199}
1200
1201
1202/*
1203 * Move the specified page to the inactive queue. If the page has
1204 * any associated swap, the swap is deallocated.
1205 *
1206 * Normally athead is 0 resulting in LRU operation. athead is set
1207 * to 1 if we want this page to be 'as if it were placed in the cache',
1208 * except without unmapping it from the process address space.
1209 *
1210 * This routine may not block.
1211 */
1212static __inline void
1213_vm_page_deactivate(vm_page_t m, int athead)
1214{
1215
1216 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1217
1218 /*
1219 * Ignore if already inactive.
1220 */
1221 if (m->queue == PQ_INACTIVE)
1222 return;
1223 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1224 if ((m->queue - m->pc) == PQ_CACHE)
1225 cnt.v_reactivated++;
1226 vm_page_flag_clear(m, PG_WINATCFLS);
1227 vm_pageq_remove(m);
1228 if (athead)
1229 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1230 else
1231 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1232 m->queue = PQ_INACTIVE;
1233 vm_page_queues[PQ_INACTIVE].lcnt++;
1234 cnt.v_inactive_count++;
1235 }
1236}
1237
1238void
1239vm_page_deactivate(vm_page_t m)
1240{
1241 _vm_page_deactivate(m, 0);
1242}
1243
1244/*
1245 * vm_page_try_to_cache:
1246 *
1247 * Returns 0 on failure, 1 on success
1248 */
1249int
1250vm_page_try_to_cache(vm_page_t m)
1251{
1252
1253 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1254 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1255 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1256 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1257 return (0);
1258 }
1259 pmap_remove_all(m);
1260 if (m->dirty)
1261 return (0);
1262 vm_page_cache(m);
1263 return (1);
1264}
1265
1266/*
1267 * vm_page_try_to_free()
1268 *
1269 * Attempt to free the page. If we cannot free it, we do nothing.
1270 * 1 is returned on success, 0 on failure.
1271 */
1272int
1273vm_page_try_to_free(vm_page_t m)
1274{
1275
1276 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1277 if (m->object != NULL)
1278 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1279 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1280 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1281 return (0);
1282 }
1283 pmap_remove_all(m);
1284 if (m->dirty)
1285 return (0);
1286 vm_page_free(m);
1287 return (1);
1288}
1289
1290/*
1291 * vm_page_cache
1292 *
1293 * Put the specified page onto the page cache queue (if appropriate).
1294 *
1295 * This routine may not block.
1296 */
1297void
1298vm_page_cache(vm_page_t m)
1299{
1300
1301 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1302 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1303 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
1304 m->hold_count || m->wire_count) {
1305 printf("vm_page_cache: attempting to cache busy page\n");
1306 return;
1307 }
1308 if ((m->queue - m->pc) == PQ_CACHE)
1309 return;
1310
1311 /*
1312 * Remove all pmaps and indicate that the page is not
1313 * writeable or mapped.
1314 */
1315 pmap_remove_all(m);
1316 if (m->dirty != 0) {
1317 panic("vm_page_cache: caching a dirty page, pindex: %ld",
1318 (long)m->pindex);
1319 }
1320 vm_pageq_remove_nowakeup(m);
1321 vm_pageq_enqueue(PQ_CACHE + m->pc, m);
1322 vm_page_free_wakeup();
1323}
1324
1325/*
1326 * vm_page_dontneed
1327 *
1328 * Cache, deactivate, or do nothing as appropriate. This routine
1329 * is typically used by madvise() MADV_DONTNEED.
1330 *
1331 * Generally speaking we want to move the page into the cache so
1332 * it gets reused quickly. However, this can result in a silly syndrome
1333 * due to the page recycling too quickly. Small objects will not be
1334 * fully cached. On the otherhand, if we move the page to the inactive
1335 * queue we wind up with a problem whereby very large objects
1336 * unnecessarily blow away our inactive and cache queues.
1337 *
1338 * The solution is to move the pages based on a fixed weighting. We
1339 * either leave them alone, deactivate them, or move them to the cache,
1340 * where moving them to the cache has the highest weighting.
1341 * By forcing some pages into other queues we eventually force the
1342 * system to balance the queues, potentially recovering other unrelated
1343 * space from active. The idea is to not force this to happen too
1344 * often.
1345 */
1346void
1347vm_page_dontneed(vm_page_t m)
1348{
1349 static int dnweight;
1350 int dnw;
1351 int head;
1352
1353 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1354 dnw = ++dnweight;
1355
1356 /*
1357 * occassionally leave the page alone
1358 */
1359 if ((dnw & 0x01F0) == 0 ||
1360 m->queue == PQ_INACTIVE ||
1361 m->queue - m->pc == PQ_CACHE
1362 ) {
1363 if (m->act_count >= ACT_INIT)
1364 --m->act_count;
1365 return;
1366 }
1367
1368 if (m->dirty == 0 && pmap_is_modified(m))
1369 vm_page_dirty(m);
1370
1371 if (m->dirty || (dnw & 0x0070) == 0) {
1372 /*
1373 * Deactivate the page 3 times out of 32.
1374 */
1375 head = 0;
1376 } else {
1377 /*
1378 * Cache the page 28 times out of every 32. Note that
1379 * the page is deactivated instead of cached, but placed
1380 * at the head of the queue instead of the tail.
1381 */
1382 head = 1;
1383 }
1384 _vm_page_deactivate(m, head);
1385}
1386
1387/*
1388 * Grab a page, waiting until we are waken up due to the page
1389 * changing state. We keep on waiting, if the page continues
1390 * to be in the object. If the page doesn't exist, first allocate it
1391 * and then conditionally zero it.
1392 *
1393 * This routine may block.
1394 */
1395vm_page_t
1396vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1397{
1398 vm_page_t m;
1399
1400 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1401retrylookup:
1402 if ((m = vm_page_lookup(object, pindex)) != NULL) {
1403 vm_page_lock_queues();
1404 if (m->busy || (m->flags & PG_BUSY)) {
1405 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
1406 VM_OBJECT_UNLOCK(object);
1407 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
1408 VM_OBJECT_LOCK(object);
1409 if ((allocflags & VM_ALLOC_RETRY) == 0)
1410 return (NULL);
1411 goto retrylookup;
1412 } else {
1413 if (allocflags & VM_ALLOC_WIRED)
1414 vm_page_wire(m);
1415 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1416 vm_page_busy(m);
1417 vm_page_unlock_queues();
1418 return (m);
1419 }
1420 }
1421 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1422 if (m == NULL) {
1423 VM_OBJECT_UNLOCK(object);
1424 VM_WAIT;
1425 VM_OBJECT_LOCK(object);
1426 if ((allocflags & VM_ALLOC_RETRY) == 0)
1427 return (NULL);
1428 goto retrylookup;
1429 }
1430 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1431 pmap_zero_page(m);
1432 return (m);
1433}
1434
1435/*
1436 * Mapping function for valid bits or for dirty bits in
1437 * a page. May not block.
1438 *
1439 * Inputs are required to range within a page.
1440 */
1441__inline int
1442vm_page_bits(int base, int size)
1443{
1444 int first_bit;
1445 int last_bit;
1446
1447 KASSERT(
1448 base + size <= PAGE_SIZE,
1449 ("vm_page_bits: illegal base/size %d/%d", base, size)
1450 );
1451
1452 if (size == 0) /* handle degenerate case */
1453 return (0);
1454
1455 first_bit = base >> DEV_BSHIFT;
1456 last_bit = (base + size - 1) >> DEV_BSHIFT;
1457
1458 return ((2 << last_bit) - (1 << first_bit));
1459}
1460
1461/*
1462 * vm_page_set_validclean:
1463 *
1464 * Sets portions of a page valid and clean. The arguments are expected
1465 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1466 * of any partial chunks touched by the range. The invalid portion of
1467 * such chunks will be zero'd.
1468 *
1469 * This routine may not block.
1470 *
1471 * (base + size) must be less then or equal to PAGE_SIZE.
1472 */
1473void
1474vm_page_set_validclean(vm_page_t m, int base, int size)
1475{
1476 int pagebits;
1477 int frag;
1478 int endoff;
1479
1480 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1481 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1482 if (size == 0) /* handle degenerate case */
1483 return;
1484
1485 /*
1486 * If the base is not DEV_BSIZE aligned and the valid
1487 * bit is clear, we have to zero out a portion of the
1488 * first block.
1489 */
1490 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1491 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1492 pmap_zero_page_area(m, frag, base - frag);
1493
1494 /*
1495 * If the ending offset is not DEV_BSIZE aligned and the
1496 * valid bit is clear, we have to zero out a portion of
1497 * the last block.
1498 */
1499 endoff = base + size;
1500 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1501 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1502 pmap_zero_page_area(m, endoff,
1503 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1504
1505 /*
1506 * Set valid, clear dirty bits. If validating the entire
1507 * page we can safely clear the pmap modify bit. We also
1508 * use this opportunity to clear the PG_NOSYNC flag. If a process
1509 * takes a write fault on a MAP_NOSYNC memory area the flag will
1510 * be set again.
1511 *
1512 * We set valid bits inclusive of any overlap, but we can only
1513 * clear dirty bits for DEV_BSIZE chunks that are fully within
1514 * the range.
1515 */
1516 pagebits = vm_page_bits(base, size);
1517 m->valid |= pagebits;
1518#if 0 /* NOT YET */
1519 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1520 frag = DEV_BSIZE - frag;
1521 base += frag;
1522 size -= frag;
1523 if (size < 0)
1524 size = 0;
1525 }
1526 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1527#endif
1528 m->dirty &= ~pagebits;
1529 if (base == 0 && size == PAGE_SIZE) {
1530 pmap_clear_modify(m);
1531 vm_page_flag_clear(m, PG_NOSYNC);
1532 }
1533}
1534
1535void
1536vm_page_clear_dirty(vm_page_t m, int base, int size)
1537{
1538
1539 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1540 m->dirty &= ~vm_page_bits(base, size);
1541}
1542
1543/*
1544 * vm_page_set_invalid:
1545 *
1546 * Invalidates DEV_BSIZE'd chunks within a page. Both the
1547 * valid and dirty bits for the effected areas are cleared.
1548 *
1549 * May not block.
1550 */
1551void
1552vm_page_set_invalid(vm_page_t m, int base, int size)
1553{
1554 int bits;
1555
1556 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1557 bits = vm_page_bits(base, size);
1558 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1559 m->valid &= ~bits;
1560 m->dirty &= ~bits;
1561 m->object->generation++;
1562}
1563
1564/*
1565 * vm_page_zero_invalid()
1566 *
1567 * The kernel assumes that the invalid portions of a page contain
1568 * garbage, but such pages can be mapped into memory by user code.
1569 * When this occurs, we must zero out the non-valid portions of the
1570 * page so user code sees what it expects.
1571 *
1572 * Pages are most often semi-valid when the end of a file is mapped
1573 * into memory and the file's size is not page aligned.
1574 */
1575void
1576vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1577{
1578 int b;
1579 int i;
1580
1581 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1582 /*
1583 * Scan the valid bits looking for invalid sections that
1584 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
1585 * valid bit may be set ) have already been zerod by
1586 * vm_page_set_validclean().
1587 */
1588 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1589 if (i == (PAGE_SIZE / DEV_BSIZE) ||
1590 (m->valid & (1 << i))
1591 ) {
1592 if (i > b) {
1593 pmap_zero_page_area(m,
1594 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1595 }
1596 b = i + 1;
1597 }
1598 }
1599
1600 /*
1601 * setvalid is TRUE when we can safely set the zero'd areas
1602 * as being valid. We can do this if there are no cache consistancy
1603 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
1604 */
1605 if (setvalid)
1606 m->valid = VM_PAGE_BITS_ALL;
1607}
1608
1609/*
1610 * vm_page_is_valid:
1611 *
1612 * Is (partial) page valid? Note that the case where size == 0
1613 * will return FALSE in the degenerate case where the page is
1614 * entirely invalid, and TRUE otherwise.
1615 *
1616 * May not block.
1617 */
1618int
1619vm_page_is_valid(vm_page_t m, int base, int size)
1620{
1621 int bits = vm_page_bits(base, size);
1622
1623 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1624 if (m->valid && ((m->valid & bits) == bits))
1625 return 1;
1626 else
1627 return 0;
1628}
1629
1630/*
1631 * update dirty bits from pmap/mmu. May not block.
1632 */
1633void
1634vm_page_test_dirty(vm_page_t m)
1635{
1636 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1637 vm_page_dirty(m);
1638 }
1639}
1640
1641int so_zerocp_fullpage = 0;
1642
1643void
1644vm_page_cowfault(vm_page_t m)
1645{
1646 vm_page_t mnew;
1647 vm_object_t object;
1648 vm_pindex_t pindex;
1649
1650 object = m->object;
1651 pindex = m->pindex;
1652
1653 retry_alloc:
1654 vm_page_remove(m);
1655 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
1656 if (mnew == NULL) {
1657 vm_page_insert(m, object, pindex);
1658 vm_page_unlock_queues();
1659 VM_OBJECT_UNLOCK(object);
1660 VM_WAIT;
1661 VM_OBJECT_LOCK(object);
1662 vm_page_lock_queues();
1663 goto retry_alloc;
1664 }
1665
1666 if (m->cow == 0) {
1667 /*
1668 * check to see if we raced with an xmit complete when
1669 * waiting to allocate a page. If so, put things back
1670 * the way they were
1671 */
1672 vm_page_free(mnew);
1673 vm_page_insert(m, object, pindex);
1674 } else { /* clear COW & copy page */
1675 if (!so_zerocp_fullpage)
1676 pmap_copy_page(m, mnew);
1677 mnew->valid = VM_PAGE_BITS_ALL;
1678 vm_page_dirty(mnew);
1679 vm_page_flag_clear(mnew, PG_BUSY);
1680 }
1681}
1682
1683void
1684vm_page_cowclear(vm_page_t m)
1685{
1686
1687 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1688 if (m->cow) {
1689 m->cow--;
1690 /*
1691 * let vm_fault add back write permission lazily
1692 */
1693 }
1694 /*
1695 * sf_buf_free() will free the page, so we needn't do it here
1696 */
1697}
1698
1699void
1700vm_page_cowsetup(vm_page_t m)
1701{
1702
1703 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1704 m->cow++;
1705 pmap_page_protect(m, VM_PROT_READ);
1706}
1707
1708#include "opt_ddb.h"
1709#ifdef DDB
1710#include <sys/kernel.h>
1711
1712#include <ddb/ddb.h>
1713
1714DB_SHOW_COMMAND(page, vm_page_print_page_info)
1715{
1716 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
1717 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
1718 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
1719 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
1720 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
1721 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
1722 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
1723 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
1724 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
1725 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
1726}
1727
1728DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
1729{
1730 int i;
1731 db_printf("PQ_FREE:");
1732 for (i = 0; i < PQ_L2_SIZE; i++) {
1733 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
1734 }
1735 db_printf("\n");
1736
1737 db_printf("PQ_CACHE:");
1738 for (i = 0; i < PQ_L2_SIZE; i++) {
1739 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
1740 }
1741 db_printf("\n");
1742
1743 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
1744 vm_page_queues[PQ_ACTIVE].lcnt,
1745 vm_page_queues[PQ_INACTIVE].lcnt);
1746}
1747#endif /* DDB */
| 234
235 /*
236 * Compute the number of pages of memory that will be available for
237 * use (taking into account the overhead of a page structure per
238 * page).
239 */
240 first_page = phys_avail[0] / PAGE_SIZE;
241 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
242 npages = (total - (page_range * sizeof(struct vm_page)) -
243 (end - new_end)) / PAGE_SIZE;
244 end = new_end;
245
246 /*
247 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
248 */
249 vaddr += PAGE_SIZE;
250
251 /*
252 * Initialize the mem entry structures now, and put them in the free
253 * queue.
254 */
255 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
256 mapped = pmap_map(&vaddr, new_end, end,
257 VM_PROT_READ | VM_PROT_WRITE);
258 vm_page_array = (vm_page_t) mapped;
259 phys_avail[biggestone + 1] = new_end;
260
261 /*
262 * Clear all of the page structures
263 */
264 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
265 vm_page_array_size = page_range;
266
267 /*
268 * Construct the free queue(s) in descending order (by physical
269 * address) so that the first 16MB of physical memory is allocated
270 * last rather than first. On large-memory machines, this avoids
271 * the exhaustion of low physical memory before isa_dma_init has run.
272 */
273 cnt.v_page_count = 0;
274 cnt.v_free_count = 0;
275 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
276 pa = phys_avail[i];
277 last_pa = phys_avail[i + 1];
278 while (pa < last_pa && npages-- > 0) {
279 vm_pageq_add_new_page(pa);
280 pa += PAGE_SIZE;
281 }
282 }
283 return (vaddr);
284}
285
286void
287vm_page_flag_set(vm_page_t m, unsigned short bits)
288{
289
290 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
291 m->flags |= bits;
292}
293
294void
295vm_page_flag_clear(vm_page_t m, unsigned short bits)
296{
297
298 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
299 m->flags &= ~bits;
300}
301
302void
303vm_page_busy(vm_page_t m)
304{
305
306 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
307 KASSERT((m->flags & PG_BUSY) == 0,
308 ("vm_page_busy: page already busy!!!"));
309 vm_page_flag_set(m, PG_BUSY);
310}
311
312/*
313 * vm_page_flash:
314 *
315 * wakeup anyone waiting for the page.
316 */
317void
318vm_page_flash(vm_page_t m)
319{
320
321 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
322 if (m->flags & PG_WANTED) {
323 vm_page_flag_clear(m, PG_WANTED);
324 wakeup(m);
325 }
326}
327
328/*
329 * vm_page_wakeup:
330 *
331 * clear the PG_BUSY flag and wakeup anyone waiting for the
332 * page.
333 *
334 */
335void
336vm_page_wakeup(vm_page_t m)
337{
338
339 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
340 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
341 vm_page_flag_clear(m, PG_BUSY);
342 vm_page_flash(m);
343}
344
345void
346vm_page_io_start(vm_page_t m)
347{
348
349 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
350 m->busy++;
351}
352
353void
354vm_page_io_finish(vm_page_t m)
355{
356
357 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
358 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
359 m->busy--;
360 if (m->busy == 0)
361 vm_page_flash(m);
362}
363
364/*
365 * Keep page from being freed by the page daemon
366 * much of the same effect as wiring, except much lower
367 * overhead and should be used only for *very* temporary
368 * holding ("wiring").
369 */
370void
371vm_page_hold(vm_page_t mem)
372{
373
374 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
375 mem->hold_count++;
376}
377
378void
379vm_page_unhold(vm_page_t mem)
380{
381
382 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
383 --mem->hold_count;
384 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
385 if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
386 vm_page_free_toq(mem);
387}
388
389/*
390 * vm_page_free:
391 *
392 * Free a page
393 *
394 * The clearing of PG_ZERO is a temporary safety until the code can be
395 * reviewed to determine that PG_ZERO is being properly cleared on
396 * write faults or maps. PG_ZERO was previously cleared in
397 * vm_page_alloc().
398 */
399void
400vm_page_free(vm_page_t m)
401{
402 vm_page_flag_clear(m, PG_ZERO);
403 vm_page_free_toq(m);
404 vm_page_zero_idle_wakeup();
405}
406
407/*
408 * vm_page_free_zero:
409 *
410 * Free a page to the zerod-pages queue
411 */
412void
413vm_page_free_zero(vm_page_t m)
414{
415 vm_page_flag_set(m, PG_ZERO);
416 vm_page_free_toq(m);
417}
418
419/*
420 * vm_page_sleep_if_busy:
421 *
422 * Sleep and release the page queues lock if PG_BUSY is set or,
423 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
424 * thread slept and the page queues lock was released.
425 * Otherwise, retains the page queues lock and returns FALSE.
426 */
427int
428vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
429{
430 vm_object_t object;
431
432 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
433 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
434 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
435 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
436 /*
437 * It's possible that while we sleep, the page will get
438 * unbusied and freed. If we are holding the object
439 * lock, we will assume we hold a reference to the object
440 * such that even if m->object changes, we can re-lock
441 * it.
442 */
443 object = m->object;
444 VM_OBJECT_UNLOCK(object);
445 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
446 VM_OBJECT_LOCK(object);
447 return (TRUE);
448 }
449 return (FALSE);
450}
451
452/*
453 * vm_page_dirty:
454 *
455 * make page all dirty
456 */
457void
458vm_page_dirty(vm_page_t m)
459{
460 KASSERT(m->queue - m->pc != PQ_CACHE,
461 ("vm_page_dirty: page in cache!"));
462 KASSERT(m->queue - m->pc != PQ_FREE,
463 ("vm_page_dirty: page is free!"));
464 m->dirty = VM_PAGE_BITS_ALL;
465}
466
467/*
468 * vm_page_splay:
469 *
470 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
471 * the vm_page containing the given pindex. If, however, that
472 * pindex is not found in the vm_object, returns a vm_page that is
473 * adjacent to the pindex, coming before or after it.
474 */
475vm_page_t
476vm_page_splay(vm_pindex_t pindex, vm_page_t root)
477{
478 struct vm_page dummy;
479 vm_page_t lefttreemax, righttreemin, y;
480
481 if (root == NULL)
482 return (root);
483 lefttreemax = righttreemin = &dummy;
484 for (;; root = y) {
485 if (pindex < root->pindex) {
486 if ((y = root->left) == NULL)
487 break;
488 if (pindex < y->pindex) {
489 /* Rotate right. */
490 root->left = y->right;
491 y->right = root;
492 root = y;
493 if ((y = root->left) == NULL)
494 break;
495 }
496 /* Link into the new root's right tree. */
497 righttreemin->left = root;
498 righttreemin = root;
499 } else if (pindex > root->pindex) {
500 if ((y = root->right) == NULL)
501 break;
502 if (pindex > y->pindex) {
503 /* Rotate left. */
504 root->right = y->left;
505 y->left = root;
506 root = y;
507 if ((y = root->right) == NULL)
508 break;
509 }
510 /* Link into the new root's left tree. */
511 lefttreemax->right = root;
512 lefttreemax = root;
513 } else
514 break;
515 }
516 /* Assemble the new root. */
517 lefttreemax->right = root->left;
518 righttreemin->left = root->right;
519 root->left = dummy.right;
520 root->right = dummy.left;
521 return (root);
522}
523
524/*
525 * vm_page_insert: [ internal use only ]
526 *
527 * Inserts the given mem entry into the object and object list.
528 *
529 * The pagetables are not updated but will presumably fault the page
530 * in if necessary, or if a kernel page the caller will at some point
531 * enter the page into the kernel's pmap. We are not allowed to block
532 * here so we *can't* do this anyway.
533 *
534 * The object and page must be locked.
535 * This routine may not block.
536 */
537void
538vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
539{
540 vm_page_t root;
541
542 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
543 if (m->object != NULL)
544 panic("vm_page_insert: page already inserted");
545
546 /*
547 * Record the object/offset pair in this page
548 */
549 m->object = object;
550 m->pindex = pindex;
551
552 /*
553 * Now link into the object's ordered list of backed pages.
554 */
555 root = object->root;
556 if (root == NULL) {
557 m->left = NULL;
558 m->right = NULL;
559 TAILQ_INSERT_TAIL(&object->memq, m, listq);
560 } else {
561 root = vm_page_splay(pindex, root);
562 if (pindex < root->pindex) {
563 m->left = root->left;
564 m->right = root;
565 root->left = NULL;
566 TAILQ_INSERT_BEFORE(root, m, listq);
567 } else if (pindex == root->pindex)
568 panic("vm_page_insert: offset already allocated");
569 else {
570 m->right = root->right;
571 m->left = root;
572 root->right = NULL;
573 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
574 }
575 }
576 object->root = m;
577 object->generation++;
578
579 /*
580 * show that the object has one more resident page.
581 */
582 object->resident_page_count++;
583 /*
584 * Hold the vnode until the last page is released.
585 */
586 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
587 vhold((struct vnode *)object->handle);
588
589 /*
590 * Since we are inserting a new and possibly dirty page,
591 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
592 */
593 if (m->flags & PG_WRITEABLE)
594 vm_object_set_writeable_dirty(object);
595}
596
597/*
598 * vm_page_remove:
599 * NOTE: used by device pager as well -wfj
600 *
601 * Removes the given mem entry from the object/offset-page
602 * table and the object page list, but do not invalidate/terminate
603 * the backing store.
604 *
605 * The object and page must be locked.
606 * The underlying pmap entry (if any) is NOT removed here.
607 * This routine may not block.
608 */
609void
610vm_page_remove(vm_page_t m)
611{
612 vm_object_t object;
613 vm_page_t root;
614
615 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
616 if ((object = m->object) == NULL)
617 return;
618 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
619 if (m->flags & PG_BUSY) {
620 vm_page_flag_clear(m, PG_BUSY);
621 vm_page_flash(m);
622 }
623
624 /*
625 * Now remove from the object's list of backed pages.
626 */
627 if (m != object->root)
628 vm_page_splay(m->pindex, object->root);
629 if (m->left == NULL)
630 root = m->right;
631 else {
632 root = vm_page_splay(m->pindex, m->left);
633 root->right = m->right;
634 }
635 object->root = root;
636 TAILQ_REMOVE(&object->memq, m, listq);
637
638 /*
639 * And show that the object has one fewer resident page.
640 */
641 object->resident_page_count--;
642 object->generation++;
643 /*
644 * The vnode may now be recycled.
645 */
646 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
647 vdrop((struct vnode *)object->handle);
648
649 m->object = NULL;
650}
651
652/*
653 * vm_page_lookup:
654 *
655 * Returns the page associated with the object/offset
656 * pair specified; if none is found, NULL is returned.
657 *
658 * The object must be locked.
659 * This routine may not block.
660 * This is a critical path routine
661 */
662vm_page_t
663vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
664{
665 vm_page_t m;
666
667 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
668 if ((m = object->root) != NULL && m->pindex != pindex) {
669 m = vm_page_splay(pindex, m);
670 if ((object->root = m)->pindex != pindex)
671 m = NULL;
672 }
673 return (m);
674}
675
676/*
677 * vm_page_rename:
678 *
679 * Move the given memory entry from its
680 * current object to the specified target object/offset.
681 *
682 * The object must be locked.
683 * This routine may not block.
684 *
685 * Note: swap associated with the page must be invalidated by the move. We
686 * have to do this for several reasons: (1) we aren't freeing the
687 * page, (2) we are dirtying the page, (3) the VM system is probably
688 * moving the page from object A to B, and will then later move
689 * the backing store from A to B and we can't have a conflict.
690 *
691 * Note: we *always* dirty the page. It is necessary both for the
692 * fact that we moved it, and because we may be invalidating
693 * swap. If the page is on the cache, we have to deactivate it
694 * or vm_page_dirty() will panic. Dirty pages are not allowed
695 * on the cache.
696 */
697void
698vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
699{
700
701 vm_page_remove(m);
702 vm_page_insert(m, new_object, new_pindex);
703 if (m->queue - m->pc == PQ_CACHE)
704 vm_page_deactivate(m);
705 vm_page_dirty(m);
706}
707
708/*
709 * vm_page_select_cache:
710 *
711 * Move a page of the given color from the cache queue to the free
712 * queue. As pages might be found, but are not applicable, they are
713 * deactivated.
714 *
715 * This routine may not block.
716 */
717vm_page_t
718vm_page_select_cache(int color)
719{
720 vm_object_t object;
721 vm_page_t m;
722 boolean_t was_trylocked;
723
724 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
725 while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
726 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
727 KASSERT(!pmap_page_is_mapped(m),
728 ("Found mapped cache page %p", m));
729 KASSERT((m->flags & PG_UNMANAGED) == 0,
730 ("Found unmanaged cache page %p", m));
731 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
732 if (m->hold_count == 0 && (object = m->object,
733 (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
734 VM_OBJECT_LOCKED(object))) {
735 KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
736 ("Found busy cache page %p", m));
737 vm_page_free(m);
738 if (was_trylocked)
739 VM_OBJECT_UNLOCK(object);
740 break;
741 }
742 vm_page_deactivate(m);
743 }
744 return (m);
745}
746
747/*
748 * vm_page_alloc:
749 *
750 * Allocate and return a memory cell associated
751 * with this VM object/offset pair.
752 *
753 * page_req classes:
754 * VM_ALLOC_NORMAL normal process request
755 * VM_ALLOC_SYSTEM system *really* needs a page
756 * VM_ALLOC_INTERRUPT interrupt time request
757 * VM_ALLOC_ZERO zero page
758 *
759 * This routine may not block.
760 *
761 * Additional special handling is required when called from an
762 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with
763 * the page cache in this case.
764 */
765vm_page_t
766vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
767{
768 vm_page_t m = NULL;
769 int color, flags, page_req;
770
771 page_req = req & VM_ALLOC_CLASS_MASK;
772 KASSERT(curthread->td_intr_nesting_level == 0 ||
773 page_req == VM_ALLOC_INTERRUPT,
774 ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
775
776 if ((req & VM_ALLOC_NOOBJ) == 0) {
777 KASSERT(object != NULL,
778 ("vm_page_alloc: NULL object."));
779 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
780 color = (pindex + object->pg_color) & PQ_L2_MASK;
781 } else
782 color = pindex & PQ_L2_MASK;
783
784 /*
785 * The pager is allowed to eat deeper into the free page list.
786 */
787 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
788 page_req = VM_ALLOC_SYSTEM;
789 };
790
791loop:
792 mtx_lock_spin(&vm_page_queue_free_mtx);
793 if (cnt.v_free_count > cnt.v_free_reserved ||
794 (page_req == VM_ALLOC_SYSTEM &&
795 cnt.v_cache_count == 0 &&
796 cnt.v_free_count > cnt.v_interrupt_free_min) ||
797 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
798 /*
799 * Allocate from the free queue if the number of free pages
800 * exceeds the minimum for the request class.
801 */
802 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
803 } else if (page_req != VM_ALLOC_INTERRUPT) {
804 mtx_unlock_spin(&vm_page_queue_free_mtx);
805 /*
806 * Allocatable from cache (non-interrupt only). On success,
807 * we must free the page and try again, thus ensuring that
808 * cnt.v_*_free_min counters are replenished.
809 */
810 vm_page_lock_queues();
811 if ((m = vm_page_select_cache(color)) == NULL) {
812#if defined(DIAGNOSTIC)
813 if (cnt.v_cache_count > 0)
814 printf("vm_page_alloc(NORMAL): missing pages on cache queue: %d\n", cnt.v_cache_count);
815#endif
816 vm_page_unlock_queues();
817 atomic_add_int(&vm_pageout_deficit, 1);
818 pagedaemon_wakeup();
819 return (NULL);
820 }
821 vm_page_unlock_queues();
822 goto loop;
823 } else {
824 /*
825 * Not allocatable from cache from interrupt, give up.
826 */
827 mtx_unlock_spin(&vm_page_queue_free_mtx);
828 atomic_add_int(&vm_pageout_deficit, 1);
829 pagedaemon_wakeup();
830 return (NULL);
831 }
832
833 /*
834 * At this point we had better have found a good page.
835 */
836
837 KASSERT(
838 m != NULL,
839 ("vm_page_alloc(): missing page on free queue")
840 );
841
842 /*
843 * Remove from free queue
844 */
845 vm_pageq_remove_nowakeup(m);
846
847 /*
848 * Initialize structure. Only the PG_ZERO flag is inherited.
849 */
850 flags = PG_BUSY;
851 if (m->flags & PG_ZERO) {
852 vm_page_zero_count--;
853 if (req & VM_ALLOC_ZERO)
854 flags = PG_ZERO | PG_BUSY;
855 }
856 if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
857 flags &= ~PG_BUSY;
858 m->flags = flags;
859 if (req & VM_ALLOC_WIRED) {
860 atomic_add_int(&cnt.v_wire_count, 1);
861 m->wire_count = 1;
862 } else
863 m->wire_count = 0;
864 m->hold_count = 0;
865 m->act_count = 0;
866 m->busy = 0;
867 m->valid = 0;
868 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
869 mtx_unlock_spin(&vm_page_queue_free_mtx);
870
871 if ((req & VM_ALLOC_NOOBJ) == 0)
872 vm_page_insert(m, object, pindex);
873 else
874 m->pindex = pindex;
875
876 /*
877 * Don't wakeup too often - wakeup the pageout daemon when
878 * we would be nearly out of memory.
879 */
880 if (vm_paging_needed())
881 pagedaemon_wakeup();
882
883 return (m);
884}
885
886/*
887 * vm_wait: (also see VM_WAIT macro)
888 *
889 * Block until free pages are available for allocation
890 * - Called in various places before memory allocations.
891 */
892void
893vm_wait(void)
894{
895
896 vm_page_lock_queues();
897 if (curproc == pageproc) {
898 vm_pageout_pages_needed = 1;
899 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
900 PDROP | PSWP, "VMWait", 0);
901 } else {
902 if (!vm_pages_needed) {
903 vm_pages_needed = 1;
904 wakeup(&vm_pages_needed);
905 }
906 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
907 "vmwait", 0);
908 }
909}
910
911/*
912 * vm_waitpfault: (also see VM_WAITPFAULT macro)
913 *
914 * Block until free pages are available for allocation
915 * - Called only in vm_fault so that processes page faulting
916 * can be easily tracked.
917 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
918 * processes will be able to grab memory first. Do not change
919 * this balance without careful testing first.
920 */
921void
922vm_waitpfault(void)
923{
924
925 vm_page_lock_queues();
926 if (!vm_pages_needed) {
927 vm_pages_needed = 1;
928 wakeup(&vm_pages_needed);
929 }
930 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
931 "pfault", 0);
932}
933
934/*
935 * vm_page_activate:
936 *
937 * Put the specified page on the active list (if appropriate).
938 * Ensure that act_count is at least ACT_INIT but do not otherwise
939 * mess with it.
940 *
941 * The page queues must be locked.
942 * This routine may not block.
943 */
944void
945vm_page_activate(vm_page_t m)
946{
947
948 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
949 if (m->queue != PQ_ACTIVE) {
950 if ((m->queue - m->pc) == PQ_CACHE)
951 cnt.v_reactivated++;
952 vm_pageq_remove(m);
953 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
954 if (m->act_count < ACT_INIT)
955 m->act_count = ACT_INIT;
956 vm_pageq_enqueue(PQ_ACTIVE, m);
957 }
958 } else {
959 if (m->act_count < ACT_INIT)
960 m->act_count = ACT_INIT;
961 }
962}
963
964/*
965 * vm_page_free_wakeup:
966 *
967 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
968 * routine is called when a page has been added to the cache or free
969 * queues.
970 *
971 * The page queues must be locked.
972 * This routine may not block.
973 */
974static __inline void
975vm_page_free_wakeup(void)
976{
977
978 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
979 /*
980 * if pageout daemon needs pages, then tell it that there are
981 * some free.
982 */
983 if (vm_pageout_pages_needed &&
984 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
985 wakeup(&vm_pageout_pages_needed);
986 vm_pageout_pages_needed = 0;
987 }
988 /*
989 * wakeup processes that are waiting on memory if we hit a
990 * high water mark. And wakeup scheduler process if we have
991 * lots of memory. this process will swapin processes.
992 */
993 if (vm_pages_needed && !vm_page_count_min()) {
994 vm_pages_needed = 0;
995 wakeup(&cnt.v_free_count);
996 }
997}
998
999/*
1000 * vm_page_free_toq:
1001 *
1002 * Returns the given page to the PQ_FREE list,
1003 * disassociating it with any VM object.
1004 *
1005 * Object and page must be locked prior to entry.
1006 * This routine may not block.
1007 */
1008
1009void
1010vm_page_free_toq(vm_page_t m)
1011{
1012 struct vpgqueues *pq;
1013
1014 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1015 cnt.v_tfree++;
1016
1017 if (m->busy || ((m->queue - m->pc) == PQ_FREE)) {
1018 printf(
1019 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
1020 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
1021 m->hold_count);
1022 if ((m->queue - m->pc) == PQ_FREE)
1023 panic("vm_page_free: freeing free page");
1024 else
1025 panic("vm_page_free: freeing busy page");
1026 }
1027
1028 /*
1029 * unqueue, then remove page. Note that we cannot destroy
1030 * the page here because we do not want to call the pager's
1031 * callback routine until after we've put the page on the
1032 * appropriate free queue.
1033 */
1034 vm_pageq_remove_nowakeup(m);
1035 vm_page_remove(m);
1036
1037 /*
1038 * If fictitious remove object association and
1039 * return, otherwise delay object association removal.
1040 */
1041 if ((m->flags & PG_FICTITIOUS) != 0) {
1042 return;
1043 }
1044
1045 m->valid = 0;
1046 vm_page_undirty(m);
1047
1048 if (m->wire_count != 0) {
1049 if (m->wire_count > 1) {
1050 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1051 m->wire_count, (long)m->pindex);
1052 }
1053 panic("vm_page_free: freeing wired page");
1054 }
1055
1056 /*
1057 * Clear the UNMANAGED flag when freeing an unmanaged page.
1058 */
1059 if (m->flags & PG_UNMANAGED) {
1060 m->flags &= ~PG_UNMANAGED;
1061 }
1062
1063 if (m->hold_count != 0) {
1064 m->flags &= ~PG_ZERO;
1065 m->queue = PQ_HOLD;
1066 } else
1067 m->queue = PQ_FREE + m->pc;
1068 pq = &vm_page_queues[m->queue];
1069 mtx_lock_spin(&vm_page_queue_free_mtx);
1070 pq->lcnt++;
1071 ++(*pq->cnt);
1072
1073 /*
1074 * Put zero'd pages on the end ( where we look for zero'd pages
1075 * first ) and non-zerod pages at the head.
1076 */
1077 if (m->flags & PG_ZERO) {
1078 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
1079 ++vm_page_zero_count;
1080 } else {
1081 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
1082 }
1083 mtx_unlock_spin(&vm_page_queue_free_mtx);
1084 vm_page_free_wakeup();
1085}
1086
1087/*
1088 * vm_page_unmanage:
1089 *
1090 * Prevent PV management from being done on the page. The page is
1091 * removed from the paging queues as if it were wired, and as a
1092 * consequence of no longer being managed the pageout daemon will not
1093 * touch it (since there is no way to locate the pte mappings for the
1094 * page). madvise() calls that mess with the pmap will also no longer
1095 * operate on the page.
1096 *
1097 * Beyond that the page is still reasonably 'normal'. Freeing the page
1098 * will clear the flag.
1099 *
1100 * This routine is used by OBJT_PHYS objects - objects using unswappable
1101 * physical memory as backing store rather then swap-backed memory and
1102 * will eventually be extended to support 4MB unmanaged physical
1103 * mappings.
1104 */
1105void
1106vm_page_unmanage(vm_page_t m)
1107{
1108
1109 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1110 if ((m->flags & PG_UNMANAGED) == 0) {
1111 if (m->wire_count == 0)
1112 vm_pageq_remove(m);
1113 }
1114 vm_page_flag_set(m, PG_UNMANAGED);
1115}
1116
1117/*
1118 * vm_page_wire:
1119 *
1120 * Mark this page as wired down by yet
1121 * another map, removing it from paging queues
1122 * as necessary.
1123 *
1124 * The page queues must be locked.
1125 * This routine may not block.
1126 */
1127void
1128vm_page_wire(vm_page_t m)
1129{
1130
1131 /*
1132 * Only bump the wire statistics if the page is not already wired,
1133 * and only unqueue the page if it is on some queue (if it is unmanaged
1134 * it is already off the queues).
1135 */
1136 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1137 if (m->flags & PG_FICTITIOUS)
1138 return;
1139 if (m->wire_count == 0) {
1140 if ((m->flags & PG_UNMANAGED) == 0)
1141 vm_pageq_remove(m);
1142 atomic_add_int(&cnt.v_wire_count, 1);
1143 }
1144 m->wire_count++;
1145 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1146}
1147
1148/*
1149 * vm_page_unwire:
1150 *
1151 * Release one wiring of this page, potentially
1152 * enabling it to be paged again.
1153 *
1154 * Many pages placed on the inactive queue should actually go
1155 * into the cache, but it is difficult to figure out which. What
1156 * we do instead, if the inactive target is well met, is to put
1157 * clean pages at the head of the inactive queue instead of the tail.
1158 * This will cause them to be moved to the cache more quickly and
1159 * if not actively re-referenced, freed more quickly. If we just
1160 * stick these pages at the end of the inactive queue, heavy filesystem
1161 * meta-data accesses can cause an unnecessary paging load on memory bound
1162 * processes. This optimization causes one-time-use metadata to be
1163 * reused more quickly.
1164 *
1165 * BUT, if we are in a low-memory situation we have no choice but to
1166 * put clean pages on the cache queue.
1167 *
1168 * A number of routines use vm_page_unwire() to guarantee that the page
1169 * will go into either the inactive or active queues, and will NEVER
1170 * be placed in the cache - for example, just after dirtying a page.
1171 * dirty pages in the cache are not allowed.
1172 *
1173 * The page queues must be locked.
1174 * This routine may not block.
1175 */
1176void
1177vm_page_unwire(vm_page_t m, int activate)
1178{
1179
1180 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1181 if (m->flags & PG_FICTITIOUS)
1182 return;
1183 if (m->wire_count > 0) {
1184 m->wire_count--;
1185 if (m->wire_count == 0) {
1186 atomic_subtract_int(&cnt.v_wire_count, 1);
1187 if (m->flags & PG_UNMANAGED) {
1188 ;
1189 } else if (activate)
1190 vm_pageq_enqueue(PQ_ACTIVE, m);
1191 else {
1192 vm_page_flag_clear(m, PG_WINATCFLS);
1193 vm_pageq_enqueue(PQ_INACTIVE, m);
1194 }
1195 }
1196 } else {
1197 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1198 }
1199}
1200
1201
1202/*
1203 * Move the specified page to the inactive queue. If the page has
1204 * any associated swap, the swap is deallocated.
1205 *
1206 * Normally athead is 0 resulting in LRU operation. athead is set
1207 * to 1 if we want this page to be 'as if it were placed in the cache',
1208 * except without unmapping it from the process address space.
1209 *
1210 * This routine may not block.
1211 */
1212static __inline void
1213_vm_page_deactivate(vm_page_t m, int athead)
1214{
1215
1216 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1217
1218 /*
1219 * Ignore if already inactive.
1220 */
1221 if (m->queue == PQ_INACTIVE)
1222 return;
1223 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1224 if ((m->queue - m->pc) == PQ_CACHE)
1225 cnt.v_reactivated++;
1226 vm_page_flag_clear(m, PG_WINATCFLS);
1227 vm_pageq_remove(m);
1228 if (athead)
1229 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1230 else
1231 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1232 m->queue = PQ_INACTIVE;
1233 vm_page_queues[PQ_INACTIVE].lcnt++;
1234 cnt.v_inactive_count++;
1235 }
1236}
1237
1238void
1239vm_page_deactivate(vm_page_t m)
1240{
1241 _vm_page_deactivate(m, 0);
1242}
1243
1244/*
1245 * vm_page_try_to_cache:
1246 *
1247 * Returns 0 on failure, 1 on success
1248 */
1249int
1250vm_page_try_to_cache(vm_page_t m)
1251{
1252
1253 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1254 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1255 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1256 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1257 return (0);
1258 }
1259 pmap_remove_all(m);
1260 if (m->dirty)
1261 return (0);
1262 vm_page_cache(m);
1263 return (1);
1264}
1265
1266/*
1267 * vm_page_try_to_free()
1268 *
1269 * Attempt to free the page. If we cannot free it, we do nothing.
1270 * 1 is returned on success, 0 on failure.
1271 */
1272int
1273vm_page_try_to_free(vm_page_t m)
1274{
1275
1276 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1277 if (m->object != NULL)
1278 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1279 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1280 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1281 return (0);
1282 }
1283 pmap_remove_all(m);
1284 if (m->dirty)
1285 return (0);
1286 vm_page_free(m);
1287 return (1);
1288}
1289
1290/*
1291 * vm_page_cache
1292 *
1293 * Put the specified page onto the page cache queue (if appropriate).
1294 *
1295 * This routine may not block.
1296 */
1297void
1298vm_page_cache(vm_page_t m)
1299{
1300
1301 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1302 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1303 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
1304 m->hold_count || m->wire_count) {
1305 printf("vm_page_cache: attempting to cache busy page\n");
1306 return;
1307 }
1308 if ((m->queue - m->pc) == PQ_CACHE)
1309 return;
1310
1311 /*
1312 * Remove all pmaps and indicate that the page is not
1313 * writeable or mapped.
1314 */
1315 pmap_remove_all(m);
1316 if (m->dirty != 0) {
1317 panic("vm_page_cache: caching a dirty page, pindex: %ld",
1318 (long)m->pindex);
1319 }
1320 vm_pageq_remove_nowakeup(m);
1321 vm_pageq_enqueue(PQ_CACHE + m->pc, m);
1322 vm_page_free_wakeup();
1323}
1324
1325/*
1326 * vm_page_dontneed
1327 *
1328 * Cache, deactivate, or do nothing as appropriate. This routine
1329 * is typically used by madvise() MADV_DONTNEED.
1330 *
1331 * Generally speaking we want to move the page into the cache so
1332 * it gets reused quickly. However, this can result in a silly syndrome
1333 * due to the page recycling too quickly. Small objects will not be
1334 * fully cached. On the otherhand, if we move the page to the inactive
1335 * queue we wind up with a problem whereby very large objects
1336 * unnecessarily blow away our inactive and cache queues.
1337 *
1338 * The solution is to move the pages based on a fixed weighting. We
1339 * either leave them alone, deactivate them, or move them to the cache,
1340 * where moving them to the cache has the highest weighting.
1341 * By forcing some pages into other queues we eventually force the
1342 * system to balance the queues, potentially recovering other unrelated
1343 * space from active. The idea is to not force this to happen too
1344 * often.
1345 */
1346void
1347vm_page_dontneed(vm_page_t m)
1348{
1349 static int dnweight;
1350 int dnw;
1351 int head;
1352
1353 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1354 dnw = ++dnweight;
1355
1356 /*
1357 * occassionally leave the page alone
1358 */
1359 if ((dnw & 0x01F0) == 0 ||
1360 m->queue == PQ_INACTIVE ||
1361 m->queue - m->pc == PQ_CACHE
1362 ) {
1363 if (m->act_count >= ACT_INIT)
1364 --m->act_count;
1365 return;
1366 }
1367
1368 if (m->dirty == 0 && pmap_is_modified(m))
1369 vm_page_dirty(m);
1370
1371 if (m->dirty || (dnw & 0x0070) == 0) {
1372 /*
1373 * Deactivate the page 3 times out of 32.
1374 */
1375 head = 0;
1376 } else {
1377 /*
1378 * Cache the page 28 times out of every 32. Note that
1379 * the page is deactivated instead of cached, but placed
1380 * at the head of the queue instead of the tail.
1381 */
1382 head = 1;
1383 }
1384 _vm_page_deactivate(m, head);
1385}
1386
1387/*
1388 * Grab a page, waiting until we are waken up due to the page
1389 * changing state. We keep on waiting, if the page continues
1390 * to be in the object. If the page doesn't exist, first allocate it
1391 * and then conditionally zero it.
1392 *
1393 * This routine may block.
1394 */
1395vm_page_t
1396vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1397{
1398 vm_page_t m;
1399
1400 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1401retrylookup:
1402 if ((m = vm_page_lookup(object, pindex)) != NULL) {
1403 vm_page_lock_queues();
1404 if (m->busy || (m->flags & PG_BUSY)) {
1405 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
1406 VM_OBJECT_UNLOCK(object);
1407 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
1408 VM_OBJECT_LOCK(object);
1409 if ((allocflags & VM_ALLOC_RETRY) == 0)
1410 return (NULL);
1411 goto retrylookup;
1412 } else {
1413 if (allocflags & VM_ALLOC_WIRED)
1414 vm_page_wire(m);
1415 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1416 vm_page_busy(m);
1417 vm_page_unlock_queues();
1418 return (m);
1419 }
1420 }
1421 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1422 if (m == NULL) {
1423 VM_OBJECT_UNLOCK(object);
1424 VM_WAIT;
1425 VM_OBJECT_LOCK(object);
1426 if ((allocflags & VM_ALLOC_RETRY) == 0)
1427 return (NULL);
1428 goto retrylookup;
1429 }
1430 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1431 pmap_zero_page(m);
1432 return (m);
1433}
1434
1435/*
1436 * Mapping function for valid bits or for dirty bits in
1437 * a page. May not block.
1438 *
1439 * Inputs are required to range within a page.
1440 */
1441__inline int
1442vm_page_bits(int base, int size)
1443{
1444 int first_bit;
1445 int last_bit;
1446
1447 KASSERT(
1448 base + size <= PAGE_SIZE,
1449 ("vm_page_bits: illegal base/size %d/%d", base, size)
1450 );
1451
1452 if (size == 0) /* handle degenerate case */
1453 return (0);
1454
1455 first_bit = base >> DEV_BSHIFT;
1456 last_bit = (base + size - 1) >> DEV_BSHIFT;
1457
1458 return ((2 << last_bit) - (1 << first_bit));
1459}
1460
1461/*
1462 * vm_page_set_validclean:
1463 *
1464 * Sets portions of a page valid and clean. The arguments are expected
1465 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1466 * of any partial chunks touched by the range. The invalid portion of
1467 * such chunks will be zero'd.
1468 *
1469 * This routine may not block.
1470 *
1471 * (base + size) must be less then or equal to PAGE_SIZE.
1472 */
1473void
1474vm_page_set_validclean(vm_page_t m, int base, int size)
1475{
1476 int pagebits;
1477 int frag;
1478 int endoff;
1479
1480 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1481 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1482 if (size == 0) /* handle degenerate case */
1483 return;
1484
1485 /*
1486 * If the base is not DEV_BSIZE aligned and the valid
1487 * bit is clear, we have to zero out a portion of the
1488 * first block.
1489 */
1490 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1491 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1492 pmap_zero_page_area(m, frag, base - frag);
1493
1494 /*
1495 * If the ending offset is not DEV_BSIZE aligned and the
1496 * valid bit is clear, we have to zero out a portion of
1497 * the last block.
1498 */
1499 endoff = base + size;
1500 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1501 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1502 pmap_zero_page_area(m, endoff,
1503 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1504
1505 /*
1506 * Set valid, clear dirty bits. If validating the entire
1507 * page we can safely clear the pmap modify bit. We also
1508 * use this opportunity to clear the PG_NOSYNC flag. If a process
1509 * takes a write fault on a MAP_NOSYNC memory area the flag will
1510 * be set again.
1511 *
1512 * We set valid bits inclusive of any overlap, but we can only
1513 * clear dirty bits for DEV_BSIZE chunks that are fully within
1514 * the range.
1515 */
1516 pagebits = vm_page_bits(base, size);
1517 m->valid |= pagebits;
1518#if 0 /* NOT YET */
1519 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1520 frag = DEV_BSIZE - frag;
1521 base += frag;
1522 size -= frag;
1523 if (size < 0)
1524 size = 0;
1525 }
1526 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1527#endif
1528 m->dirty &= ~pagebits;
1529 if (base == 0 && size == PAGE_SIZE) {
1530 pmap_clear_modify(m);
1531 vm_page_flag_clear(m, PG_NOSYNC);
1532 }
1533}
1534
1535void
1536vm_page_clear_dirty(vm_page_t m, int base, int size)
1537{
1538
1539 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1540 m->dirty &= ~vm_page_bits(base, size);
1541}
1542
1543/*
1544 * vm_page_set_invalid:
1545 *
1546 * Invalidates DEV_BSIZE'd chunks within a page. Both the
1547 * valid and dirty bits for the effected areas are cleared.
1548 *
1549 * May not block.
1550 */
1551void
1552vm_page_set_invalid(vm_page_t m, int base, int size)
1553{
1554 int bits;
1555
1556 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1557 bits = vm_page_bits(base, size);
1558 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1559 m->valid &= ~bits;
1560 m->dirty &= ~bits;
1561 m->object->generation++;
1562}
1563
1564/*
1565 * vm_page_zero_invalid()
1566 *
1567 * The kernel assumes that the invalid portions of a page contain
1568 * garbage, but such pages can be mapped into memory by user code.
1569 * When this occurs, we must zero out the non-valid portions of the
1570 * page so user code sees what it expects.
1571 *
1572 * Pages are most often semi-valid when the end of a file is mapped
1573 * into memory and the file's size is not page aligned.
1574 */
1575void
1576vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1577{
1578 int b;
1579 int i;
1580
1581 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1582 /*
1583 * Scan the valid bits looking for invalid sections that
1584 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
1585 * valid bit may be set ) have already been zerod by
1586 * vm_page_set_validclean().
1587 */
1588 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1589 if (i == (PAGE_SIZE / DEV_BSIZE) ||
1590 (m->valid & (1 << i))
1591 ) {
1592 if (i > b) {
1593 pmap_zero_page_area(m,
1594 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1595 }
1596 b = i + 1;
1597 }
1598 }
1599
1600 /*
1601 * setvalid is TRUE when we can safely set the zero'd areas
1602 * as being valid. We can do this if there are no cache consistancy
1603 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
1604 */
1605 if (setvalid)
1606 m->valid = VM_PAGE_BITS_ALL;
1607}
1608
1609/*
1610 * vm_page_is_valid:
1611 *
1612 * Is (partial) page valid? Note that the case where size == 0
1613 * will return FALSE in the degenerate case where the page is
1614 * entirely invalid, and TRUE otherwise.
1615 *
1616 * May not block.
1617 */
1618int
1619vm_page_is_valid(vm_page_t m, int base, int size)
1620{
1621 int bits = vm_page_bits(base, size);
1622
1623 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1624 if (m->valid && ((m->valid & bits) == bits))
1625 return 1;
1626 else
1627 return 0;
1628}
1629
1630/*
1631 * update dirty bits from pmap/mmu. May not block.
1632 */
1633void
1634vm_page_test_dirty(vm_page_t m)
1635{
1636 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1637 vm_page_dirty(m);
1638 }
1639}
1640
1641int so_zerocp_fullpage = 0;
1642
1643void
1644vm_page_cowfault(vm_page_t m)
1645{
1646 vm_page_t mnew;
1647 vm_object_t object;
1648 vm_pindex_t pindex;
1649
1650 object = m->object;
1651 pindex = m->pindex;
1652
1653 retry_alloc:
1654 vm_page_remove(m);
1655 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
1656 if (mnew == NULL) {
1657 vm_page_insert(m, object, pindex);
1658 vm_page_unlock_queues();
1659 VM_OBJECT_UNLOCK(object);
1660 VM_WAIT;
1661 VM_OBJECT_LOCK(object);
1662 vm_page_lock_queues();
1663 goto retry_alloc;
1664 }
1665
1666 if (m->cow == 0) {
1667 /*
1668 * check to see if we raced with an xmit complete when
1669 * waiting to allocate a page. If so, put things back
1670 * the way they were
1671 */
1672 vm_page_free(mnew);
1673 vm_page_insert(m, object, pindex);
1674 } else { /* clear COW & copy page */
1675 if (!so_zerocp_fullpage)
1676 pmap_copy_page(m, mnew);
1677 mnew->valid = VM_PAGE_BITS_ALL;
1678 vm_page_dirty(mnew);
1679 vm_page_flag_clear(mnew, PG_BUSY);
1680 }
1681}
1682
1683void
1684vm_page_cowclear(vm_page_t m)
1685{
1686
1687 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1688 if (m->cow) {
1689 m->cow--;
1690 /*
1691 * let vm_fault add back write permission lazily
1692 */
1693 }
1694 /*
1695 * sf_buf_free() will free the page, so we needn't do it here
1696 */
1697}
1698
1699void
1700vm_page_cowsetup(vm_page_t m)
1701{
1702
1703 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1704 m->cow++;
1705 pmap_page_protect(m, VM_PROT_READ);
1706}
1707
1708#include "opt_ddb.h"
1709#ifdef DDB
1710#include <sys/kernel.h>
1711
1712#include <ddb/ddb.h>
1713
1714DB_SHOW_COMMAND(page, vm_page_print_page_info)
1715{
1716 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
1717 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
1718 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
1719 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
1720 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
1721 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
1722 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
1723 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
1724 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
1725 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
1726}
1727
1728DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
1729{
1730 int i;
1731 db_printf("PQ_FREE:");
1732 for (i = 0; i < PQ_L2_SIZE; i++) {
1733 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
1734 }
1735 db_printf("\n");
1736
1737 db_printf("PQ_CACHE:");
1738 for (i = 0; i < PQ_L2_SIZE; i++) {
1739 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
1740 }
1741 db_printf("\n");
1742
1743 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
1744 vm_page_queues[PQ_ACTIVE].lcnt,
1745 vm_page_queues[PQ_INACTIVE].lcnt);
1746}
1747#endif /* DDB */
|