236 * Compute the number of pages of memory that will be available for
237 * use (taking into account the overhead of a page structure per
238 * page).
239 */
240 first_page = phys_avail[0] / PAGE_SIZE;
241 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
242 npages = (total - (page_range * sizeof(struct vm_page)) -
243 (end - new_end)) / PAGE_SIZE;
244 end = new_end;
245
246 /*
247 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
248 */
249 vaddr += PAGE_SIZE;
250
251 /*
252 * Initialize the mem entry structures now, and put them in the free
253 * queue.
254 */
255 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
256 mapped = pmap_map(&vaddr, new_end, end,
257 VM_PROT_READ | VM_PROT_WRITE);
258 vm_page_array = (vm_page_t) mapped;
259 phys_avail[biggestone + 1] = new_end;
260
261 /*
262 * Clear all of the page structures
263 */
264 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
265 vm_page_array_size = page_range;
266
267 /*
268 * Construct the free queue(s) in descending order (by physical
269 * address) so that the first 16MB of physical memory is allocated
270 * last rather than first. On large-memory machines, this avoids
271 * the exhaustion of low physical memory before isa_dma_init has run.
272 */
273 cnt.v_page_count = 0;
274 cnt.v_free_count = 0;
275 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
276 pa = phys_avail[i];
277 last_pa = phys_avail[i + 1];
278 while (pa < last_pa && npages-- > 0) {
279 vm_pageq_add_new_page(pa);
280 pa += PAGE_SIZE;
281 }
282 }
283 return (vaddr);
284}
285
286void
287vm_page_flag_set(vm_page_t m, unsigned short bits)
288{
289
290 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
291 m->flags |= bits;
292}
293
294void
295vm_page_flag_clear(vm_page_t m, unsigned short bits)
296{
297
298 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
299 m->flags &= ~bits;
300}
301
302void
303vm_page_busy(vm_page_t m)
304{
305
306 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
307 KASSERT((m->flags & PG_BUSY) == 0,
308 ("vm_page_busy: page already busy!!!"));
309 vm_page_flag_set(m, PG_BUSY);
310}
311
312/*
313 * vm_page_flash:
314 *
315 * wakeup anyone waiting for the page.
316 */
317void
318vm_page_flash(vm_page_t m)
319{
320
321 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
322 if (m->flags & PG_WANTED) {
323 vm_page_flag_clear(m, PG_WANTED);
324 wakeup(m);
325 }
326}
327
328/*
329 * vm_page_wakeup:
330 *
331 * clear the PG_BUSY flag and wakeup anyone waiting for the
332 * page.
333 *
334 */
335void
336vm_page_wakeup(vm_page_t m)
337{
338
339 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
340 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
341 vm_page_flag_clear(m, PG_BUSY);
342 vm_page_flash(m);
343}
344
345void
346vm_page_io_start(vm_page_t m)
347{
348
349 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
350 m->busy++;
351}
352
353void
354vm_page_io_finish(vm_page_t m)
355{
356
357 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
358 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
359 m->busy--;
360 if (m->busy == 0)
361 vm_page_flash(m);
362}
363
364/*
365 * Keep page from being freed by the page daemon
366 * much of the same effect as wiring, except much lower
367 * overhead and should be used only for *very* temporary
368 * holding ("wiring").
369 */
370void
371vm_page_hold(vm_page_t mem)
372{
373
374 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
375 mem->hold_count++;
376}
377
378void
379vm_page_unhold(vm_page_t mem)
380{
381
382 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
383 --mem->hold_count;
384 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
385 if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD))
386 vm_page_free_toq(mem);
387}
388
389/*
390 * vm_page_free:
391 *
392 * Free a page
393 *
394 * The clearing of PG_ZERO is a temporary safety until the code can be
395 * reviewed to determine that PG_ZERO is being properly cleared on
396 * write faults or maps. PG_ZERO was previously cleared in
397 * vm_page_alloc().
398 */
399void
400vm_page_free(vm_page_t m)
401{
402 vm_page_flag_clear(m, PG_ZERO);
403 vm_page_free_toq(m);
404 vm_page_zero_idle_wakeup();
405}
406
407/*
408 * vm_page_free_zero:
409 *
410 * Free a page to the zerod-pages queue
411 */
412void
413vm_page_free_zero(vm_page_t m)
414{
415 vm_page_flag_set(m, PG_ZERO);
416 vm_page_free_toq(m);
417}
418
419/*
420 * vm_page_sleep_if_busy:
421 *
422 * Sleep and release the page queues lock if PG_BUSY is set or,
423 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
424 * thread slept and the page queues lock was released.
425 * Otherwise, retains the page queues lock and returns FALSE.
426 */
427int
428vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
429{
430 vm_object_t object;
431
432 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
433 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
434 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
435 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
436 /*
437 * It's possible that while we sleep, the page will get
438 * unbusied and freed. If we are holding the object
439 * lock, we will assume we hold a reference to the object
440 * such that even if m->object changes, we can re-lock
441 * it.
442 */
443 object = m->object;
444 VM_OBJECT_UNLOCK(object);
445 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
446 VM_OBJECT_LOCK(object);
447 return (TRUE);
448 }
449 return (FALSE);
450}
451
452/*
453 * vm_page_dirty:
454 *
455 * make page all dirty
456 */
457void
458vm_page_dirty(vm_page_t m)
459{
460 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_CACHE,
461 ("vm_page_dirty: page in cache!"));
462 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_FREE,
463 ("vm_page_dirty: page is free!"));
464 m->dirty = VM_PAGE_BITS_ALL;
465}
466
467/*
468 * vm_page_splay:
469 *
470 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
471 * the vm_page containing the given pindex. If, however, that
472 * pindex is not found in the vm_object, returns a vm_page that is
473 * adjacent to the pindex, coming before or after it.
474 */
475vm_page_t
476vm_page_splay(vm_pindex_t pindex, vm_page_t root)
477{
478 struct vm_page dummy;
479 vm_page_t lefttreemax, righttreemin, y;
480
481 if (root == NULL)
482 return (root);
483 lefttreemax = righttreemin = &dummy;
484 for (;; root = y) {
485 if (pindex < root->pindex) {
486 if ((y = root->left) == NULL)
487 break;
488 if (pindex < y->pindex) {
489 /* Rotate right. */
490 root->left = y->right;
491 y->right = root;
492 root = y;
493 if ((y = root->left) == NULL)
494 break;
495 }
496 /* Link into the new root's right tree. */
497 righttreemin->left = root;
498 righttreemin = root;
499 } else if (pindex > root->pindex) {
500 if ((y = root->right) == NULL)
501 break;
502 if (pindex > y->pindex) {
503 /* Rotate left. */
504 root->right = y->left;
505 y->left = root;
506 root = y;
507 if ((y = root->right) == NULL)
508 break;
509 }
510 /* Link into the new root's left tree. */
511 lefttreemax->right = root;
512 lefttreemax = root;
513 } else
514 break;
515 }
516 /* Assemble the new root. */
517 lefttreemax->right = root->left;
518 righttreemin->left = root->right;
519 root->left = dummy.right;
520 root->right = dummy.left;
521 return (root);
522}
523
524/*
525 * vm_page_insert: [ internal use only ]
526 *
527 * Inserts the given mem entry into the object and object list.
528 *
529 * The pagetables are not updated but will presumably fault the page
530 * in if necessary, or if a kernel page the caller will at some point
531 * enter the page into the kernel's pmap. We are not allowed to block
532 * here so we *can't* do this anyway.
533 *
534 * The object and page must be locked.
535 * This routine may not block.
536 */
537void
538vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
539{
540 vm_page_t root;
541
542 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
543 if (m->object != NULL)
544 panic("vm_page_insert: page already inserted");
545
546 /*
547 * Record the object/offset pair in this page
548 */
549 m->object = object;
550 m->pindex = pindex;
551
552 /*
553 * Now link into the object's ordered list of backed pages.
554 */
555 root = object->root;
556 if (root == NULL) {
557 m->left = NULL;
558 m->right = NULL;
559 TAILQ_INSERT_TAIL(&object->memq, m, listq);
560 } else {
561 root = vm_page_splay(pindex, root);
562 if (pindex < root->pindex) {
563 m->left = root->left;
564 m->right = root;
565 root->left = NULL;
566 TAILQ_INSERT_BEFORE(root, m, listq);
567 } else if (pindex == root->pindex)
568 panic("vm_page_insert: offset already allocated");
569 else {
570 m->right = root->right;
571 m->left = root;
572 root->right = NULL;
573 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
574 }
575 }
576 object->root = m;
577 object->generation++;
578
579 /*
580 * show that the object has one more resident page.
581 */
582 object->resident_page_count++;
583 /*
584 * Hold the vnode until the last page is released.
585 */
586 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
587 vhold((struct vnode *)object->handle);
588
589 /*
590 * Since we are inserting a new and possibly dirty page,
591 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
592 */
593 if (m->flags & PG_WRITEABLE)
594 vm_object_set_writeable_dirty(object);
595}
596
597/*
598 * vm_page_remove:
599 * NOTE: used by device pager as well -wfj
600 *
601 * Removes the given mem entry from the object/offset-page
602 * table and the object page list, but do not invalidate/terminate
603 * the backing store.
604 *
605 * The object and page must be locked.
606 * The underlying pmap entry (if any) is NOT removed here.
607 * This routine may not block.
608 */
609void
610vm_page_remove(vm_page_t m)
611{
612 vm_object_t object;
613 vm_page_t root;
614
615 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
616 if ((object = m->object) == NULL)
617 return;
618 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
619 if (m->flags & PG_BUSY) {
620 vm_page_flag_clear(m, PG_BUSY);
621 vm_page_flash(m);
622 }
623
624 /*
625 * Now remove from the object's list of backed pages.
626 */
627 if (m != object->root)
628 vm_page_splay(m->pindex, object->root);
629 if (m->left == NULL)
630 root = m->right;
631 else {
632 root = vm_page_splay(m->pindex, m->left);
633 root->right = m->right;
634 }
635 object->root = root;
636 TAILQ_REMOVE(&object->memq, m, listq);
637
638 /*
639 * And show that the object has one fewer resident page.
640 */
641 object->resident_page_count--;
642 object->generation++;
643 /*
644 * The vnode may now be recycled.
645 */
646 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
647 vdrop((struct vnode *)object->handle);
648
649 m->object = NULL;
650}
651
652/*
653 * vm_page_lookup:
654 *
655 * Returns the page associated with the object/offset
656 * pair specified; if none is found, NULL is returned.
657 *
658 * The object must be locked.
659 * This routine may not block.
660 * This is a critical path routine
661 */
662vm_page_t
663vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
664{
665 vm_page_t m;
666
667 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
668 if ((m = object->root) != NULL && m->pindex != pindex) {
669 m = vm_page_splay(pindex, m);
670 if ((object->root = m)->pindex != pindex)
671 m = NULL;
672 }
673 return (m);
674}
675
676/*
677 * vm_page_rename:
678 *
679 * Move the given memory entry from its
680 * current object to the specified target object/offset.
681 *
682 * The object must be locked.
683 * This routine may not block.
684 *
685 * Note: swap associated with the page must be invalidated by the move. We
686 * have to do this for several reasons: (1) we aren't freeing the
687 * page, (2) we are dirtying the page, (3) the VM system is probably
688 * moving the page from object A to B, and will then later move
689 * the backing store from A to B and we can't have a conflict.
690 *
691 * Note: we *always* dirty the page. It is necessary both for the
692 * fact that we moved it, and because we may be invalidating
693 * swap. If the page is on the cache, we have to deactivate it
694 * or vm_page_dirty() will panic. Dirty pages are not allowed
695 * on the cache.
696 */
697void
698vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
699{
700
701 vm_page_remove(m);
702 vm_page_insert(m, new_object, new_pindex);
703 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
704 vm_page_deactivate(m);
705 vm_page_dirty(m);
706}
707
708/*
709 * vm_page_select_cache:
710 *
711 * Move a page of the given color from the cache queue to the free
712 * queue. As pages might be found, but are not applicable, they are
713 * deactivated.
714 *
715 * This routine may not block.
716 */
717vm_page_t
718vm_page_select_cache(int color)
719{
720 vm_object_t object;
721 vm_page_t m;
722 boolean_t was_trylocked;
723
724 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
725 while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
726 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
727 KASSERT(!pmap_page_is_mapped(m),
728 ("Found mapped cache page %p", m));
729 KASSERT((m->flags & PG_UNMANAGED) == 0,
730 ("Found unmanaged cache page %p", m));
731 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
732 if (m->hold_count == 0 && (object = m->object,
733 (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
734 VM_OBJECT_LOCKED(object))) {
735 KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
736 ("Found busy cache page %p", m));
737 vm_page_free(m);
738 if (was_trylocked)
739 VM_OBJECT_UNLOCK(object);
740 break;
741 }
742 vm_page_deactivate(m);
743 }
744 return (m);
745}
746
747/*
748 * vm_page_alloc:
749 *
750 * Allocate and return a memory cell associated
751 * with this VM object/offset pair.
752 *
753 * page_req classes:
754 * VM_ALLOC_NORMAL normal process request
755 * VM_ALLOC_SYSTEM system *really* needs a page
756 * VM_ALLOC_INTERRUPT interrupt time request
757 * VM_ALLOC_ZERO zero page
758 *
759 * This routine may not block.
760 *
761 * Additional special handling is required when called from an
762 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with
763 * the page cache in this case.
764 */
765vm_page_t
766vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
767{
768 vm_page_t m = NULL;
769 int color, flags, page_req;
770
771 page_req = req & VM_ALLOC_CLASS_MASK;
772 KASSERT(curthread->td_intr_nesting_level == 0 ||
773 page_req == VM_ALLOC_INTERRUPT,
774 ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
775
776 if ((req & VM_ALLOC_NOOBJ) == 0) {
777 KASSERT(object != NULL,
778 ("vm_page_alloc: NULL object."));
779 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
780 color = (pindex + object->pg_color) & PQ_COLORMASK;
781 } else
782 color = pindex & PQ_COLORMASK;
783
784 /*
785 * The pager is allowed to eat deeper into the free page list.
786 */
787 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
788 page_req = VM_ALLOC_SYSTEM;
789 };
790
791loop:
792 mtx_lock_spin(&vm_page_queue_free_mtx);
793 if (cnt.v_free_count > cnt.v_free_reserved ||
794 (page_req == VM_ALLOC_SYSTEM &&
795 cnt.v_cache_count == 0 &&
796 cnt.v_free_count > cnt.v_interrupt_free_min) ||
797 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
798 /*
799 * Allocate from the free queue if the number of free pages
800 * exceeds the minimum for the request class.
801 */
802 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
803 } else if (page_req != VM_ALLOC_INTERRUPT) {
804 mtx_unlock_spin(&vm_page_queue_free_mtx);
805 /*
806 * Allocatable from cache (non-interrupt only). On success,
807 * we must free the page and try again, thus ensuring that
808 * cnt.v_*_free_min counters are replenished.
809 */
810 vm_page_lock_queues();
811 if ((m = vm_page_select_cache(color)) == NULL) {
812 KASSERT(cnt.v_cache_count == 0,
813 ("vm_page_alloc: cache queue is missing %d pages",
814 cnt.v_cache_count));
815 vm_page_unlock_queues();
816 atomic_add_int(&vm_pageout_deficit, 1);
817 pagedaemon_wakeup();
818
819 if (page_req != VM_ALLOC_SYSTEM)
820 return NULL;
821
822 mtx_lock_spin(&vm_page_queue_free_mtx);
823 if (cnt.v_free_count <= cnt.v_interrupt_free_min) {
824 mtx_unlock_spin(&vm_page_queue_free_mtx);
825 return (NULL);
826 }
827 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
828 } else {
829 vm_page_unlock_queues();
830 goto loop;
831 }
832 } else {
833 /*
834 * Not allocatable from cache from interrupt, give up.
835 */
836 mtx_unlock_spin(&vm_page_queue_free_mtx);
837 atomic_add_int(&vm_pageout_deficit, 1);
838 pagedaemon_wakeup();
839 return (NULL);
840 }
841
842 /*
843 * At this point we had better have found a good page.
844 */
845
846 KASSERT(
847 m != NULL,
848 ("vm_page_alloc(): missing page on free queue")
849 );
850
851 /*
852 * Remove from free queue
853 */
854 vm_pageq_remove_nowakeup(m);
855
856 /*
857 * Initialize structure. Only the PG_ZERO flag is inherited.
858 */
859 flags = PG_BUSY;
860 if (m->flags & PG_ZERO) {
861 vm_page_zero_count--;
862 if (req & VM_ALLOC_ZERO)
863 flags = PG_ZERO | PG_BUSY;
864 }
865 if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
866 flags &= ~PG_BUSY;
867 m->flags = flags;
868 if (req & VM_ALLOC_WIRED) {
869 atomic_add_int(&cnt.v_wire_count, 1);
870 m->wire_count = 1;
871 } else
872 m->wire_count = 0;
873 m->hold_count = 0;
874 m->act_count = 0;
875 m->busy = 0;
876 m->valid = 0;
877 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
878 mtx_unlock_spin(&vm_page_queue_free_mtx);
879
880 if ((req & VM_ALLOC_NOOBJ) == 0)
881 vm_page_insert(m, object, pindex);
882 else
883 m->pindex = pindex;
884
885 /*
886 * Don't wakeup too often - wakeup the pageout daemon when
887 * we would be nearly out of memory.
888 */
889 if (vm_paging_needed())
890 pagedaemon_wakeup();
891
892 return (m);
893}
894
895/*
896 * vm_wait: (also see VM_WAIT macro)
897 *
898 * Block until free pages are available for allocation
899 * - Called in various places before memory allocations.
900 */
901void
902vm_wait(void)
903{
904
905 vm_page_lock_queues();
906 if (curproc == pageproc) {
907 vm_pageout_pages_needed = 1;
908 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
909 PDROP | PSWP, "VMWait", 0);
910 } else {
911 if (!vm_pages_needed) {
912 vm_pages_needed = 1;
913 wakeup(&vm_pages_needed);
914 }
915 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
916 "vmwait", 0);
917 }
918}
919
920/*
921 * vm_waitpfault: (also see VM_WAITPFAULT macro)
922 *
923 * Block until free pages are available for allocation
924 * - Called only in vm_fault so that processes page faulting
925 * can be easily tracked.
926 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
927 * processes will be able to grab memory first. Do not change
928 * this balance without careful testing first.
929 */
930void
931vm_waitpfault(void)
932{
933
934 vm_page_lock_queues();
935 if (!vm_pages_needed) {
936 vm_pages_needed = 1;
937 wakeup(&vm_pages_needed);
938 }
939 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
940 "pfault", 0);
941}
942
943/*
944 * vm_page_activate:
945 *
946 * Put the specified page on the active list (if appropriate).
947 * Ensure that act_count is at least ACT_INIT but do not otherwise
948 * mess with it.
949 *
950 * The page queues must be locked.
951 * This routine may not block.
952 */
953void
954vm_page_activate(vm_page_t m)
955{
956
957 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
958 if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
959 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
960 cnt.v_reactivated++;
961 vm_pageq_remove(m);
962 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
963 if (m->act_count < ACT_INIT)
964 m->act_count = ACT_INIT;
965 vm_pageq_enqueue(PQ_ACTIVE, m);
966 }
967 } else {
968 if (m->act_count < ACT_INIT)
969 m->act_count = ACT_INIT;
970 }
971}
972
973/*
974 * vm_page_free_wakeup:
975 *
976 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
977 * routine is called when a page has been added to the cache or free
978 * queues.
979 *
980 * The page queues must be locked.
981 * This routine may not block.
982 */
983static inline void
984vm_page_free_wakeup(void)
985{
986
987 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
988 /*
989 * if pageout daemon needs pages, then tell it that there are
990 * some free.
991 */
992 if (vm_pageout_pages_needed &&
993 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
994 wakeup(&vm_pageout_pages_needed);
995 vm_pageout_pages_needed = 0;
996 }
997 /*
998 * wakeup processes that are waiting on memory if we hit a
999 * high water mark. And wakeup scheduler process if we have
1000 * lots of memory. this process will swapin processes.
1001 */
1002 if (vm_pages_needed && !vm_page_count_min()) {
1003 vm_pages_needed = 0;
1004 wakeup(&cnt.v_free_count);
1005 }
1006}
1007
1008/*
1009 * vm_page_free_toq:
1010 *
1011 * Returns the given page to the PQ_FREE list,
1012 * disassociating it with any VM object.
1013 *
1014 * Object and page must be locked prior to entry.
1015 * This routine may not block.
1016 */
1017
1018void
1019vm_page_free_toq(vm_page_t m)
1020{
1021 struct vpgqueues *pq;
1022
1023 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1024 KASSERT(!pmap_page_is_mapped(m),
1025 ("vm_page_free_toq: freeing mapped page %p", m));
1026 cnt.v_tfree++;
1027
1028 if (m->busy || VM_PAGE_INQUEUE1(m, PQ_FREE)) {
1029 printf(
1030 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
1031 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
1032 m->hold_count);
1033 if (VM_PAGE_INQUEUE1(m, PQ_FREE))
1034 panic("vm_page_free: freeing free page");
1035 else
1036 panic("vm_page_free: freeing busy page");
1037 }
1038
1039 /*
1040 * unqueue, then remove page. Note that we cannot destroy
1041 * the page here because we do not want to call the pager's
1042 * callback routine until after we've put the page on the
1043 * appropriate free queue.
1044 */
1045 vm_pageq_remove_nowakeup(m);
1046 vm_page_remove(m);
1047
1048 /*
1049 * If fictitious remove object association and
1050 * return, otherwise delay object association removal.
1051 */
1052 if ((m->flags & PG_FICTITIOUS) != 0) {
1053 return;
1054 }
1055
1056 m->valid = 0;
1057 vm_page_undirty(m);
1058
1059 if (m->wire_count != 0) {
1060 if (m->wire_count > 1) {
1061 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1062 m->wire_count, (long)m->pindex);
1063 }
1064 panic("vm_page_free: freeing wired page");
1065 }
1066
1067 /*
1068 * Clear the UNMANAGED flag when freeing an unmanaged page.
1069 */
1070 if (m->flags & PG_UNMANAGED) {
1071 m->flags &= ~PG_UNMANAGED;
1072 }
1073
1074 if (m->hold_count != 0) {
1075 m->flags &= ~PG_ZERO;
1076 VM_PAGE_SETQUEUE2(m, PQ_HOLD);
1077 } else
1078 VM_PAGE_SETQUEUE1(m, PQ_FREE);
1079 pq = &vm_page_queues[VM_PAGE_GETQUEUE(m)];
1080 mtx_lock_spin(&vm_page_queue_free_mtx);
1081 pq->lcnt++;
1082 ++(*pq->cnt);
1083
1084 /*
1085 * Put zero'd pages on the end ( where we look for zero'd pages
1086 * first ) and non-zerod pages at the head.
1087 */
1088 if (m->flags & PG_ZERO) {
1089 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
1090 ++vm_page_zero_count;
1091 } else {
1092 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
1093 }
1094 mtx_unlock_spin(&vm_page_queue_free_mtx);
1095 vm_page_free_wakeup();
1096}
1097
1098/*
1099 * vm_page_unmanage:
1100 *
1101 * Prevent PV management from being done on the page. The page is
1102 * removed from the paging queues as if it were wired, and as a
1103 * consequence of no longer being managed the pageout daemon will not
1104 * touch it (since there is no way to locate the pte mappings for the
1105 * page). madvise() calls that mess with the pmap will also no longer
1106 * operate on the page.
1107 *
1108 * Beyond that the page is still reasonably 'normal'. Freeing the page
1109 * will clear the flag.
1110 *
1111 * This routine is used by OBJT_PHYS objects - objects using unswappable
1112 * physical memory as backing store rather then swap-backed memory and
1113 * will eventually be extended to support 4MB unmanaged physical
1114 * mappings.
1115 */
1116void
1117vm_page_unmanage(vm_page_t m)
1118{
1119
1120 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1121 if ((m->flags & PG_UNMANAGED) == 0) {
1122 if (m->wire_count == 0)
1123 vm_pageq_remove(m);
1124 }
1125 vm_page_flag_set(m, PG_UNMANAGED);
1126}
1127
1128/*
1129 * vm_page_wire:
1130 *
1131 * Mark this page as wired down by yet
1132 * another map, removing it from paging queues
1133 * as necessary.
1134 *
1135 * The page queues must be locked.
1136 * This routine may not block.
1137 */
1138void
1139vm_page_wire(vm_page_t m)
1140{
1141
1142 /*
1143 * Only bump the wire statistics if the page is not already wired,
1144 * and only unqueue the page if it is on some queue (if it is unmanaged
1145 * it is already off the queues).
1146 */
1147 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1148 if (m->flags & PG_FICTITIOUS)
1149 return;
1150 if (m->wire_count == 0) {
1151 if ((m->flags & PG_UNMANAGED) == 0)
1152 vm_pageq_remove(m);
1153 atomic_add_int(&cnt.v_wire_count, 1);
1154 }
1155 m->wire_count++;
1156 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1157}
1158
1159/*
1160 * vm_page_unwire:
1161 *
1162 * Release one wiring of this page, potentially
1163 * enabling it to be paged again.
1164 *
1165 * Many pages placed on the inactive queue should actually go
1166 * into the cache, but it is difficult to figure out which. What
1167 * we do instead, if the inactive target is well met, is to put
1168 * clean pages at the head of the inactive queue instead of the tail.
1169 * This will cause them to be moved to the cache more quickly and
1170 * if not actively re-referenced, freed more quickly. If we just
1171 * stick these pages at the end of the inactive queue, heavy filesystem
1172 * meta-data accesses can cause an unnecessary paging load on memory bound
1173 * processes. This optimization causes one-time-use metadata to be
1174 * reused more quickly.
1175 *
1176 * BUT, if we are in a low-memory situation we have no choice but to
1177 * put clean pages on the cache queue.
1178 *
1179 * A number of routines use vm_page_unwire() to guarantee that the page
1180 * will go into either the inactive or active queues, and will NEVER
1181 * be placed in the cache - for example, just after dirtying a page.
1182 * dirty pages in the cache are not allowed.
1183 *
1184 * The page queues must be locked.
1185 * This routine may not block.
1186 */
1187void
1188vm_page_unwire(vm_page_t m, int activate)
1189{
1190
1191 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1192 if (m->flags & PG_FICTITIOUS)
1193 return;
1194 if (m->wire_count > 0) {
1195 m->wire_count--;
1196 if (m->wire_count == 0) {
1197 atomic_subtract_int(&cnt.v_wire_count, 1);
1198 if (m->flags & PG_UNMANAGED) {
1199 ;
1200 } else if (activate)
1201 vm_pageq_enqueue(PQ_ACTIVE, m);
1202 else {
1203 vm_page_flag_clear(m, PG_WINATCFLS);
1204 vm_pageq_enqueue(PQ_INACTIVE, m);
1205 }
1206 }
1207 } else {
1208 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1209 }
1210}
1211
1212
1213/*
1214 * Move the specified page to the inactive queue. If the page has
1215 * any associated swap, the swap is deallocated.
1216 *
1217 * Normally athead is 0 resulting in LRU operation. athead is set
1218 * to 1 if we want this page to be 'as if it were placed in the cache',
1219 * except without unmapping it from the process address space.
1220 *
1221 * This routine may not block.
1222 */
1223static inline void
1224_vm_page_deactivate(vm_page_t m, int athead)
1225{
1226
1227 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1228
1229 /*
1230 * Ignore if already inactive.
1231 */
1232 if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE))
1233 return;
1234 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1235 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1236 cnt.v_reactivated++;
1237 vm_page_flag_clear(m, PG_WINATCFLS);
1238 vm_pageq_remove(m);
1239 if (athead)
1240 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1241 else
1242 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1243 VM_PAGE_SETQUEUE2(m, PQ_INACTIVE);
1244 vm_page_queues[PQ_INACTIVE].lcnt++;
1245 cnt.v_inactive_count++;
1246 }
1247}
1248
1249void
1250vm_page_deactivate(vm_page_t m)
1251{
1252 _vm_page_deactivate(m, 0);
1253}
1254
1255/*
1256 * vm_page_try_to_cache:
1257 *
1258 * Returns 0 on failure, 1 on success
1259 */
1260int
1261vm_page_try_to_cache(vm_page_t m)
1262{
1263
1264 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1265 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1266 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1267 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1268 return (0);
1269 }
1270 pmap_remove_all(m);
1271 if (m->dirty)
1272 return (0);
1273 vm_page_cache(m);
1274 return (1);
1275}
1276
1277/*
1278 * vm_page_try_to_free()
1279 *
1280 * Attempt to free the page. If we cannot free it, we do nothing.
1281 * 1 is returned on success, 0 on failure.
1282 */
1283int
1284vm_page_try_to_free(vm_page_t m)
1285{
1286
1287 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1288 if (m->object != NULL)
1289 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1290 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1291 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1292 return (0);
1293 }
1294 pmap_remove_all(m);
1295 if (m->dirty)
1296 return (0);
1297 vm_page_free(m);
1298 return (1);
1299}
1300
1301/*
1302 * vm_page_cache
1303 *
1304 * Put the specified page onto the page cache queue (if appropriate).
1305 *
1306 * This routine may not block.
1307 */
1308void
1309vm_page_cache(vm_page_t m)
1310{
1311
1312 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1313 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1314 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
1315 m->hold_count || m->wire_count) {
1316 printf("vm_page_cache: attempting to cache busy page\n");
1317 return;
1318 }
1319 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1320 return;
1321
1322 /*
1323 * Remove all pmaps and indicate that the page is not
1324 * writeable or mapped.
1325 */
1326 pmap_remove_all(m);
1327 if (m->dirty != 0) {
1328 panic("vm_page_cache: caching a dirty page, pindex: %ld",
1329 (long)m->pindex);
1330 }
1331 vm_pageq_remove_nowakeup(m);
1332 vm_pageq_enqueue(PQ_CACHE + m->pc, m);
1333 vm_page_free_wakeup();
1334}
1335
1336/*
1337 * vm_page_dontneed
1338 *
1339 * Cache, deactivate, or do nothing as appropriate. This routine
1340 * is typically used by madvise() MADV_DONTNEED.
1341 *
1342 * Generally speaking we want to move the page into the cache so
1343 * it gets reused quickly. However, this can result in a silly syndrome
1344 * due to the page recycling too quickly. Small objects will not be
1345 * fully cached. On the otherhand, if we move the page to the inactive
1346 * queue we wind up with a problem whereby very large objects
1347 * unnecessarily blow away our inactive and cache queues.
1348 *
1349 * The solution is to move the pages based on a fixed weighting. We
1350 * either leave them alone, deactivate them, or move them to the cache,
1351 * where moving them to the cache has the highest weighting.
1352 * By forcing some pages into other queues we eventually force the
1353 * system to balance the queues, potentially recovering other unrelated
1354 * space from active. The idea is to not force this to happen too
1355 * often.
1356 */
1357void
1358vm_page_dontneed(vm_page_t m)
1359{
1360 static int dnweight;
1361 int dnw;
1362 int head;
1363
1364 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1365 dnw = ++dnweight;
1366
1367 /*
1368 * occassionally leave the page alone
1369 */
1370 if ((dnw & 0x01F0) == 0 ||
1371 VM_PAGE_INQUEUE2(m, PQ_INACTIVE) ||
1372 VM_PAGE_INQUEUE1(m, PQ_CACHE)
1373 ) {
1374 if (m->act_count >= ACT_INIT)
1375 --m->act_count;
1376 return;
1377 }
1378
1379 if (m->dirty == 0 && pmap_is_modified(m))
1380 vm_page_dirty(m);
1381
1382 if (m->dirty || (dnw & 0x0070) == 0) {
1383 /*
1384 * Deactivate the page 3 times out of 32.
1385 */
1386 head = 0;
1387 } else {
1388 /*
1389 * Cache the page 28 times out of every 32. Note that
1390 * the page is deactivated instead of cached, but placed
1391 * at the head of the queue instead of the tail.
1392 */
1393 head = 1;
1394 }
1395 _vm_page_deactivate(m, head);
1396}
1397
1398/*
1399 * Grab a page, waiting until we are waken up due to the page
1400 * changing state. We keep on waiting, if the page continues
1401 * to be in the object. If the page doesn't exist, first allocate it
1402 * and then conditionally zero it.
1403 *
1404 * This routine may block.
1405 */
1406vm_page_t
1407vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1408{
1409 vm_page_t m;
1410
1411 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1412retrylookup:
1413 if ((m = vm_page_lookup(object, pindex)) != NULL) {
1414 vm_page_lock_queues();
1415 if (m->busy || (m->flags & PG_BUSY)) {
1416 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
1417 VM_OBJECT_UNLOCK(object);
1418 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
1419 VM_OBJECT_LOCK(object);
1420 if ((allocflags & VM_ALLOC_RETRY) == 0)
1421 return (NULL);
1422 goto retrylookup;
1423 } else {
1424 if (allocflags & VM_ALLOC_WIRED)
1425 vm_page_wire(m);
1426 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1427 vm_page_busy(m);
1428 vm_page_unlock_queues();
1429 return (m);
1430 }
1431 }
1432 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1433 if (m == NULL) {
1434 VM_OBJECT_UNLOCK(object);
1435 VM_WAIT;
1436 VM_OBJECT_LOCK(object);
1437 if ((allocflags & VM_ALLOC_RETRY) == 0)
1438 return (NULL);
1439 goto retrylookup;
1440 }
1441 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1442 pmap_zero_page(m);
1443 return (m);
1444}
1445
1446/*
1447 * Mapping function for valid bits or for dirty bits in
1448 * a page. May not block.
1449 *
1450 * Inputs are required to range within a page.
1451 */
1452inline int
1453vm_page_bits(int base, int size)
1454{
1455 int first_bit;
1456 int last_bit;
1457
1458 KASSERT(
1459 base + size <= PAGE_SIZE,
1460 ("vm_page_bits: illegal base/size %d/%d", base, size)
1461 );
1462
1463 if (size == 0) /* handle degenerate case */
1464 return (0);
1465
1466 first_bit = base >> DEV_BSHIFT;
1467 last_bit = (base + size - 1) >> DEV_BSHIFT;
1468
1469 return ((2 << last_bit) - (1 << first_bit));
1470}
1471
1472/*
1473 * vm_page_set_validclean:
1474 *
1475 * Sets portions of a page valid and clean. The arguments are expected
1476 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1477 * of any partial chunks touched by the range. The invalid portion of
1478 * such chunks will be zero'd.
1479 *
1480 * This routine may not block.
1481 *
1482 * (base + size) must be less then or equal to PAGE_SIZE.
1483 */
1484void
1485vm_page_set_validclean(vm_page_t m, int base, int size)
1486{
1487 int pagebits;
1488 int frag;
1489 int endoff;
1490
1491 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1492 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1493 if (size == 0) /* handle degenerate case */
1494 return;
1495
1496 /*
1497 * If the base is not DEV_BSIZE aligned and the valid
1498 * bit is clear, we have to zero out a portion of the
1499 * first block.
1500 */
1501 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1502 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1503 pmap_zero_page_area(m, frag, base - frag);
1504
1505 /*
1506 * If the ending offset is not DEV_BSIZE aligned and the
1507 * valid bit is clear, we have to zero out a portion of
1508 * the last block.
1509 */
1510 endoff = base + size;
1511 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1512 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1513 pmap_zero_page_area(m, endoff,
1514 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1515
1516 /*
1517 * Set valid, clear dirty bits. If validating the entire
1518 * page we can safely clear the pmap modify bit. We also
1519 * use this opportunity to clear the PG_NOSYNC flag. If a process
1520 * takes a write fault on a MAP_NOSYNC memory area the flag will
1521 * be set again.
1522 *
1523 * We set valid bits inclusive of any overlap, but we can only
1524 * clear dirty bits for DEV_BSIZE chunks that are fully within
1525 * the range.
1526 */
1527 pagebits = vm_page_bits(base, size);
1528 m->valid |= pagebits;
1529#if 0 /* NOT YET */
1530 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1531 frag = DEV_BSIZE - frag;
1532 base += frag;
1533 size -= frag;
1534 if (size < 0)
1535 size = 0;
1536 }
1537 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1538#endif
1539 m->dirty &= ~pagebits;
1540 if (base == 0 && size == PAGE_SIZE) {
1541 pmap_clear_modify(m);
1542 vm_page_flag_clear(m, PG_NOSYNC);
1543 }
1544}
1545
1546void
1547vm_page_clear_dirty(vm_page_t m, int base, int size)
1548{
1549
1550 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1551 m->dirty &= ~vm_page_bits(base, size);
1552}
1553
1554/*
1555 * vm_page_set_invalid:
1556 *
1557 * Invalidates DEV_BSIZE'd chunks within a page. Both the
1558 * valid and dirty bits for the effected areas are cleared.
1559 *
1560 * May not block.
1561 */
1562void
1563vm_page_set_invalid(vm_page_t m, int base, int size)
1564{
1565 int bits;
1566
1567 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1568 bits = vm_page_bits(base, size);
1569 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1570 if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
1571 pmap_remove_all(m);
1572 m->valid &= ~bits;
1573 m->dirty &= ~bits;
1574 m->object->generation++;
1575}
1576
1577/*
1578 * vm_page_zero_invalid()
1579 *
1580 * The kernel assumes that the invalid portions of a page contain
1581 * garbage, but such pages can be mapped into memory by user code.
1582 * When this occurs, we must zero out the non-valid portions of the
1583 * page so user code sees what it expects.
1584 *
1585 * Pages are most often semi-valid when the end of a file is mapped
1586 * into memory and the file's size is not page aligned.
1587 */
1588void
1589vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1590{
1591 int b;
1592 int i;
1593
1594 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1595 /*
1596 * Scan the valid bits looking for invalid sections that
1597 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
1598 * valid bit may be set ) have already been zerod by
1599 * vm_page_set_validclean().
1600 */
1601 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1602 if (i == (PAGE_SIZE / DEV_BSIZE) ||
1603 (m->valid & (1 << i))
1604 ) {
1605 if (i > b) {
1606 pmap_zero_page_area(m,
1607 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1608 }
1609 b = i + 1;
1610 }
1611 }
1612
1613 /*
1614 * setvalid is TRUE when we can safely set the zero'd areas
1615 * as being valid. We can do this if there are no cache consistancy
1616 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
1617 */
1618 if (setvalid)
1619 m->valid = VM_PAGE_BITS_ALL;
1620}
1621
1622/*
1623 * vm_page_is_valid:
1624 *
1625 * Is (partial) page valid? Note that the case where size == 0
1626 * will return FALSE in the degenerate case where the page is
1627 * entirely invalid, and TRUE otherwise.
1628 *
1629 * May not block.
1630 */
1631int
1632vm_page_is_valid(vm_page_t m, int base, int size)
1633{
1634 int bits = vm_page_bits(base, size);
1635
1636 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1637 if (m->valid && ((m->valid & bits) == bits))
1638 return 1;
1639 else
1640 return 0;
1641}
1642
1643/*
1644 * update dirty bits from pmap/mmu. May not block.
1645 */
1646void
1647vm_page_test_dirty(vm_page_t m)
1648{
1649 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1650 vm_page_dirty(m);
1651 }
1652}
1653
1654int so_zerocp_fullpage = 0;
1655
1656void
1657vm_page_cowfault(vm_page_t m)
1658{
1659 vm_page_t mnew;
1660 vm_object_t object;
1661 vm_pindex_t pindex;
1662
1663 object = m->object;
1664 pindex = m->pindex;
1665
1666 retry_alloc:
1667 pmap_remove_all(m);
1668 vm_page_remove(m);
1669 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
1670 if (mnew == NULL) {
1671 vm_page_insert(m, object, pindex);
1672 vm_page_unlock_queues();
1673 VM_OBJECT_UNLOCK(object);
1674 VM_WAIT;
1675 VM_OBJECT_LOCK(object);
1676 vm_page_lock_queues();
1677 goto retry_alloc;
1678 }
1679
1680 if (m->cow == 0) {
1681 /*
1682 * check to see if we raced with an xmit complete when
1683 * waiting to allocate a page. If so, put things back
1684 * the way they were
1685 */
1686 vm_page_free(mnew);
1687 vm_page_insert(m, object, pindex);
1688 } else { /* clear COW & copy page */
1689 if (!so_zerocp_fullpage)
1690 pmap_copy_page(m, mnew);
1691 mnew->valid = VM_PAGE_BITS_ALL;
1692 vm_page_dirty(mnew);
1693 vm_page_flag_clear(mnew, PG_BUSY);
1694 mnew->wire_count = m->wire_count - m->cow;
1695 m->wire_count = m->cow;
1696 }
1697}
1698
1699void
1700vm_page_cowclear(vm_page_t m)
1701{
1702
1703 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1704 if (m->cow) {
1705 m->cow--;
1706 /*
1707 * let vm_fault add back write permission lazily
1708 */
1709 }
1710 /*
1711 * sf_buf_free() will free the page, so we needn't do it here
1712 */
1713}
1714
1715void
1716vm_page_cowsetup(vm_page_t m)
1717{
1718
1719 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1720 m->cow++;
1721 pmap_page_protect(m, VM_PROT_READ);
1722}
1723
1724#include "opt_ddb.h"
1725#ifdef DDB
1726#include <sys/kernel.h>
1727
1728#include <ddb/ddb.h>
1729
1730DB_SHOW_COMMAND(page, vm_page_print_page_info)
1731{
1732 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
1733 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
1734 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
1735 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
1736 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
1737 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
1738 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
1739 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
1740 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
1741 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
1742}
1743
1744DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
1745{
1746 int i;
1747 db_printf("PQ_FREE:");
1748 for (i = 0; i < PQ_NUMCOLORS; i++) {
1749 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
1750 }
1751 db_printf("\n");
1752
1753 db_printf("PQ_CACHE:");
1754 for (i = 0; i < PQ_NUMCOLORS; i++) {
1755 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
1756 }
1757 db_printf("\n");
1758
1759 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
1760 vm_page_queues[PQ_ACTIVE].lcnt,
1761 vm_page_queues[PQ_INACTIVE].lcnt);
1762}
1763#endif /* DDB */
| 257 * Compute the number of pages of memory that will be available for
258 * use (taking into account the overhead of a page structure per
259 * page).
260 */
261 first_page = phys_avail[0] / PAGE_SIZE;
262 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
263 npages = (total - (page_range * sizeof(struct vm_page)) -
264 (end - new_end)) / PAGE_SIZE;
265 end = new_end;
266
267 /*
268 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
269 */
270 vaddr += PAGE_SIZE;
271
272 /*
273 * Initialize the mem entry structures now, and put them in the free
274 * queue.
275 */
276 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
277 mapped = pmap_map(&vaddr, new_end, end,
278 VM_PROT_READ | VM_PROT_WRITE);
279 vm_page_array = (vm_page_t) mapped;
280 phys_avail[biggestone + 1] = new_end;
281
282 /*
283 * Clear all of the page structures
284 */
285 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
286 vm_page_array_size = page_range;
287
288 /*
289 * Construct the free queue(s) in descending order (by physical
290 * address) so that the first 16MB of physical memory is allocated
291 * last rather than first. On large-memory machines, this avoids
292 * the exhaustion of low physical memory before isa_dma_init has run.
293 */
294 cnt.v_page_count = 0;
295 cnt.v_free_count = 0;
296 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
297 pa = phys_avail[i];
298 last_pa = phys_avail[i + 1];
299 while (pa < last_pa && npages-- > 0) {
300 vm_pageq_add_new_page(pa);
301 pa += PAGE_SIZE;
302 }
303 }
304 return (vaddr);
305}
306
307void
308vm_page_flag_set(vm_page_t m, unsigned short bits)
309{
310
311 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
312 m->flags |= bits;
313}
314
315void
316vm_page_flag_clear(vm_page_t m, unsigned short bits)
317{
318
319 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
320 m->flags &= ~bits;
321}
322
323void
324vm_page_busy(vm_page_t m)
325{
326
327 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
328 KASSERT((m->flags & PG_BUSY) == 0,
329 ("vm_page_busy: page already busy!!!"));
330 vm_page_flag_set(m, PG_BUSY);
331}
332
333/*
334 * vm_page_flash:
335 *
336 * wakeup anyone waiting for the page.
337 */
338void
339vm_page_flash(vm_page_t m)
340{
341
342 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
343 if (m->flags & PG_WANTED) {
344 vm_page_flag_clear(m, PG_WANTED);
345 wakeup(m);
346 }
347}
348
349/*
350 * vm_page_wakeup:
351 *
352 * clear the PG_BUSY flag and wakeup anyone waiting for the
353 * page.
354 *
355 */
356void
357vm_page_wakeup(vm_page_t m)
358{
359
360 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
361 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
362 vm_page_flag_clear(m, PG_BUSY);
363 vm_page_flash(m);
364}
365
366void
367vm_page_io_start(vm_page_t m)
368{
369
370 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
371 m->busy++;
372}
373
374void
375vm_page_io_finish(vm_page_t m)
376{
377
378 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
379 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
380 m->busy--;
381 if (m->busy == 0)
382 vm_page_flash(m);
383}
384
385/*
386 * Keep page from being freed by the page daemon
387 * much of the same effect as wiring, except much lower
388 * overhead and should be used only for *very* temporary
389 * holding ("wiring").
390 */
391void
392vm_page_hold(vm_page_t mem)
393{
394
395 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
396 mem->hold_count++;
397}
398
399void
400vm_page_unhold(vm_page_t mem)
401{
402
403 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
404 --mem->hold_count;
405 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
406 if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD))
407 vm_page_free_toq(mem);
408}
409
410/*
411 * vm_page_free:
412 *
413 * Free a page
414 *
415 * The clearing of PG_ZERO is a temporary safety until the code can be
416 * reviewed to determine that PG_ZERO is being properly cleared on
417 * write faults or maps. PG_ZERO was previously cleared in
418 * vm_page_alloc().
419 */
420void
421vm_page_free(vm_page_t m)
422{
423 vm_page_flag_clear(m, PG_ZERO);
424 vm_page_free_toq(m);
425 vm_page_zero_idle_wakeup();
426}
427
428/*
429 * vm_page_free_zero:
430 *
431 * Free a page to the zerod-pages queue
432 */
433void
434vm_page_free_zero(vm_page_t m)
435{
436 vm_page_flag_set(m, PG_ZERO);
437 vm_page_free_toq(m);
438}
439
440/*
441 * vm_page_sleep_if_busy:
442 *
443 * Sleep and release the page queues lock if PG_BUSY is set or,
444 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
445 * thread slept and the page queues lock was released.
446 * Otherwise, retains the page queues lock and returns FALSE.
447 */
448int
449vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
450{
451 vm_object_t object;
452
453 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
454 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
455 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
456 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
457 /*
458 * It's possible that while we sleep, the page will get
459 * unbusied and freed. If we are holding the object
460 * lock, we will assume we hold a reference to the object
461 * such that even if m->object changes, we can re-lock
462 * it.
463 */
464 object = m->object;
465 VM_OBJECT_UNLOCK(object);
466 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
467 VM_OBJECT_LOCK(object);
468 return (TRUE);
469 }
470 return (FALSE);
471}
472
473/*
474 * vm_page_dirty:
475 *
476 * make page all dirty
477 */
478void
479vm_page_dirty(vm_page_t m)
480{
481 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_CACHE,
482 ("vm_page_dirty: page in cache!"));
483 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_FREE,
484 ("vm_page_dirty: page is free!"));
485 m->dirty = VM_PAGE_BITS_ALL;
486}
487
488/*
489 * vm_page_splay:
490 *
491 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
492 * the vm_page containing the given pindex. If, however, that
493 * pindex is not found in the vm_object, returns a vm_page that is
494 * adjacent to the pindex, coming before or after it.
495 */
496vm_page_t
497vm_page_splay(vm_pindex_t pindex, vm_page_t root)
498{
499 struct vm_page dummy;
500 vm_page_t lefttreemax, righttreemin, y;
501
502 if (root == NULL)
503 return (root);
504 lefttreemax = righttreemin = &dummy;
505 for (;; root = y) {
506 if (pindex < root->pindex) {
507 if ((y = root->left) == NULL)
508 break;
509 if (pindex < y->pindex) {
510 /* Rotate right. */
511 root->left = y->right;
512 y->right = root;
513 root = y;
514 if ((y = root->left) == NULL)
515 break;
516 }
517 /* Link into the new root's right tree. */
518 righttreemin->left = root;
519 righttreemin = root;
520 } else if (pindex > root->pindex) {
521 if ((y = root->right) == NULL)
522 break;
523 if (pindex > y->pindex) {
524 /* Rotate left. */
525 root->right = y->left;
526 y->left = root;
527 root = y;
528 if ((y = root->right) == NULL)
529 break;
530 }
531 /* Link into the new root's left tree. */
532 lefttreemax->right = root;
533 lefttreemax = root;
534 } else
535 break;
536 }
537 /* Assemble the new root. */
538 lefttreemax->right = root->left;
539 righttreemin->left = root->right;
540 root->left = dummy.right;
541 root->right = dummy.left;
542 return (root);
543}
544
545/*
546 * vm_page_insert: [ internal use only ]
547 *
548 * Inserts the given mem entry into the object and object list.
549 *
550 * The pagetables are not updated but will presumably fault the page
551 * in if necessary, or if a kernel page the caller will at some point
552 * enter the page into the kernel's pmap. We are not allowed to block
553 * here so we *can't* do this anyway.
554 *
555 * The object and page must be locked.
556 * This routine may not block.
557 */
558void
559vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
560{
561 vm_page_t root;
562
563 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
564 if (m->object != NULL)
565 panic("vm_page_insert: page already inserted");
566
567 /*
568 * Record the object/offset pair in this page
569 */
570 m->object = object;
571 m->pindex = pindex;
572
573 /*
574 * Now link into the object's ordered list of backed pages.
575 */
576 root = object->root;
577 if (root == NULL) {
578 m->left = NULL;
579 m->right = NULL;
580 TAILQ_INSERT_TAIL(&object->memq, m, listq);
581 } else {
582 root = vm_page_splay(pindex, root);
583 if (pindex < root->pindex) {
584 m->left = root->left;
585 m->right = root;
586 root->left = NULL;
587 TAILQ_INSERT_BEFORE(root, m, listq);
588 } else if (pindex == root->pindex)
589 panic("vm_page_insert: offset already allocated");
590 else {
591 m->right = root->right;
592 m->left = root;
593 root->right = NULL;
594 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
595 }
596 }
597 object->root = m;
598 object->generation++;
599
600 /*
601 * show that the object has one more resident page.
602 */
603 object->resident_page_count++;
604 /*
605 * Hold the vnode until the last page is released.
606 */
607 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
608 vhold((struct vnode *)object->handle);
609
610 /*
611 * Since we are inserting a new and possibly dirty page,
612 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
613 */
614 if (m->flags & PG_WRITEABLE)
615 vm_object_set_writeable_dirty(object);
616}
617
618/*
619 * vm_page_remove:
620 * NOTE: used by device pager as well -wfj
621 *
622 * Removes the given mem entry from the object/offset-page
623 * table and the object page list, but do not invalidate/terminate
624 * the backing store.
625 *
626 * The object and page must be locked.
627 * The underlying pmap entry (if any) is NOT removed here.
628 * This routine may not block.
629 */
630void
631vm_page_remove(vm_page_t m)
632{
633 vm_object_t object;
634 vm_page_t root;
635
636 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
637 if ((object = m->object) == NULL)
638 return;
639 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
640 if (m->flags & PG_BUSY) {
641 vm_page_flag_clear(m, PG_BUSY);
642 vm_page_flash(m);
643 }
644
645 /*
646 * Now remove from the object's list of backed pages.
647 */
648 if (m != object->root)
649 vm_page_splay(m->pindex, object->root);
650 if (m->left == NULL)
651 root = m->right;
652 else {
653 root = vm_page_splay(m->pindex, m->left);
654 root->right = m->right;
655 }
656 object->root = root;
657 TAILQ_REMOVE(&object->memq, m, listq);
658
659 /*
660 * And show that the object has one fewer resident page.
661 */
662 object->resident_page_count--;
663 object->generation++;
664 /*
665 * The vnode may now be recycled.
666 */
667 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
668 vdrop((struct vnode *)object->handle);
669
670 m->object = NULL;
671}
672
673/*
674 * vm_page_lookup:
675 *
676 * Returns the page associated with the object/offset
677 * pair specified; if none is found, NULL is returned.
678 *
679 * The object must be locked.
680 * This routine may not block.
681 * This is a critical path routine
682 */
683vm_page_t
684vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
685{
686 vm_page_t m;
687
688 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
689 if ((m = object->root) != NULL && m->pindex != pindex) {
690 m = vm_page_splay(pindex, m);
691 if ((object->root = m)->pindex != pindex)
692 m = NULL;
693 }
694 return (m);
695}
696
697/*
698 * vm_page_rename:
699 *
700 * Move the given memory entry from its
701 * current object to the specified target object/offset.
702 *
703 * The object must be locked.
704 * This routine may not block.
705 *
706 * Note: swap associated with the page must be invalidated by the move. We
707 * have to do this for several reasons: (1) we aren't freeing the
708 * page, (2) we are dirtying the page, (3) the VM system is probably
709 * moving the page from object A to B, and will then later move
710 * the backing store from A to B and we can't have a conflict.
711 *
712 * Note: we *always* dirty the page. It is necessary both for the
713 * fact that we moved it, and because we may be invalidating
714 * swap. If the page is on the cache, we have to deactivate it
715 * or vm_page_dirty() will panic. Dirty pages are not allowed
716 * on the cache.
717 */
718void
719vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
720{
721
722 vm_page_remove(m);
723 vm_page_insert(m, new_object, new_pindex);
724 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
725 vm_page_deactivate(m);
726 vm_page_dirty(m);
727}
728
729/*
730 * vm_page_select_cache:
731 *
732 * Move a page of the given color from the cache queue to the free
733 * queue. As pages might be found, but are not applicable, they are
734 * deactivated.
735 *
736 * This routine may not block.
737 */
738vm_page_t
739vm_page_select_cache(int color)
740{
741 vm_object_t object;
742 vm_page_t m;
743 boolean_t was_trylocked;
744
745 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
746 while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
747 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
748 KASSERT(!pmap_page_is_mapped(m),
749 ("Found mapped cache page %p", m));
750 KASSERT((m->flags & PG_UNMANAGED) == 0,
751 ("Found unmanaged cache page %p", m));
752 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
753 if (m->hold_count == 0 && (object = m->object,
754 (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
755 VM_OBJECT_LOCKED(object))) {
756 KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
757 ("Found busy cache page %p", m));
758 vm_page_free(m);
759 if (was_trylocked)
760 VM_OBJECT_UNLOCK(object);
761 break;
762 }
763 vm_page_deactivate(m);
764 }
765 return (m);
766}
767
768/*
769 * vm_page_alloc:
770 *
771 * Allocate and return a memory cell associated
772 * with this VM object/offset pair.
773 *
774 * page_req classes:
775 * VM_ALLOC_NORMAL normal process request
776 * VM_ALLOC_SYSTEM system *really* needs a page
777 * VM_ALLOC_INTERRUPT interrupt time request
778 * VM_ALLOC_ZERO zero page
779 *
780 * This routine may not block.
781 *
782 * Additional special handling is required when called from an
783 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with
784 * the page cache in this case.
785 */
786vm_page_t
787vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
788{
789 vm_page_t m = NULL;
790 int color, flags, page_req;
791
792 page_req = req & VM_ALLOC_CLASS_MASK;
793 KASSERT(curthread->td_intr_nesting_level == 0 ||
794 page_req == VM_ALLOC_INTERRUPT,
795 ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
796
797 if ((req & VM_ALLOC_NOOBJ) == 0) {
798 KASSERT(object != NULL,
799 ("vm_page_alloc: NULL object."));
800 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
801 color = (pindex + object->pg_color) & PQ_COLORMASK;
802 } else
803 color = pindex & PQ_COLORMASK;
804
805 /*
806 * The pager is allowed to eat deeper into the free page list.
807 */
808 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
809 page_req = VM_ALLOC_SYSTEM;
810 };
811
812loop:
813 mtx_lock_spin(&vm_page_queue_free_mtx);
814 if (cnt.v_free_count > cnt.v_free_reserved ||
815 (page_req == VM_ALLOC_SYSTEM &&
816 cnt.v_cache_count == 0 &&
817 cnt.v_free_count > cnt.v_interrupt_free_min) ||
818 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
819 /*
820 * Allocate from the free queue if the number of free pages
821 * exceeds the minimum for the request class.
822 */
823 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
824 } else if (page_req != VM_ALLOC_INTERRUPT) {
825 mtx_unlock_spin(&vm_page_queue_free_mtx);
826 /*
827 * Allocatable from cache (non-interrupt only). On success,
828 * we must free the page and try again, thus ensuring that
829 * cnt.v_*_free_min counters are replenished.
830 */
831 vm_page_lock_queues();
832 if ((m = vm_page_select_cache(color)) == NULL) {
833 KASSERT(cnt.v_cache_count == 0,
834 ("vm_page_alloc: cache queue is missing %d pages",
835 cnt.v_cache_count));
836 vm_page_unlock_queues();
837 atomic_add_int(&vm_pageout_deficit, 1);
838 pagedaemon_wakeup();
839
840 if (page_req != VM_ALLOC_SYSTEM)
841 return NULL;
842
843 mtx_lock_spin(&vm_page_queue_free_mtx);
844 if (cnt.v_free_count <= cnt.v_interrupt_free_min) {
845 mtx_unlock_spin(&vm_page_queue_free_mtx);
846 return (NULL);
847 }
848 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
849 } else {
850 vm_page_unlock_queues();
851 goto loop;
852 }
853 } else {
854 /*
855 * Not allocatable from cache from interrupt, give up.
856 */
857 mtx_unlock_spin(&vm_page_queue_free_mtx);
858 atomic_add_int(&vm_pageout_deficit, 1);
859 pagedaemon_wakeup();
860 return (NULL);
861 }
862
863 /*
864 * At this point we had better have found a good page.
865 */
866
867 KASSERT(
868 m != NULL,
869 ("vm_page_alloc(): missing page on free queue")
870 );
871
872 /*
873 * Remove from free queue
874 */
875 vm_pageq_remove_nowakeup(m);
876
877 /*
878 * Initialize structure. Only the PG_ZERO flag is inherited.
879 */
880 flags = PG_BUSY;
881 if (m->flags & PG_ZERO) {
882 vm_page_zero_count--;
883 if (req & VM_ALLOC_ZERO)
884 flags = PG_ZERO | PG_BUSY;
885 }
886 if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
887 flags &= ~PG_BUSY;
888 m->flags = flags;
889 if (req & VM_ALLOC_WIRED) {
890 atomic_add_int(&cnt.v_wire_count, 1);
891 m->wire_count = 1;
892 } else
893 m->wire_count = 0;
894 m->hold_count = 0;
895 m->act_count = 0;
896 m->busy = 0;
897 m->valid = 0;
898 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
899 mtx_unlock_spin(&vm_page_queue_free_mtx);
900
901 if ((req & VM_ALLOC_NOOBJ) == 0)
902 vm_page_insert(m, object, pindex);
903 else
904 m->pindex = pindex;
905
906 /*
907 * Don't wakeup too often - wakeup the pageout daemon when
908 * we would be nearly out of memory.
909 */
910 if (vm_paging_needed())
911 pagedaemon_wakeup();
912
913 return (m);
914}
915
916/*
917 * vm_wait: (also see VM_WAIT macro)
918 *
919 * Block until free pages are available for allocation
920 * - Called in various places before memory allocations.
921 */
922void
923vm_wait(void)
924{
925
926 vm_page_lock_queues();
927 if (curproc == pageproc) {
928 vm_pageout_pages_needed = 1;
929 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
930 PDROP | PSWP, "VMWait", 0);
931 } else {
932 if (!vm_pages_needed) {
933 vm_pages_needed = 1;
934 wakeup(&vm_pages_needed);
935 }
936 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
937 "vmwait", 0);
938 }
939}
940
941/*
942 * vm_waitpfault: (also see VM_WAITPFAULT macro)
943 *
944 * Block until free pages are available for allocation
945 * - Called only in vm_fault so that processes page faulting
946 * can be easily tracked.
947 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
948 * processes will be able to grab memory first. Do not change
949 * this balance without careful testing first.
950 */
951void
952vm_waitpfault(void)
953{
954
955 vm_page_lock_queues();
956 if (!vm_pages_needed) {
957 vm_pages_needed = 1;
958 wakeup(&vm_pages_needed);
959 }
960 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
961 "pfault", 0);
962}
963
964/*
965 * vm_page_activate:
966 *
967 * Put the specified page on the active list (if appropriate).
968 * Ensure that act_count is at least ACT_INIT but do not otherwise
969 * mess with it.
970 *
971 * The page queues must be locked.
972 * This routine may not block.
973 */
974void
975vm_page_activate(vm_page_t m)
976{
977
978 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
979 if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
980 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
981 cnt.v_reactivated++;
982 vm_pageq_remove(m);
983 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
984 if (m->act_count < ACT_INIT)
985 m->act_count = ACT_INIT;
986 vm_pageq_enqueue(PQ_ACTIVE, m);
987 }
988 } else {
989 if (m->act_count < ACT_INIT)
990 m->act_count = ACT_INIT;
991 }
992}
993
994/*
995 * vm_page_free_wakeup:
996 *
997 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
998 * routine is called when a page has been added to the cache or free
999 * queues.
1000 *
1001 * The page queues must be locked.
1002 * This routine may not block.
1003 */
1004static inline void
1005vm_page_free_wakeup(void)
1006{
1007
1008 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1009 /*
1010 * if pageout daemon needs pages, then tell it that there are
1011 * some free.
1012 */
1013 if (vm_pageout_pages_needed &&
1014 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1015 wakeup(&vm_pageout_pages_needed);
1016 vm_pageout_pages_needed = 0;
1017 }
1018 /*
1019 * wakeup processes that are waiting on memory if we hit a
1020 * high water mark. And wakeup scheduler process if we have
1021 * lots of memory. this process will swapin processes.
1022 */
1023 if (vm_pages_needed && !vm_page_count_min()) {
1024 vm_pages_needed = 0;
1025 wakeup(&cnt.v_free_count);
1026 }
1027}
1028
1029/*
1030 * vm_page_free_toq:
1031 *
1032 * Returns the given page to the PQ_FREE list,
1033 * disassociating it with any VM object.
1034 *
1035 * Object and page must be locked prior to entry.
1036 * This routine may not block.
1037 */
1038
1039void
1040vm_page_free_toq(vm_page_t m)
1041{
1042 struct vpgqueues *pq;
1043
1044 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1045 KASSERT(!pmap_page_is_mapped(m),
1046 ("vm_page_free_toq: freeing mapped page %p", m));
1047 cnt.v_tfree++;
1048
1049 if (m->busy || VM_PAGE_INQUEUE1(m, PQ_FREE)) {
1050 printf(
1051 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
1052 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
1053 m->hold_count);
1054 if (VM_PAGE_INQUEUE1(m, PQ_FREE))
1055 panic("vm_page_free: freeing free page");
1056 else
1057 panic("vm_page_free: freeing busy page");
1058 }
1059
1060 /*
1061 * unqueue, then remove page. Note that we cannot destroy
1062 * the page here because we do not want to call the pager's
1063 * callback routine until after we've put the page on the
1064 * appropriate free queue.
1065 */
1066 vm_pageq_remove_nowakeup(m);
1067 vm_page_remove(m);
1068
1069 /*
1070 * If fictitious remove object association and
1071 * return, otherwise delay object association removal.
1072 */
1073 if ((m->flags & PG_FICTITIOUS) != 0) {
1074 return;
1075 }
1076
1077 m->valid = 0;
1078 vm_page_undirty(m);
1079
1080 if (m->wire_count != 0) {
1081 if (m->wire_count > 1) {
1082 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1083 m->wire_count, (long)m->pindex);
1084 }
1085 panic("vm_page_free: freeing wired page");
1086 }
1087
1088 /*
1089 * Clear the UNMANAGED flag when freeing an unmanaged page.
1090 */
1091 if (m->flags & PG_UNMANAGED) {
1092 m->flags &= ~PG_UNMANAGED;
1093 }
1094
1095 if (m->hold_count != 0) {
1096 m->flags &= ~PG_ZERO;
1097 VM_PAGE_SETQUEUE2(m, PQ_HOLD);
1098 } else
1099 VM_PAGE_SETQUEUE1(m, PQ_FREE);
1100 pq = &vm_page_queues[VM_PAGE_GETQUEUE(m)];
1101 mtx_lock_spin(&vm_page_queue_free_mtx);
1102 pq->lcnt++;
1103 ++(*pq->cnt);
1104
1105 /*
1106 * Put zero'd pages on the end ( where we look for zero'd pages
1107 * first ) and non-zerod pages at the head.
1108 */
1109 if (m->flags & PG_ZERO) {
1110 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
1111 ++vm_page_zero_count;
1112 } else {
1113 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
1114 }
1115 mtx_unlock_spin(&vm_page_queue_free_mtx);
1116 vm_page_free_wakeup();
1117}
1118
1119/*
1120 * vm_page_unmanage:
1121 *
1122 * Prevent PV management from being done on the page. The page is
1123 * removed from the paging queues as if it were wired, and as a
1124 * consequence of no longer being managed the pageout daemon will not
1125 * touch it (since there is no way to locate the pte mappings for the
1126 * page). madvise() calls that mess with the pmap will also no longer
1127 * operate on the page.
1128 *
1129 * Beyond that the page is still reasonably 'normal'. Freeing the page
1130 * will clear the flag.
1131 *
1132 * This routine is used by OBJT_PHYS objects - objects using unswappable
1133 * physical memory as backing store rather then swap-backed memory and
1134 * will eventually be extended to support 4MB unmanaged physical
1135 * mappings.
1136 */
1137void
1138vm_page_unmanage(vm_page_t m)
1139{
1140
1141 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1142 if ((m->flags & PG_UNMANAGED) == 0) {
1143 if (m->wire_count == 0)
1144 vm_pageq_remove(m);
1145 }
1146 vm_page_flag_set(m, PG_UNMANAGED);
1147}
1148
1149/*
1150 * vm_page_wire:
1151 *
1152 * Mark this page as wired down by yet
1153 * another map, removing it from paging queues
1154 * as necessary.
1155 *
1156 * The page queues must be locked.
1157 * This routine may not block.
1158 */
1159void
1160vm_page_wire(vm_page_t m)
1161{
1162
1163 /*
1164 * Only bump the wire statistics if the page is not already wired,
1165 * and only unqueue the page if it is on some queue (if it is unmanaged
1166 * it is already off the queues).
1167 */
1168 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1169 if (m->flags & PG_FICTITIOUS)
1170 return;
1171 if (m->wire_count == 0) {
1172 if ((m->flags & PG_UNMANAGED) == 0)
1173 vm_pageq_remove(m);
1174 atomic_add_int(&cnt.v_wire_count, 1);
1175 }
1176 m->wire_count++;
1177 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1178}
1179
1180/*
1181 * vm_page_unwire:
1182 *
1183 * Release one wiring of this page, potentially
1184 * enabling it to be paged again.
1185 *
1186 * Many pages placed on the inactive queue should actually go
1187 * into the cache, but it is difficult to figure out which. What
1188 * we do instead, if the inactive target is well met, is to put
1189 * clean pages at the head of the inactive queue instead of the tail.
1190 * This will cause them to be moved to the cache more quickly and
1191 * if not actively re-referenced, freed more quickly. If we just
1192 * stick these pages at the end of the inactive queue, heavy filesystem
1193 * meta-data accesses can cause an unnecessary paging load on memory bound
1194 * processes. This optimization causes one-time-use metadata to be
1195 * reused more quickly.
1196 *
1197 * BUT, if we are in a low-memory situation we have no choice but to
1198 * put clean pages on the cache queue.
1199 *
1200 * A number of routines use vm_page_unwire() to guarantee that the page
1201 * will go into either the inactive or active queues, and will NEVER
1202 * be placed in the cache - for example, just after dirtying a page.
1203 * dirty pages in the cache are not allowed.
1204 *
1205 * The page queues must be locked.
1206 * This routine may not block.
1207 */
1208void
1209vm_page_unwire(vm_page_t m, int activate)
1210{
1211
1212 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1213 if (m->flags & PG_FICTITIOUS)
1214 return;
1215 if (m->wire_count > 0) {
1216 m->wire_count--;
1217 if (m->wire_count == 0) {
1218 atomic_subtract_int(&cnt.v_wire_count, 1);
1219 if (m->flags & PG_UNMANAGED) {
1220 ;
1221 } else if (activate)
1222 vm_pageq_enqueue(PQ_ACTIVE, m);
1223 else {
1224 vm_page_flag_clear(m, PG_WINATCFLS);
1225 vm_pageq_enqueue(PQ_INACTIVE, m);
1226 }
1227 }
1228 } else {
1229 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1230 }
1231}
1232
1233
1234/*
1235 * Move the specified page to the inactive queue. If the page has
1236 * any associated swap, the swap is deallocated.
1237 *
1238 * Normally athead is 0 resulting in LRU operation. athead is set
1239 * to 1 if we want this page to be 'as if it were placed in the cache',
1240 * except without unmapping it from the process address space.
1241 *
1242 * This routine may not block.
1243 */
1244static inline void
1245_vm_page_deactivate(vm_page_t m, int athead)
1246{
1247
1248 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1249
1250 /*
1251 * Ignore if already inactive.
1252 */
1253 if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE))
1254 return;
1255 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1256 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1257 cnt.v_reactivated++;
1258 vm_page_flag_clear(m, PG_WINATCFLS);
1259 vm_pageq_remove(m);
1260 if (athead)
1261 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1262 else
1263 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1264 VM_PAGE_SETQUEUE2(m, PQ_INACTIVE);
1265 vm_page_queues[PQ_INACTIVE].lcnt++;
1266 cnt.v_inactive_count++;
1267 }
1268}
1269
1270void
1271vm_page_deactivate(vm_page_t m)
1272{
1273 _vm_page_deactivate(m, 0);
1274}
1275
1276/*
1277 * vm_page_try_to_cache:
1278 *
1279 * Returns 0 on failure, 1 on success
1280 */
1281int
1282vm_page_try_to_cache(vm_page_t m)
1283{
1284
1285 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1286 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1287 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1288 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1289 return (0);
1290 }
1291 pmap_remove_all(m);
1292 if (m->dirty)
1293 return (0);
1294 vm_page_cache(m);
1295 return (1);
1296}
1297
1298/*
1299 * vm_page_try_to_free()
1300 *
1301 * Attempt to free the page. If we cannot free it, we do nothing.
1302 * 1 is returned on success, 0 on failure.
1303 */
1304int
1305vm_page_try_to_free(vm_page_t m)
1306{
1307
1308 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1309 if (m->object != NULL)
1310 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1311 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1312 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1313 return (0);
1314 }
1315 pmap_remove_all(m);
1316 if (m->dirty)
1317 return (0);
1318 vm_page_free(m);
1319 return (1);
1320}
1321
1322/*
1323 * vm_page_cache
1324 *
1325 * Put the specified page onto the page cache queue (if appropriate).
1326 *
1327 * This routine may not block.
1328 */
1329void
1330vm_page_cache(vm_page_t m)
1331{
1332
1333 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1334 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1335 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
1336 m->hold_count || m->wire_count) {
1337 printf("vm_page_cache: attempting to cache busy page\n");
1338 return;
1339 }
1340 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1341 return;
1342
1343 /*
1344 * Remove all pmaps and indicate that the page is not
1345 * writeable or mapped.
1346 */
1347 pmap_remove_all(m);
1348 if (m->dirty != 0) {
1349 panic("vm_page_cache: caching a dirty page, pindex: %ld",
1350 (long)m->pindex);
1351 }
1352 vm_pageq_remove_nowakeup(m);
1353 vm_pageq_enqueue(PQ_CACHE + m->pc, m);
1354 vm_page_free_wakeup();
1355}
1356
1357/*
1358 * vm_page_dontneed
1359 *
1360 * Cache, deactivate, or do nothing as appropriate. This routine
1361 * is typically used by madvise() MADV_DONTNEED.
1362 *
1363 * Generally speaking we want to move the page into the cache so
1364 * it gets reused quickly. However, this can result in a silly syndrome
1365 * due to the page recycling too quickly. Small objects will not be
1366 * fully cached. On the otherhand, if we move the page to the inactive
1367 * queue we wind up with a problem whereby very large objects
1368 * unnecessarily blow away our inactive and cache queues.
1369 *
1370 * The solution is to move the pages based on a fixed weighting. We
1371 * either leave them alone, deactivate them, or move them to the cache,
1372 * where moving them to the cache has the highest weighting.
1373 * By forcing some pages into other queues we eventually force the
1374 * system to balance the queues, potentially recovering other unrelated
1375 * space from active. The idea is to not force this to happen too
1376 * often.
1377 */
1378void
1379vm_page_dontneed(vm_page_t m)
1380{
1381 static int dnweight;
1382 int dnw;
1383 int head;
1384
1385 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1386 dnw = ++dnweight;
1387
1388 /*
1389 * occassionally leave the page alone
1390 */
1391 if ((dnw & 0x01F0) == 0 ||
1392 VM_PAGE_INQUEUE2(m, PQ_INACTIVE) ||
1393 VM_PAGE_INQUEUE1(m, PQ_CACHE)
1394 ) {
1395 if (m->act_count >= ACT_INIT)
1396 --m->act_count;
1397 return;
1398 }
1399
1400 if (m->dirty == 0 && pmap_is_modified(m))
1401 vm_page_dirty(m);
1402
1403 if (m->dirty || (dnw & 0x0070) == 0) {
1404 /*
1405 * Deactivate the page 3 times out of 32.
1406 */
1407 head = 0;
1408 } else {
1409 /*
1410 * Cache the page 28 times out of every 32. Note that
1411 * the page is deactivated instead of cached, but placed
1412 * at the head of the queue instead of the tail.
1413 */
1414 head = 1;
1415 }
1416 _vm_page_deactivate(m, head);
1417}
1418
1419/*
1420 * Grab a page, waiting until we are waken up due to the page
1421 * changing state. We keep on waiting, if the page continues
1422 * to be in the object. If the page doesn't exist, first allocate it
1423 * and then conditionally zero it.
1424 *
1425 * This routine may block.
1426 */
1427vm_page_t
1428vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1429{
1430 vm_page_t m;
1431
1432 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1433retrylookup:
1434 if ((m = vm_page_lookup(object, pindex)) != NULL) {
1435 vm_page_lock_queues();
1436 if (m->busy || (m->flags & PG_BUSY)) {
1437 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
1438 VM_OBJECT_UNLOCK(object);
1439 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
1440 VM_OBJECT_LOCK(object);
1441 if ((allocflags & VM_ALLOC_RETRY) == 0)
1442 return (NULL);
1443 goto retrylookup;
1444 } else {
1445 if (allocflags & VM_ALLOC_WIRED)
1446 vm_page_wire(m);
1447 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1448 vm_page_busy(m);
1449 vm_page_unlock_queues();
1450 return (m);
1451 }
1452 }
1453 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1454 if (m == NULL) {
1455 VM_OBJECT_UNLOCK(object);
1456 VM_WAIT;
1457 VM_OBJECT_LOCK(object);
1458 if ((allocflags & VM_ALLOC_RETRY) == 0)
1459 return (NULL);
1460 goto retrylookup;
1461 }
1462 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1463 pmap_zero_page(m);
1464 return (m);
1465}
1466
1467/*
1468 * Mapping function for valid bits or for dirty bits in
1469 * a page. May not block.
1470 *
1471 * Inputs are required to range within a page.
1472 */
1473inline int
1474vm_page_bits(int base, int size)
1475{
1476 int first_bit;
1477 int last_bit;
1478
1479 KASSERT(
1480 base + size <= PAGE_SIZE,
1481 ("vm_page_bits: illegal base/size %d/%d", base, size)
1482 );
1483
1484 if (size == 0) /* handle degenerate case */
1485 return (0);
1486
1487 first_bit = base >> DEV_BSHIFT;
1488 last_bit = (base + size - 1) >> DEV_BSHIFT;
1489
1490 return ((2 << last_bit) - (1 << first_bit));
1491}
1492
1493/*
1494 * vm_page_set_validclean:
1495 *
1496 * Sets portions of a page valid and clean. The arguments are expected
1497 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1498 * of any partial chunks touched by the range. The invalid portion of
1499 * such chunks will be zero'd.
1500 *
1501 * This routine may not block.
1502 *
1503 * (base + size) must be less then or equal to PAGE_SIZE.
1504 */
1505void
1506vm_page_set_validclean(vm_page_t m, int base, int size)
1507{
1508 int pagebits;
1509 int frag;
1510 int endoff;
1511
1512 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1513 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1514 if (size == 0) /* handle degenerate case */
1515 return;
1516
1517 /*
1518 * If the base is not DEV_BSIZE aligned and the valid
1519 * bit is clear, we have to zero out a portion of the
1520 * first block.
1521 */
1522 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1523 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1524 pmap_zero_page_area(m, frag, base - frag);
1525
1526 /*
1527 * If the ending offset is not DEV_BSIZE aligned and the
1528 * valid bit is clear, we have to zero out a portion of
1529 * the last block.
1530 */
1531 endoff = base + size;
1532 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1533 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1534 pmap_zero_page_area(m, endoff,
1535 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1536
1537 /*
1538 * Set valid, clear dirty bits. If validating the entire
1539 * page we can safely clear the pmap modify bit. We also
1540 * use this opportunity to clear the PG_NOSYNC flag. If a process
1541 * takes a write fault on a MAP_NOSYNC memory area the flag will
1542 * be set again.
1543 *
1544 * We set valid bits inclusive of any overlap, but we can only
1545 * clear dirty bits for DEV_BSIZE chunks that are fully within
1546 * the range.
1547 */
1548 pagebits = vm_page_bits(base, size);
1549 m->valid |= pagebits;
1550#if 0 /* NOT YET */
1551 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1552 frag = DEV_BSIZE - frag;
1553 base += frag;
1554 size -= frag;
1555 if (size < 0)
1556 size = 0;
1557 }
1558 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1559#endif
1560 m->dirty &= ~pagebits;
1561 if (base == 0 && size == PAGE_SIZE) {
1562 pmap_clear_modify(m);
1563 vm_page_flag_clear(m, PG_NOSYNC);
1564 }
1565}
1566
1567void
1568vm_page_clear_dirty(vm_page_t m, int base, int size)
1569{
1570
1571 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1572 m->dirty &= ~vm_page_bits(base, size);
1573}
1574
1575/*
1576 * vm_page_set_invalid:
1577 *
1578 * Invalidates DEV_BSIZE'd chunks within a page. Both the
1579 * valid and dirty bits for the effected areas are cleared.
1580 *
1581 * May not block.
1582 */
1583void
1584vm_page_set_invalid(vm_page_t m, int base, int size)
1585{
1586 int bits;
1587
1588 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1589 bits = vm_page_bits(base, size);
1590 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1591 if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
1592 pmap_remove_all(m);
1593 m->valid &= ~bits;
1594 m->dirty &= ~bits;
1595 m->object->generation++;
1596}
1597
1598/*
1599 * vm_page_zero_invalid()
1600 *
1601 * The kernel assumes that the invalid portions of a page contain
1602 * garbage, but such pages can be mapped into memory by user code.
1603 * When this occurs, we must zero out the non-valid portions of the
1604 * page so user code sees what it expects.
1605 *
1606 * Pages are most often semi-valid when the end of a file is mapped
1607 * into memory and the file's size is not page aligned.
1608 */
1609void
1610vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1611{
1612 int b;
1613 int i;
1614
1615 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1616 /*
1617 * Scan the valid bits looking for invalid sections that
1618 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
1619 * valid bit may be set ) have already been zerod by
1620 * vm_page_set_validclean().
1621 */
1622 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1623 if (i == (PAGE_SIZE / DEV_BSIZE) ||
1624 (m->valid & (1 << i))
1625 ) {
1626 if (i > b) {
1627 pmap_zero_page_area(m,
1628 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1629 }
1630 b = i + 1;
1631 }
1632 }
1633
1634 /*
1635 * setvalid is TRUE when we can safely set the zero'd areas
1636 * as being valid. We can do this if there are no cache consistancy
1637 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
1638 */
1639 if (setvalid)
1640 m->valid = VM_PAGE_BITS_ALL;
1641}
1642
1643/*
1644 * vm_page_is_valid:
1645 *
1646 * Is (partial) page valid? Note that the case where size == 0
1647 * will return FALSE in the degenerate case where the page is
1648 * entirely invalid, and TRUE otherwise.
1649 *
1650 * May not block.
1651 */
1652int
1653vm_page_is_valid(vm_page_t m, int base, int size)
1654{
1655 int bits = vm_page_bits(base, size);
1656
1657 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1658 if (m->valid && ((m->valid & bits) == bits))
1659 return 1;
1660 else
1661 return 0;
1662}
1663
1664/*
1665 * update dirty bits from pmap/mmu. May not block.
1666 */
1667void
1668vm_page_test_dirty(vm_page_t m)
1669{
1670 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1671 vm_page_dirty(m);
1672 }
1673}
1674
1675int so_zerocp_fullpage = 0;
1676
1677void
1678vm_page_cowfault(vm_page_t m)
1679{
1680 vm_page_t mnew;
1681 vm_object_t object;
1682 vm_pindex_t pindex;
1683
1684 object = m->object;
1685 pindex = m->pindex;
1686
1687 retry_alloc:
1688 pmap_remove_all(m);
1689 vm_page_remove(m);
1690 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
1691 if (mnew == NULL) {
1692 vm_page_insert(m, object, pindex);
1693 vm_page_unlock_queues();
1694 VM_OBJECT_UNLOCK(object);
1695 VM_WAIT;
1696 VM_OBJECT_LOCK(object);
1697 vm_page_lock_queues();
1698 goto retry_alloc;
1699 }
1700
1701 if (m->cow == 0) {
1702 /*
1703 * check to see if we raced with an xmit complete when
1704 * waiting to allocate a page. If so, put things back
1705 * the way they were
1706 */
1707 vm_page_free(mnew);
1708 vm_page_insert(m, object, pindex);
1709 } else { /* clear COW & copy page */
1710 if (!so_zerocp_fullpage)
1711 pmap_copy_page(m, mnew);
1712 mnew->valid = VM_PAGE_BITS_ALL;
1713 vm_page_dirty(mnew);
1714 vm_page_flag_clear(mnew, PG_BUSY);
1715 mnew->wire_count = m->wire_count - m->cow;
1716 m->wire_count = m->cow;
1717 }
1718}
1719
1720void
1721vm_page_cowclear(vm_page_t m)
1722{
1723
1724 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1725 if (m->cow) {
1726 m->cow--;
1727 /*
1728 * let vm_fault add back write permission lazily
1729 */
1730 }
1731 /*
1732 * sf_buf_free() will free the page, so we needn't do it here
1733 */
1734}
1735
1736void
1737vm_page_cowsetup(vm_page_t m)
1738{
1739
1740 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1741 m->cow++;
1742 pmap_page_protect(m, VM_PROT_READ);
1743}
1744
1745#include "opt_ddb.h"
1746#ifdef DDB
1747#include <sys/kernel.h>
1748
1749#include <ddb/ddb.h>
1750
1751DB_SHOW_COMMAND(page, vm_page_print_page_info)
1752{
1753 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
1754 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
1755 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
1756 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
1757 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
1758 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
1759 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
1760 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
1761 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
1762 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
1763}
1764
1765DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
1766{
1767 int i;
1768 db_printf("PQ_FREE:");
1769 for (i = 0; i < PQ_NUMCOLORS; i++) {
1770 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
1771 }
1772 db_printf("\n");
1773
1774 db_printf("PQ_CACHE:");
1775 for (i = 0; i < PQ_NUMCOLORS; i++) {
1776 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
1777 }
1778 db_printf("\n");
1779
1780 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
1781 vm_page_queues[PQ_ACTIVE].lcnt,
1782 vm_page_queues[PQ_INACTIVE].lcnt);
1783}
1784#endif /* DDB */
|