Deleted Added
sdiff udiff text old ( 12662 ) new ( 12767 )
full compact
vm_fault.c (12662) vm_fault.c (12767)
1/*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *

--- 52 unchanged lines hidden (view full) ---

61 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62 * School of Computer Science
63 * Carnegie Mellon University
64 * Pittsburgh PA 15213-3890
65 *
66 * any improvements or extensions that they make and grant Carnegie the
67 * rights to redistribute these changes.
68 *
1/*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *

--- 52 unchanged lines hidden (view full) ---

61 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62 * School of Computer Science
63 * Carnegie Mellon University
64 * Pittsburgh PA 15213-3890
65 *
66 * any improvements or extensions that they make and grant Carnegie the
67 * rights to redistribute these changes.
68 *
69 * $Id: vm_fault.c,v 1.37 1995/11/20 12:19:53 phk Exp $
69 * $Id: vm_fault.c,v 1.38 1995/12/07 12:48:10 davidg Exp $
70 */
71
72/*
73 * Page fault handling module.
74 */
75
76#include <sys/param.h>
77#include <sys/systm.h>

--- 46 unchanged lines hidden (view full) ---

124int
125vm_fault(map, vaddr, fault_type, change_wiring)
126 vm_map_t map;
127 vm_offset_t vaddr;
128 vm_prot_t fault_type;
129 boolean_t change_wiring;
130{
131 vm_object_t first_object;
70 */
71
72/*
73 * Page fault handling module.
74 */
75
76#include <sys/param.h>
77#include <sys/systm.h>

--- 46 unchanged lines hidden (view full) ---

124int
125vm_fault(map, vaddr, fault_type, change_wiring)
126 vm_map_t map;
127 vm_offset_t vaddr;
128 vm_prot_t fault_type;
129 boolean_t change_wiring;
130{
131 vm_object_t first_object;
132 vm_offset_t first_offset;
132 vm_pindex_t first_pindex;
133 vm_map_entry_t entry;
134 register vm_object_t object;
133 vm_map_entry_t entry;
134 register vm_object_t object;
135 register vm_offset_t offset;
135 register vm_pindex_t pindex;
136 vm_page_t m;
137 vm_page_t first_m;
138 vm_prot_t prot;
139 int result;
140 boolean_t wired;
141 boolean_t su;
142 boolean_t lookup_still_valid;
143 vm_page_t old_m;

--- 43 unchanged lines hidden (view full) ---

187
188 /*
189 * Find the backing store object and offset into it to begin the
190 * search.
191 */
192
193 if ((result = vm_map_lookup(&map, vaddr,
194 fault_type, &entry, &first_object,
136 vm_page_t m;
137 vm_page_t first_m;
138 vm_prot_t prot;
139 int result;
140 boolean_t wired;
141 boolean_t su;
142 boolean_t lookup_still_valid;
143 vm_page_t old_m;

--- 43 unchanged lines hidden (view full) ---

187
188 /*
189 * Find the backing store object and offset into it to begin the
190 * search.
191 */
192
193 if ((result = vm_map_lookup(&map, vaddr,
194 fault_type, &entry, &first_object,
195 &first_offset, &prot, &wired, &su)) != KERN_SUCCESS) {
195 &first_pindex, &prot, &wired, &su)) != KERN_SUCCESS) {
196 return (result);
197 }
198
199 vp = vnode_pager_lock(first_object);
200
201 lookup_still_valid = TRUE;
202
203 if (wired)

--- 39 unchanged lines hidden (view full) ---

243 * the busy page without our noticing.
244 */
245
246 /*
247 * Search for the page at object/offset.
248 */
249
250 object = first_object;
196 return (result);
197 }
198
199 vp = vnode_pager_lock(first_object);
200
201 lookup_still_valid = TRUE;
202
203 if (wired)

--- 39 unchanged lines hidden (view full) ---

243 * the busy page without our noticing.
244 */
245
246 /*
247 * Search for the page at object/offset.
248 */
249
250 object = first_object;
251 offset = first_offset;
251 pindex = first_pindex;
252
253 /*
254 * See whether this page is resident
255 */
256
257 while (TRUE) {
252
253 /*
254 * See whether this page is resident
255 */
256
257 while (TRUE) {
258 m = vm_page_lookup(object, offset);
258 m = vm_page_lookup(object, pindex);
259 if (m != NULL) {
260 /*
261 * If the page is being brought in, wait for it and
262 * then retry.
263 */
264 if ((m->flags & PG_BUSY) || m->busy) {
265 int s;
266

--- 25 unchanged lines hidden (view full) ---

292 m->object != kernel_object && m->object != kmem_object) {
293 goto readrest;
294 }
295 break;
296 }
297 if (((object->type != OBJT_DEFAULT) && (!change_wiring || wired))
298 || (object == first_object)) {
299
259 if (m != NULL) {
260 /*
261 * If the page is being brought in, wait for it and
262 * then retry.
263 */
264 if ((m->flags & PG_BUSY) || m->busy) {
265 int s;
266

--- 25 unchanged lines hidden (view full) ---

292 m->object != kernel_object && m->object != kmem_object) {
293 goto readrest;
294 }
295 break;
296 }
297 if (((object->type != OBJT_DEFAULT) && (!change_wiring || wired))
298 || (object == first_object)) {
299
300 if (offset >= object->size) {
300 if (pindex >= object->size) {
301 UNLOCK_AND_DEALLOCATE;
302 return (KERN_PROTECTION_FAILURE);
303 }
304
305 /*
306 * Allocate a new page for this object/offset pair.
307 */
301 UNLOCK_AND_DEALLOCATE;
302 return (KERN_PROTECTION_FAILURE);
303 }
304
305 /*
306 * Allocate a new page for this object/offset pair.
307 */
308 m = vm_page_alloc(object, offset,
308 m = vm_page_alloc(object, pindex,
309 vp?VM_ALLOC_NORMAL:(VM_ALLOC_NORMAL|VM_ALLOC_ZERO));
310
311 if (m == NULL) {
312 UNLOCK_AND_DEALLOCATE;
313 VM_WAIT;
314 goto RetryFault;
315 }
316 }

--- 35 unchanged lines hidden (view full) ---

352 * with it.
353 */
354
355 /*
356 * Relookup in case pager changed page. Pager
357 * is responsible for disposition of old page
358 * if moved.
359 */
309 vp?VM_ALLOC_NORMAL:(VM_ALLOC_NORMAL|VM_ALLOC_ZERO));
310
311 if (m == NULL) {
312 UNLOCK_AND_DEALLOCATE;
313 VM_WAIT;
314 goto RetryFault;
315 }
316 }

--- 35 unchanged lines hidden (view full) ---

352 * with it.
353 */
354
355 /*
356 * Relookup in case pager changed page. Pager
357 * is responsible for disposition of old page
358 * if moved.
359 */
360 m = vm_page_lookup(object, offset);
360 m = vm_page_lookup(object, pindex);
361 if( !m) {
362 UNLOCK_AND_DEALLOCATE;
363 goto RetryFault;
364 }
365
366 hardfault++;
367 break;
368 }

--- 42 unchanged lines hidden (view full) ---

411 if (object == first_object)
412 first_m = m;
413
414 /*
415 * Move on to the next object. Lock the next object before
416 * unlocking the current one.
417 */
418
361 if( !m) {
362 UNLOCK_AND_DEALLOCATE;
363 goto RetryFault;
364 }
365
366 hardfault++;
367 break;
368 }

--- 42 unchanged lines hidden (view full) ---

411 if (object == first_object)
412 first_m = m;
413
414 /*
415 * Move on to the next object. Lock the next object before
416 * unlocking the current one.
417 */
418
419 offset += object->backing_object_offset;
419 pindex += OFF_TO_IDX(object->backing_object_offset);
420 next_object = object->backing_object;
421 if (next_object == NULL) {
422 /*
423 * If there's no object left, fill the page in the top
424 * object with zeros.
425 */
426 if (object != first_object) {
427 vm_object_pip_wakeup(object);
428
429 object = first_object;
420 next_object = object->backing_object;
421 if (next_object == NULL) {
422 /*
423 * If there's no object left, fill the page in the top
424 * object with zeros.
425 */
426 if (object != first_object) {
427 vm_object_pip_wakeup(object);
428
429 object = first_object;
430 offset = first_offset;
430 pindex = first_pindex;
431 m = first_m;
432 }
433 first_m = NULL;
434
435 if ((m->flags & PG_ZERO) == 0)
436 vm_page_zero_fill(m);
437 m->valid = VM_PAGE_BITS_ALL;
438 cnt.v_zfod++;

--- 77 unchanged lines hidden (view full) ---

516
517 /*
518 * Only use the new page below...
519 */
520
521 cnt.v_cow_faults++;
522 m = first_m;
523 object = first_object;
431 m = first_m;
432 }
433 first_m = NULL;
434
435 if ((m->flags & PG_ZERO) == 0)
436 vm_page_zero_fill(m);
437 m->valid = VM_PAGE_BITS_ALL;
438 cnt.v_zfod++;

--- 77 unchanged lines hidden (view full) ---

516
517 /*
518 * Only use the new page below...
519 */
520
521 cnt.v_cow_faults++;
522 m = first_m;
523 object = first_object;
524 offset = first_offset;
524 pindex = first_pindex;
525
526 /*
527 * Now that we've gotten the copy out of the way,
528 * let's try to collapse the top object.
529 *
530 * But we have to play ugly games with
531 * paging_in_progress to do that...
532 */

--- 7 unchanged lines hidden (view full) ---

540
541 /*
542 * We must verify that the maps have not changed since our last
543 * lookup.
544 */
545
546 if (!lookup_still_valid) {
547 vm_object_t retry_object;
525
526 /*
527 * Now that we've gotten the copy out of the way,
528 * let's try to collapse the top object.
529 *
530 * But we have to play ugly games with
531 * paging_in_progress to do that...
532 */

--- 7 unchanged lines hidden (view full) ---

540
541 /*
542 * We must verify that the maps have not changed since our last
543 * lookup.
544 */
545
546 if (!lookup_still_valid) {
547 vm_object_t retry_object;
548 vm_offset_t retry_offset;
548 vm_pindex_t retry_pindex;
549 vm_prot_t retry_prot;
550
551 /*
552 * Since map entries may be pageable, make sure we can take a
553 * page fault on them.
554 */
555
556 /*
557 * To avoid trying to write_lock the map while another process
558 * has it read_locked (in vm_map_pageable), we do not try for
559 * write permission. If the page is still writable, we will
560 * get write permission. If it is not, or has been marked
561 * needs_copy, we enter the mapping without write permission,
562 * and will merely take another fault.
563 */
564 result = vm_map_lookup(&map, vaddr, fault_type & ~VM_PROT_WRITE,
549 vm_prot_t retry_prot;
550
551 /*
552 * Since map entries may be pageable, make sure we can take a
553 * page fault on them.
554 */
555
556 /*
557 * To avoid trying to write_lock the map while another process
558 * has it read_locked (in vm_map_pageable), we do not try for
559 * write permission. If the page is still writable, we will
560 * get write permission. If it is not, or has been marked
561 * needs_copy, we enter the mapping without write permission,
562 * and will merely take another fault.
563 */
564 result = vm_map_lookup(&map, vaddr, fault_type & ~VM_PROT_WRITE,
565 &entry, &retry_object, &retry_offset, &retry_prot, &wired, &su);
565 &entry, &retry_object, &retry_pindex, &retry_prot, &wired, &su);
566
567 /*
568 * If we don't need the page any longer, put it on the active
569 * list (the easiest thing to do here). If no one needs it,
570 * pageout will grab it eventually.
571 */
572
573 if (result != KERN_SUCCESS) {
574 RELEASE_PAGE(m);
575 UNLOCK_AND_DEALLOCATE;
576 return (result);
577 }
578 lookup_still_valid = TRUE;
579
580 if ((retry_object != first_object) ||
566
567 /*
568 * If we don't need the page any longer, put it on the active
569 * list (the easiest thing to do here). If no one needs it,
570 * pageout will grab it eventually.
571 */
572
573 if (result != KERN_SUCCESS) {
574 RELEASE_PAGE(m);
575 UNLOCK_AND_DEALLOCATE;
576 return (result);
577 }
578 lookup_still_valid = TRUE;
579
580 if ((retry_object != first_object) ||
581 (retry_offset != first_offset)) {
581 (retry_pindex != first_pindex)) {
582 RELEASE_PAGE(m);
583 UNLOCK_AND_DEALLOCATE;
584 goto RetryFault;
585 }
586 /*
587 * Check whether the protection has changed or the object has
588 * been copied while we left the map unlocked. Changing from
589 * read to write permission is OK - we leave the page

--- 60 unchanged lines hidden (view full) ---

650 if (hardfault) {
651 curproc->p_stats->p_ru.ru_majflt++;
652 } else {
653 curproc->p_stats->p_ru.ru_minflt++;
654 }
655 }
656
657 if ((m->flags & PG_BUSY) == 0)
582 RELEASE_PAGE(m);
583 UNLOCK_AND_DEALLOCATE;
584 goto RetryFault;
585 }
586 /*
587 * Check whether the protection has changed or the object has
588 * been copied while we left the map unlocked. Changing from
589 * read to write permission is OK - we leave the page

--- 60 unchanged lines hidden (view full) ---

650 if (hardfault) {
651 curproc->p_stats->p_ru.ru_majflt++;
652 } else {
653 curproc->p_stats->p_ru.ru_minflt++;
654 }
655 }
656
657 if ((m->flags & PG_BUSY) == 0)
658 printf("page not busy: %d\n", m->offset);
658 printf("page not busy: %d\n", m->pindex);
659 /*
660 * Unlock everything, and return
661 */
662
663 PAGE_WAKEUP(m);
664 UNLOCK_AND_DEALLOCATE;
665
666 return (KERN_SUCCESS);

--- 101 unchanged lines hidden (view full) ---

768vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
769 vm_map_t dst_map;
770 vm_map_t src_map;
771 vm_map_entry_t dst_entry;
772 vm_map_entry_t src_entry;
773{
774 vm_object_t dst_object;
775 vm_object_t src_object;
659 /*
660 * Unlock everything, and return
661 */
662
663 PAGE_WAKEUP(m);
664 UNLOCK_AND_DEALLOCATE;
665
666 return (KERN_SUCCESS);

--- 101 unchanged lines hidden (view full) ---

768vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
769 vm_map_t dst_map;
770 vm_map_t src_map;
771 vm_map_entry_t dst_entry;
772 vm_map_entry_t src_entry;
773{
774 vm_object_t dst_object;
775 vm_object_t src_object;
776 vm_offset_t dst_offset;
777 vm_offset_t src_offset;
776 vm_ooffset_t dst_offset;
777 vm_ooffset_t src_offset;
778 vm_prot_t prot;
779 vm_offset_t vaddr;
780 vm_page_t dst_m;
781 vm_page_t src_m;
782
783#ifdef lint
784 src_map++;
785#endif /* lint */
786
787 src_object = src_entry->object.vm_object;
788 src_offset = src_entry->offset;
789
790 /*
791 * Create the top-level object for the destination entry. (Doesn't
792 * actually shadow anything - we copy the pages directly.)
793 */
794 dst_object = vm_object_allocate(OBJT_DEFAULT,
778 vm_prot_t prot;
779 vm_offset_t vaddr;
780 vm_page_t dst_m;
781 vm_page_t src_m;
782
783#ifdef lint
784 src_map++;
785#endif /* lint */
786
787 src_object = src_entry->object.vm_object;
788 src_offset = src_entry->offset;
789
790 /*
791 * Create the top-level object for the destination entry. (Doesn't
792 * actually shadow anything - we copy the pages directly.)
793 */
794 dst_object = vm_object_allocate(OBJT_DEFAULT,
795 (vm_size_t) (dst_entry->end - dst_entry->start));
795 (vm_size_t) OFF_TO_IDX(dst_entry->end - dst_entry->start));
796
797 dst_entry->object.vm_object = dst_object;
798 dst_entry->offset = 0;
799
800 prot = dst_entry->max_protection;
801
802 /*
803 * Loop through all of the pages in the entry's range, copying each
804 * one from the source object (it should be there) to the destination
805 * object.
806 */
807 for (vaddr = dst_entry->start, dst_offset = 0;
808 vaddr < dst_entry->end;
809 vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
810
811 /*
812 * Allocate a page in the destination object
813 */
814 do {
796
797 dst_entry->object.vm_object = dst_object;
798 dst_entry->offset = 0;
799
800 prot = dst_entry->max_protection;
801
802 /*
803 * Loop through all of the pages in the entry's range, copying each
804 * one from the source object (it should be there) to the destination
805 * object.
806 */
807 for (vaddr = dst_entry->start, dst_offset = 0;
808 vaddr < dst_entry->end;
809 vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
810
811 /*
812 * Allocate a page in the destination object
813 */
814 do {
815 dst_m = vm_page_alloc(dst_object, dst_offset, VM_ALLOC_NORMAL);
815 dst_m = vm_page_alloc(dst_object,
816 OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
816 if (dst_m == NULL) {
817 VM_WAIT;
818 }
819 } while (dst_m == NULL);
820
821 /*
822 * Find the page in the source object, and copy it in.
823 * (Because the source is wired down, the page will be in
824 * memory.)
825 */
817 if (dst_m == NULL) {
818 VM_WAIT;
819 }
820 } while (dst_m == NULL);
821
822 /*
823 * Find the page in the source object, and copy it in.
824 * (Because the source is wired down, the page will be in
825 * memory.)
826 */
826 src_m = vm_page_lookup(src_object, dst_offset + src_offset);
827 src_m = vm_page_lookup(src_object,
828 OFF_TO_IDX(dst_offset + src_offset));
827 if (src_m == NULL)
828 panic("vm_fault_copy_wired: page missing");
829
830 vm_page_copy(src_m, dst_m);
831
832 /*
833 * Enter it in the pmap...
834 */

--- 31 unchanged lines hidden (view full) ---

866 vm_page_t m;
867 int rbehind;
868 int rahead;
869 vm_page_t *marray;
870 int *reqpage;
871{
872 int i;
873 vm_object_t object;
829 if (src_m == NULL)
830 panic("vm_fault_copy_wired: page missing");
831
832 vm_page_copy(src_m, dst_m);
833
834 /*
835 * Enter it in the pmap...
836 */

--- 31 unchanged lines hidden (view full) ---

868 vm_page_t m;
869 int rbehind;
870 int rahead;
871 vm_page_t *marray;
872 int *reqpage;
873{
874 int i;
875 vm_object_t object;
874 vm_offset_t offset, startoffset, endoffset, toffset, size;
876 vm_pindex_t pindex, startpindex, endpindex, tpindex;
877 vm_offset_t size;
875 vm_page_t rtm;
876 int treqpage;
877 int cbehind, cahead;
878
879 object = m->object;
878 vm_page_t rtm;
879 int treqpage;
880 int cbehind, cahead;
881
882 object = m->object;
880 offset = m->offset;
883 pindex = m->pindex;
881
882 /*
883 * if the requested page is not available, then give up now
884 */
885
886 if (!vm_pager_has_page(object,
884
885 /*
886 * if the requested page is not available, then give up now
887 */
888
889 if (!vm_pager_has_page(object,
887 object->paging_offset + offset, &cbehind, &cahead))
890 OFF_TO_IDX(object->paging_offset) + pindex, &cbehind, &cahead))
888 return 0;
889
890 if ((cbehind == 0) && (cahead == 0)) {
891 *reqpage = 0;
892 marray[0] = m;
893 return 1;
894 }
895

--- 15 unchanged lines hidden (view full) ---

911 marray[0] = m;
912 return 1;
913 }
914
915 /*
916 * scan backward for the read behind pages -- in memory or on disk not
917 * in same object
918 */
891 return 0;
892
893 if ((cbehind == 0) && (cahead == 0)) {
894 *reqpage = 0;
895 marray[0] = m;
896 return 1;
897 }
898

--- 15 unchanged lines hidden (view full) ---

914 marray[0] = m;
915 return 1;
916 }
917
918 /*
919 * scan backward for the read behind pages -- in memory or on disk not
920 * in same object
921 */
919 toffset = offset - PAGE_SIZE;
920 if (toffset < offset) {
921 if (rbehind * PAGE_SIZE > offset)
922 rbehind = offset / PAGE_SIZE;
923 startoffset = offset - rbehind * PAGE_SIZE;
924 while (toffset >= startoffset) {
925 if (vm_page_lookup( object, toffset)) {
926 startoffset = toffset + PAGE_SIZE;
922 tpindex = pindex - 1;
923 if (tpindex < pindex) {
924 if (rbehind > pindex)
925 rbehind = pindex;
926 startpindex = pindex - rbehind;
927 while (tpindex >= startpindex) {
928 if (vm_page_lookup( object, tpindex)) {
929 startpindex = tpindex + 1;
927 break;
928 }
930 break;
931 }
929 if (toffset == 0)
932 if (tpindex == 0)
930 break;
933 break;
931 toffset -= PAGE_SIZE;
934 tpindex -= 1;
932 }
933 } else {
935 }
936 } else {
934 startoffset = offset;
937 startpindex = pindex;
935 }
936
937 /*
938 * scan forward for the read ahead pages -- in memory or on disk not
939 * in same object
940 */
938 }
939
940 /*
941 * scan forward for the read ahead pages -- in memory or on disk not
942 * in same object
943 */
941 toffset = offset + PAGE_SIZE;
942 endoffset = offset + (rahead + 1) * PAGE_SIZE;
943 if (endoffset > object->size)
944 endoffset = object->size;
945 while (toffset < endoffset) {
946 if ( vm_page_lookup(object, toffset)) {
944 tpindex = pindex + 1;
945 endpindex = pindex + (rahead + 1);
946 if (endpindex > object->size)
947 endpindex = object->size;
948 while (tpindex < endpindex) {
949 if ( vm_page_lookup(object, tpindex)) {
947 break;
948 }
950 break;
951 }
949 toffset += PAGE_SIZE;
952 tpindex += 1;
950 }
953 }
951 endoffset = toffset;
954 endpindex = tpindex;
952
953 /* calculate number of bytes of pages */
955
956 /* calculate number of bytes of pages */
954 size = (endoffset - startoffset) / PAGE_SIZE;
957 size = endpindex - startpindex;
955
956 /* calculate the page offset of the required page */
958
959 /* calculate the page offset of the required page */
957 treqpage = (offset - startoffset) / PAGE_SIZE;
960 treqpage = pindex - startpindex;
958
959 /* see if we have space (again) */
960 if ((cnt.v_free_count + cnt.v_cache_count) >
961 (cnt.v_free_reserved + size)) {
962 /*
963 * get our pages and don't block for them
964 */
965 for (i = 0; i < size; i++) {
966 if (i != treqpage) {
967 rtm = vm_page_alloc(object,
961
962 /* see if we have space (again) */
963 if ((cnt.v_free_count + cnt.v_cache_count) >
964 (cnt.v_free_reserved + size)) {
965 /*
966 * get our pages and don't block for them
967 */
968 for (i = 0; i < size; i++) {
969 if (i != treqpage) {
970 rtm = vm_page_alloc(object,
968 startoffset + i * PAGE_SIZE,
971 startpindex + i,
969 VM_ALLOC_NORMAL);
970 if (rtm == NULL) {
971 if (i < treqpage) {
972 int j;
973 for (j = 0; j < i; j++) {
974 FREE_PAGE(marray[j]);
975 }
976 *reqpage = 0;

--- 21 unchanged lines hidden --- 		972 VM_ALLOC_NORMAL);
973 if (rtm == NULL) {
974 if (i < treqpage) {
975 int j;
976 for (j = 0; j < i; j++) {
977 FREE_PAGE(marray[j]);
978 }
979 *reqpage = 0;

--- 21 unchanged lines hidden ---