164static void swap_pager_reclaim __P((void)); 165static void swap_pager_ridpages __P((vm_page_t *m, int count, 166 int reqpage)); 167static void swap_pager_setvalid __P((vm_object_t object, 168 vm_offset_t offset, int valid)); 169static void swapsizecheck __P((void)); 170 171static inline void 172swapsizecheck() 173{ 174 if (vm_swap_size < 128 * btodb(PAGE_SIZE)) { 175 if (swap_pager_full == 0) 176 printf("swap_pager: out of space\n"); 177 swap_pager_full = 1; 178 } else if (vm_swap_size > 192 * btodb(PAGE_SIZE)) 179 swap_pager_full = 0; 180} 181 182static void 183swap_pager_init() 184{ 185 TAILQ_INIT(&swap_pager_object_list); 186 TAILQ_INIT(&swap_pager_un_object_list); 187 188 /* 189 * Initialize clean lists 190 */ 191 TAILQ_INIT(&swap_pager_inuse); 192 TAILQ_INIT(&swap_pager_done); 193 TAILQ_INIT(&swap_pager_free); 194 195 /* 196 * Calculate the swap allocation constants. 197 */ 198 dmmin = CLBYTES / DEV_BSIZE; 199 dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2; 200} 201 202void 203swap_pager_swap_init() 204{ 205 swp_clean_t spc; 206 struct buf *bp; 207 int i; 208 209 /* 210 * kva's are allocated here so that we dont need to keep doing 211 * kmem_alloc pageables at runtime 212 */ 213 for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) { 214 spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER); 215 if (!spc->spc_kva) { 216 break; 217 } 218 spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL); 219 if (!spc->spc_bp) { 220 kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 221 break; 222 } 223 spc->spc_flags = 0; 224 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 225 } 226} 227 228int 229swap_pager_swp_alloc(object, wait) 230 vm_object_t object; 231 int wait; 232{ 233 sw_blk_t swb; 234 int nblocks; 235 int i, j; 236 237 nblocks = (object->size + SWB_NPAGES - 1) / SWB_NPAGES; 238 swb = malloc(nblocks * sizeof(*swb), M_VMPGDATA, wait); 239 if (swb == NULL) 240 return 1; 241 242 for (i = 0; i < nblocks; i++) { 243 swb[i].swb_valid = 0; 244 swb[i].swb_locked = 0; 245 for (j = 0; j < SWB_NPAGES; j++) 246 swb[i].swb_block[j] = SWB_EMPTY; 247 } 248 249 object->un_pager.swp.swp_nblocks = nblocks; 250 object->un_pager.swp.swp_allocsize = 0; 251 object->un_pager.swp.swp_blocks = swb; 252 object->un_pager.swp.swp_poip = 0; 253 254 if (object->handle != NULL) { 255 TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list); 256 } else { 257 TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 258 } 259 260 return 0; 261} 262 263/* 264 * Allocate an object and associated resources. 265 * Note that if we are called from the pageout daemon (handle == NULL) 266 * we should not wait for memory as it could resulting in deadlock. 267 */ 268static vm_object_t 269swap_pager_alloc(handle, size, prot, offset) 270 void *handle; 271 register vm_size_t size; 272 vm_prot_t prot; 273 vm_ooffset_t offset; 274{ 275 vm_object_t object; 276 277 /* 278 * If this is a "named" anonymous region, look it up and use the 279 * object if it exists, otherwise allocate a new one. 280 */ 281 if (handle) { 282 object = vm_pager_object_lookup(&swap_pager_object_list, handle); 283 if (object != NULL) { 284 vm_object_reference(object); 285 } else { 286 /* 287 * XXX - there is a race condition here. Two processes 288 * can request the same named object simultaneuously, 289 * and if one blocks for memory, the result is a disaster. 290 * Probably quite rare, but is yet another reason to just 291 * rip support of "named anonymous regions" out altogether. 292 */ 293 object = vm_object_allocate(OBJT_SWAP, 294 OFF_TO_IDX(offset+ PAGE_SIZE - 1 + size)); 295 object->handle = handle; 296 (void) swap_pager_swp_alloc(object, M_WAITOK); 297 } 298 } else { 299 object = vm_object_allocate(OBJT_SWAP, 300 OFF_TO_IDX(offset + PAGE_SIZE - 1 + size)); 301 (void) swap_pager_swp_alloc(object, M_WAITOK); 302 } 303 304 return (object); 305} 306 307/* 308 * returns disk block associated with pager and offset 309 * additionally, as a side effect returns a flag indicating 310 * if the block has been written 311 */ 312 313inline static daddr_t * 314swap_pager_diskaddr(object, pindex, valid) 315 vm_object_t object; 316 vm_pindex_t pindex; 317 int *valid; 318{ 319 register sw_blk_t swb; 320 int ix; 321 322 if (valid) 323 *valid = 0; 324 ix = pindex / SWB_NPAGES; 325 if ((ix >= object->un_pager.swp.swp_nblocks) || 326 (pindex >= object->size)) { 327 return (FALSE); 328 } 329 swb = &object->un_pager.swp.swp_blocks[ix]; 330 ix = pindex % SWB_NPAGES; 331 if (valid) 332 *valid = swb->swb_valid & (1 << ix); 333 return &swb->swb_block[ix]; 334} 335 336/* 337 * Utility routine to set the valid (written) bit for 338 * a block associated with a pager and offset 339 */ 340static void 341swap_pager_setvalid(object, offset, valid) 342 vm_object_t object; 343 vm_offset_t offset; 344 int valid; 345{ 346 register sw_blk_t swb; 347 int ix; 348 349 ix = offset / SWB_NPAGES; 350 if (ix >= object->un_pager.swp.swp_nblocks) 351 return; 352 353 swb = &object->un_pager.swp.swp_blocks[ix]; 354 ix = offset % SWB_NPAGES; 355 if (valid) 356 swb->swb_valid |= (1 << ix); 357 else 358 swb->swb_valid &= ~(1 << ix); 359 return; 360} 361 362/* 363 * this routine allocates swap space with a fragmentation 364 * minimization policy. 365 */ 366static int 367swap_pager_getswapspace(object, amount, rtval) 368 vm_object_t object; 369 unsigned int amount; 370 daddr_t *rtval; 371{ 372 unsigned location; 373 vm_swap_size -= amount; 374 if (!rlist_alloc(&swaplist, amount, &location)) { 375 vm_swap_size += amount; 376 return 0; 377 } else { 378 swapsizecheck(); 379 object->un_pager.swp.swp_allocsize += amount; 380 *rtval = location; 381 return 1; 382 } 383} 384 385/* 386 * this routine frees swap space with a fragmentation 387 * minimization policy. 388 */ 389static void 390swap_pager_freeswapspace(object, from, to) 391 vm_object_t object; 392 unsigned int from; 393 unsigned int to; 394{ 395 rlist_free(&swaplist, from, to); 396 vm_swap_size += (to - from) + 1; 397 object->un_pager.swp.swp_allocsize -= (to - from) + 1; 398 swapsizecheck(); 399} 400/* 401 * this routine frees swap blocks from a specified pager 402 */ 403void 404swap_pager_freespace(object, start, size) 405 vm_object_t object; 406 vm_pindex_t start; 407 vm_size_t size; 408{ 409 vm_pindex_t i; 410 int s; 411 412 s = splbio(); 413 for (i = start; i < start + size; i += 1) { 414 int valid; 415 daddr_t *addr = swap_pager_diskaddr(object, i, &valid); 416 417 if (addr && *addr != SWB_EMPTY) { 418 swap_pager_freeswapspace(object, *addr, *addr + btodb(PAGE_SIZE) - 1); 419 if (valid) { 420 swap_pager_setvalid(object, i, 0); 421 } 422 *addr = SWB_EMPTY; 423 } 424 } 425 splx(s); 426} 427 428static void 429swap_pager_free_swap(object) 430 vm_object_t object; 431{ 432 register int i, j; 433 register sw_blk_t swb; 434 int first_block=0, block_count=0; 435 int s; 436 /* 437 * Free left over swap blocks 438 */ 439 s = splbio(); 440 for (i = 0, swb = object->un_pager.swp.swp_blocks; 441 i < object->un_pager.swp.swp_nblocks; i++, swb++) { 442 for (j = 0; j < SWB_NPAGES; j++) { 443 if (swb->swb_block[j] != SWB_EMPTY) { 444 /* 445 * initially the length of the run is zero 446 */ 447 if (block_count == 0) { 448 first_block = swb->swb_block[j]; 449 block_count = btodb(PAGE_SIZE); 450 swb->swb_block[j] = SWB_EMPTY; 451 /* 452 * if the new block can be included into the current run 453 */ 454 } else if (swb->swb_block[j] == first_block + block_count) { 455 block_count += btodb(PAGE_SIZE); 456 swb->swb_block[j] = SWB_EMPTY; 457 /* 458 * terminate the previous run, and start a new one 459 */ 460 } else { 461 swap_pager_freeswapspace(object, first_block, 462 (unsigned) first_block + block_count - 1); 463 first_block = swb->swb_block[j]; 464 block_count = btodb(PAGE_SIZE); 465 swb->swb_block[j] = SWB_EMPTY; 466 } 467 } 468 } 469 } 470 471 if (block_count) { 472 swap_pager_freeswapspace(object, first_block, 473 (unsigned) first_block + block_count - 1); 474 } 475 splx(s); 476} 477 478 479/* 480 * swap_pager_reclaim frees up over-allocated space from all pagers 481 * this eliminates internal fragmentation due to allocation of space 482 * for segments that are never swapped to. It has been written so that 483 * it does not block until the rlist_free operation occurs; it keeps 484 * the queues consistant. 485 */ 486 487/* 488 * Maximum number of blocks (pages) to reclaim per pass 489 */ 490#define MAXRECLAIM 128 491 492static void 493swap_pager_reclaim() 494{ 495 vm_object_t object; 496 int i, j, k; 497 int s; 498 int reclaimcount; 499 static struct { 500 int address; 501 vm_object_t object; 502 } reclaims[MAXRECLAIM]; 503 static int in_reclaim; 504 505 /* 506 * allow only one process to be in the swap_pager_reclaim subroutine 507 */ 508 s = splbio(); 509 if (in_reclaim) { 510 tsleep(&in_reclaim, PSWP, "swrclm", 0); 511 splx(s); 512 return; 513 } 514 in_reclaim = 1; 515 reclaimcount = 0; 516 517 /* for each pager queue */ 518 for (k = 0; swp_qs[k]; k++) { 519 520 object = swp_qs[k]->tqh_first; 521 while (object && (reclaimcount < MAXRECLAIM)) { 522 523 /* 524 * see if any blocks associated with a pager has been 525 * allocated but not used (written) 526 */ 527 if (object->paging_in_progress == 0) { 528 for (i = 0; i < object->un_pager.swp.swp_nblocks; i++) { 529 sw_blk_t swb = &object->un_pager.swp.swp_blocks[i]; 530 531 if (swb->swb_locked) 532 continue; 533 for (j = 0; j < SWB_NPAGES; j++) { 534 if (swb->swb_block[j] != SWB_EMPTY && 535 (swb->swb_valid & (1 << j)) == 0) { 536 reclaims[reclaimcount].address = swb->swb_block[j]; 537 reclaims[reclaimcount++].object = object; 538 swb->swb_block[j] = SWB_EMPTY; 539 if (reclaimcount >= MAXRECLAIM) 540 goto rfinished; 541 } 542 } 543 } 544 } 545 object = object->pager_object_list.tqe_next; 546 } 547 } 548 549rfinished: 550 551 /* 552 * free the blocks that have been added to the reclaim list 553 */ 554 for (i = 0; i < reclaimcount; i++) { 555 swap_pager_freeswapspace(reclaims[i].object, 556 reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1); 557 } 558 splx(s); 559 in_reclaim = 0; 560 wakeup(&in_reclaim); 561} 562 563 564/* 565 * swap_pager_copy copies blocks from one pager to another and 566 * destroys the source pager 567 */ 568 569void 570swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset) 571 vm_object_t srcobject; 572 vm_pindex_t srcoffset; 573 vm_object_t dstobject; 574 vm_pindex_t dstoffset; 575 vm_pindex_t offset; 576{ 577 vm_pindex_t i; 578 int origsize; 579 int s; 580 581 if (vm_swap_size) 582 no_swap_space = 0; 583 584 origsize = srcobject->un_pager.swp.swp_allocsize; 585 586 /* 587 * remove the source object from the swap_pager internal queue 588 */ 589 if (srcobject->handle == NULL) { 590 TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list); 591 } else { 592 TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list); 593 } 594 595 s = splbio(); 596 while (srcobject->un_pager.swp.swp_poip) { 597 tsleep(srcobject, PVM, "spgout", 0); 598 } 599 splx(s); 600 601 /* 602 * clean all of the pages that are currently active and finished 603 */ 604 swap_pager_sync(); 605 606 s = splbio(); 607 /* 608 * transfer source to destination 609 */ 610 for (i = 0; i < dstobject->size; i += 1) { 611 int srcvalid, dstvalid; 612 daddr_t *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset, 613 &srcvalid); 614 daddr_t *dstaddrp; 615 616 /* 617 * see if the source has space allocated 618 */ 619 if (srcaddrp && *srcaddrp != SWB_EMPTY) { 620 /* 621 * if the source is valid and the dest has no space, 622 * then copy the allocation from the srouce to the 623 * dest. 624 */ 625 if (srcvalid) { 626 dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset, 627 &dstvalid); 628 /* 629 * if the dest already has a valid block, 630 * deallocate the source block without 631 * copying. 632 */ 633 if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 634 swap_pager_freeswapspace(dstobject, *dstaddrp, 635 *dstaddrp + btodb(PAGE_SIZE) - 1); 636 *dstaddrp = SWB_EMPTY; 637 } 638 if (dstaddrp && *dstaddrp == SWB_EMPTY) { 639 *dstaddrp = *srcaddrp; 640 *srcaddrp = SWB_EMPTY; 641 dstobject->un_pager.swp.swp_allocsize += btodb(PAGE_SIZE); 642 srcobject->un_pager.swp.swp_allocsize -= btodb(PAGE_SIZE); 643 swap_pager_setvalid(dstobject, i + dstoffset, 1); 644 } 645 } 646 /* 647 * if the source is not empty at this point, then 648 * deallocate the space. 649 */ 650 if (*srcaddrp != SWB_EMPTY) { 651 swap_pager_freeswapspace(srcobject, *srcaddrp, 652 *srcaddrp + btodb(PAGE_SIZE) - 1); 653 *srcaddrp = SWB_EMPTY; 654 } 655 } 656 } 657 splx(s); 658 659 /* 660 * Free left over swap blocks 661 */ 662 swap_pager_free_swap(srcobject); 663 664 if (srcobject->un_pager.swp.swp_allocsize) { 665 printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n", 666 srcobject->un_pager.swp.swp_allocsize, origsize); 667 } 668 669 free(srcobject->un_pager.swp.swp_blocks, M_VMPGDATA); 670 srcobject->un_pager.swp.swp_blocks = NULL; 671 672 return; 673} 674 675static void 676swap_pager_dealloc(object) 677 vm_object_t object; 678{ 679 int s; 680 681 /* 682 * Remove from list right away so lookups will fail if we block for 683 * pageout completion. 684 */ 685 if (object->handle == NULL) { 686 TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list); 687 } else { 688 TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list); 689 } 690 691 /* 692 * Wait for all pageouts to finish and remove all entries from 693 * cleaning list. 694 */ 695 696 s = splbio(); 697 while (object->un_pager.swp.swp_poip) { 698 tsleep(object, PVM, "swpout", 0); 699 } 700 splx(s); 701 702 703 swap_pager_sync(); 704 705 /* 706 * Free left over swap blocks 707 */ 708 swap_pager_free_swap(object); 709 710 if (object->un_pager.swp.swp_allocsize) { 711 printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n", 712 object->un_pager.swp.swp_allocsize); 713 } 714 /* 715 * Free swap management resources 716 */ 717 free(object->un_pager.swp.swp_blocks, M_VMPGDATA); 718 object->un_pager.swp.swp_blocks = NULL; 719} 720 721static inline __pure int 722swap_pager_block_index(pindex) 723 vm_pindex_t pindex; 724{ 725 return (pindex / SWB_NPAGES); 726} 727 728static inline __pure int 729swap_pager_block_offset(pindex) 730 vm_pindex_t pindex; 731{ 732 return (pindex % SWB_NPAGES); 733} 734 735/* 736 * swap_pager_haspage returns TRUE if the pager has data that has 737 * been written out. 738 */ 739static boolean_t 740swap_pager_haspage(object, pindex, before, after) 741 vm_object_t object; 742 vm_pindex_t pindex; 743 int *before; 744 int *after; 745{ 746 register sw_blk_t swb; 747 int ix; 748 749 if (before != NULL) 750 *before = 0; 751 if (after != NULL) 752 *after = 0; 753 ix = pindex / SWB_NPAGES; 754 if (ix >= object->un_pager.swp.swp_nblocks) { 755 return (FALSE); 756 } 757 swb = &object->un_pager.swp.swp_blocks[ix]; 758 ix = pindex % SWB_NPAGES; 759 760 if (swb->swb_block[ix] != SWB_EMPTY) { 761 762 if (swb->swb_valid & (1 << ix)) { 763 int tix; 764 if (before) { 765 for(tix = ix - 1; tix >= 0; --tix) { 766 if ((swb->swb_valid & (1 << tix)) == 0) 767 break; 768 if ((swb->swb_block[tix] + 769 (ix - tix) * (PAGE_SIZE/DEV_BSIZE)) != 770 swb->swb_block[ix]) 771 break; 772 (*before)++; 773 } 774 } 775 776 if (after) { 777 for(tix = ix + 1; tix < SWB_NPAGES; tix++) { 778 if ((swb->swb_valid & (1 << tix)) == 0) 779 break; 780 if ((swb->swb_block[tix] - 781 (tix - ix) * (PAGE_SIZE/DEV_BSIZE)) != 782 swb->swb_block[ix]) 783 break; 784 (*after)++; 785 } 786 } 787 788 return TRUE; 789 } 790 } 791 return (FALSE); 792} 793 794/* 795 * swap_pager_freepage is a convienience routine that clears the busy 796 * bit and deallocates a page. 797 */ 798static void 799swap_pager_freepage(m) 800 vm_page_t m; 801{ 802 PAGE_WAKEUP(m); 803 vm_page_free(m); 804} 805 806/* 807 * swap_pager_ridpages is a convienience routine that deallocates all 808 * but the required page. this is usually used in error returns that 809 * need to invalidate the "extra" readahead pages. 810 */ 811static void 812swap_pager_ridpages(m, count, reqpage) 813 vm_page_t *m; 814 int count; 815 int reqpage; 816{ 817 int i; 818 819 for (i = 0; i < count; i++) 820 if (i != reqpage) 821 swap_pager_freepage(m[i]); 822} 823 824/* 825 * swap_pager_iodone1 is the completion routine for both reads and async writes 826 */ 827static void 828swap_pager_iodone1(bp) 829 struct buf *bp; 830{ 831 bp->b_flags |= B_DONE; 832 bp->b_flags &= ~B_ASYNC; 833 wakeup(bp); 834} 835 836int 837swap_pager_getpages(object, m, count, reqpage) 838 vm_object_t object; 839 vm_page_t *m; 840 int count, reqpage; 841{ 842 register struct buf *bp; 843 sw_blk_t swb[count]; 844 register int s; 845 int i; 846 boolean_t rv; 847 vm_offset_t kva, off[count]; 848 swp_clean_t spc; 849 vm_pindex_t paging_offset; 850 int reqaddr[count]; 851 int sequential; 852 853 int first, last; 854 int failed; 855 int reqdskregion; 856 857 object = m[reqpage]->object; 858 paging_offset = OFF_TO_IDX(object->paging_offset); 859 sequential = (m[reqpage]->pindex == (object->last_read + 1)); 860 861 for (i = 0; i < count; i++) { 862 vm_pindex_t fidx = m[i]->pindex + paging_offset; 863 int ix = swap_pager_block_index(fidx); 864 865 if (ix >= object->un_pager.swp.swp_nblocks) { 866 int j; 867 868 if (i <= reqpage) { 869 swap_pager_ridpages(m, count, reqpage); 870 return (VM_PAGER_FAIL); 871 } 872 for (j = i; j < count; j++) { 873 swap_pager_freepage(m[j]); 874 } 875 count = i; 876 break; 877 } 878 swb[i] = &object->un_pager.swp.swp_blocks[ix]; 879 off[i] = swap_pager_block_offset(fidx); 880 reqaddr[i] = swb[i]->swb_block[off[i]]; 881 } 882 883 /* make sure that our required input request is existant */ 884 885 if (reqaddr[reqpage] == SWB_EMPTY || 886 (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 887 swap_pager_ridpages(m, count, reqpage); 888 return (VM_PAGER_FAIL); 889 } 890 reqdskregion = reqaddr[reqpage] / dmmax; 891 892 /* 893 * search backwards for the first contiguous page to transfer 894 */ 895 failed = 0; 896 first = 0; 897 for (i = reqpage - 1; i >= 0; --i) { 898 if (sequential || failed || (reqaddr[i] == SWB_EMPTY) || 899 (swb[i]->swb_valid & (1 << off[i])) == 0 || 900 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 901 ((reqaddr[i] / dmmax) != reqdskregion)) { 902 failed = 1; 903 swap_pager_freepage(m[i]); 904 if (first == 0) 905 first = i + 1; 906 } 907 } 908 /* 909 * search forwards for the last contiguous page to transfer 910 */ 911 failed = 0; 912 last = count; 913 for (i = reqpage + 1; i < count; i++) { 914 if (failed || (reqaddr[i] == SWB_EMPTY) || 915 (swb[i]->swb_valid & (1 << off[i])) == 0 || 916 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 917 ((reqaddr[i] / dmmax) != reqdskregion)) { 918 failed = 1; 919 swap_pager_freepage(m[i]); 920 if (last == count) 921 last = i; 922 } 923 } 924 925 count = last; 926 if (first != 0) { 927 for (i = first; i < count; i++) { 928 m[i - first] = m[i]; 929 reqaddr[i - first] = reqaddr[i]; 930 off[i - first] = off[i]; 931 } 932 count -= first; 933 reqpage -= first; 934 } 935 ++swb[reqpage]->swb_locked; 936 937 /* 938 * at this point: "m" is a pointer to the array of vm_page_t for 939 * paging I/O "count" is the number of vm_page_t entries represented 940 * by "m" "object" is the vm_object_t for I/O "reqpage" is the index 941 * into "m" for the page actually faulted 942 */ 943 944 spc = NULL; /* we might not use an spc data structure */ 945 946 if ((count == 1) && (swap_pager_free.tqh_first != NULL)) { 947 spc = swap_pager_free.tqh_first; 948 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 949 kva = spc->spc_kva; 950 bp = spc->spc_bp; 951 bzero(bp, sizeof *bp); 952 bp->b_spc = spc; 953 bp->b_vnbufs.le_next = NOLIST; 954 } else { 955 /* 956 * Get a swap buffer header to perform the IO 957 */ 958 bp = getpbuf(); 959 kva = (vm_offset_t) bp->b_data; 960 } 961 962 /* 963 * map our page(s) into kva for input 964 */ 965 pmap_qenter(kva, m, count); 966 967 bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING; 968 bp->b_iodone = swap_pager_iodone1; 969 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 970 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 971 crhold(bp->b_rcred); 972 crhold(bp->b_wcred); 973 bp->b_un.b_addr = (caddr_t) kva; 974 bp->b_blkno = reqaddr[0]; 975 bp->b_bcount = PAGE_SIZE * count; 976 bp->b_bufsize = PAGE_SIZE * count; 977 978 pbgetvp(swapdev_vp, bp); 979 980 cnt.v_swapin++; 981 cnt.v_swappgsin += count; 982 /* 983 * perform the I/O 984 */ 985 VOP_STRATEGY(bp); 986 987 /* 988 * wait for the sync I/O to complete 989 */ 990 s = splbio(); 991 while ((bp->b_flags & B_DONE) == 0) { 992 tsleep(bp, PVM, "swread", 0); 993 } 994 995 if (bp->b_flags & B_ERROR) { 996 printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 997 bp->b_blkno, bp->b_bcount, bp->b_error); 998 rv = VM_PAGER_ERROR; 999 } else { 1000 rv = VM_PAGER_OK; 1001 } 1002 1003 /* 1004 * relpbuf does this, but we maintain our own buffer list also... 1005 */ 1006 if (bp->b_vp) 1007 pbrelvp(bp); 1008 1009 splx(s); 1010 swb[reqpage]->swb_locked--; 1011 1012 /* 1013 * remove the mapping for kernel virtual 1014 */ 1015 pmap_qremove(kva, count); 1016 1017 if (spc) { 1018 m[reqpage]->object->last_read = m[reqpage]->pindex; 1019 if (bp->b_flags & B_WANTED) 1020 wakeup(bp); 1021 /* 1022 * if we have used an spc, we need to free it. 1023 */ 1024 if (bp->b_rcred != NOCRED) 1025 crfree(bp->b_rcred); 1026 if (bp->b_wcred != NOCRED) 1027 crfree(bp->b_wcred); 1028 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1029 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1030 wakeup(&swap_pager_free); 1031 } 1032 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1033 pagedaemon_wakeup(); 1034 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1035 } else { 1036 /* 1037 * release the physical I/O buffer 1038 */ 1039 relpbuf(bp); 1040 /* 1041 * finish up input if everything is ok 1042 */ 1043 if (rv == VM_PAGER_OK) { 1044 for (i = 0; i < count; i++) { 1045 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1046 m[i]->dirty = 0; 1047 m[i]->flags &= ~PG_ZERO; 1048 if (i != reqpage) { 1049 /* 1050 * whether or not to leave the page 1051 * activated is up in the air, but we 1052 * should put the page on a page queue 1053 * somewhere. (it already is in the 1054 * object). After some emperical 1055 * results, it is best to deactivate 1056 * the readahead pages. 1057 */ 1058 vm_page_deactivate(m[i]); 1059 1060 /* 1061 * just in case someone was asking for 1062 * this page we now tell them that it 1063 * is ok to use 1064 */ 1065 m[i]->valid = VM_PAGE_BITS_ALL; 1066 PAGE_WAKEUP(m[i]); 1067 } 1068 } 1069 1070 m[reqpage]->object->last_read = m[count-1]->pindex; 1071 1072 /* 1073 * If we're out of swap space, then attempt to free 1074 * some whenever pages are brought in. We must clear 1075 * the clean flag so that the page contents will be 1076 * preserved. 1077 */ 1078 if (swap_pager_full) { 1079 for (i = 0; i < count; i++) { 1080 m[i]->dirty = VM_PAGE_BITS_ALL; 1081 } 1082 swap_pager_freespace(object, m[0]->pindex + paging_offset, count); 1083 } 1084 } else { 1085 swap_pager_ridpages(m, count, reqpage); 1086 } 1087 } 1088 if (rv == VM_PAGER_OK) { 1089 pmap_clear_modify(VM_PAGE_TO_PHYS(m[reqpage])); 1090 m[reqpage]->valid = VM_PAGE_BITS_ALL; 1091 m[reqpage]->dirty = 0; 1092 } 1093 return (rv); 1094} 1095 1096int 1097swap_pager_putpages(object, m, count, sync, rtvals) 1098 vm_object_t object; 1099 vm_page_t *m; 1100 int count; 1101 boolean_t sync; 1102 int *rtvals; 1103{ 1104 register struct buf *bp; 1105 sw_blk_t swb[count]; 1106 register int s; 1107 int i, j, ix; 1108 boolean_t rv; 1109 vm_offset_t kva, off, fidx; 1110 swp_clean_t spc; 1111 vm_pindex_t paging_pindex; 1112 int reqaddr[count]; 1113 int failed; 1114 1115 if (vm_swap_size) 1116 no_swap_space = 0; 1117 if (no_swap_space) { 1118 for (i = 0; i < count; i++) 1119 rtvals[i] = VM_PAGER_FAIL; 1120 return VM_PAGER_FAIL; 1121 } 1122 spc = NULL; 1123 1124 object = m[0]->object; 1125 paging_pindex = OFF_TO_IDX(object->paging_offset); 1126 1127 failed = 0; 1128 for (j = 0; j < count; j++) { 1129 fidx = m[j]->pindex + paging_pindex; 1130 ix = swap_pager_block_index(fidx); 1131 swb[j] = 0; 1132 if (ix >= object->un_pager.swp.swp_nblocks) { 1133 rtvals[j] = VM_PAGER_FAIL; 1134 failed = 1; 1135 continue; 1136 } else { 1137 rtvals[j] = VM_PAGER_OK; 1138 } 1139 swb[j] = &object->un_pager.swp.swp_blocks[ix]; 1140 swb[j]->swb_locked++; 1141 if (failed) { 1142 rtvals[j] = VM_PAGER_FAIL; 1143 continue; 1144 } 1145 off = swap_pager_block_offset(fidx); 1146 reqaddr[j] = swb[j]->swb_block[off]; 1147 if (reqaddr[j] == SWB_EMPTY) { 1148 daddr_t blk; 1149 int tries; 1150 int ntoget; 1151 1152 tries = 0; 1153 s = splbio(); 1154 1155 /* 1156 * if any other pages have been allocated in this 1157 * block, we only try to get one page. 1158 */ 1159 for (i = 0; i < SWB_NPAGES; i++) { 1160 if (swb[j]->swb_block[i] != SWB_EMPTY) 1161 break; 1162 } 1163 1164 ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 1165 /* 1166 * this code is alittle conservative, but works (the 1167 * intent of this code is to allocate small chunks for 1168 * small objects) 1169 */ 1170 if ((off == 0) && ((fidx + ntoget) > object->size)) { 1171 ntoget = object->size - fidx; 1172 } 1173 retrygetspace: 1174 if (!swap_pager_full && ntoget > 1 && 1175 swap_pager_getswapspace(object, ntoget * btodb(PAGE_SIZE), 1176 &blk)) { 1177 1178 for (i = 0; i < ntoget; i++) { 1179 swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 1180 swb[j]->swb_valid = 0; 1181 } 1182 1183 reqaddr[j] = swb[j]->swb_block[off]; 1184 } else if (!swap_pager_getswapspace(object, btodb(PAGE_SIZE), 1185 &swb[j]->swb_block[off])) { 1186 /* 1187 * if the allocation has failed, we try to 1188 * reclaim space and retry. 1189 */ 1190 if (++tries == 1) { 1191 swap_pager_reclaim(); 1192 goto retrygetspace; 1193 } 1194 rtvals[j] = VM_PAGER_AGAIN; 1195 failed = 1; 1196 swap_pager_full = 1; 1197 } else { 1198 reqaddr[j] = swb[j]->swb_block[off]; 1199 swb[j]->swb_valid &= ~(1 << off); 1200 } 1201 splx(s); 1202 } 1203 } 1204 1205 /* 1206 * search forwards for the last contiguous page to transfer 1207 */ 1208 failed = 0; 1209 for (i = 0; i < count; i++) { 1210 if (failed || 1211 (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) || 1212 ((reqaddr[i] / dmmax) != (reqaddr[0] / dmmax)) || 1213 (rtvals[i] != VM_PAGER_OK)) { 1214 failed = 1; 1215 if (rtvals[i] == VM_PAGER_OK) 1216 rtvals[i] = VM_PAGER_AGAIN; 1217 } 1218 } 1219 1220 for (i = 0; i < count; i++) { 1221 if (rtvals[i] != VM_PAGER_OK) { 1222 if (swb[i]) 1223 --swb[i]->swb_locked; 1224 } 1225 } 1226 1227 for (i = 0; i < count; i++) 1228 if (rtvals[i] != VM_PAGER_OK) 1229 break; 1230 1231 if (i == 0) { 1232 return VM_PAGER_AGAIN; 1233 } 1234 count = i; 1235 for (i = 0; i < count; i++) { 1236 if (reqaddr[i] == SWB_EMPTY) { 1237 printf("I/O to empty block???? -- pindex: %d, i: %d\n", 1238 m[i]->pindex, i); 1239 } 1240 } 1241 1242 /* 1243 * For synchronous writes, we clean up all completed async pageouts. 1244 */ 1245 if (sync == TRUE) { 1246 swap_pager_sync(); 1247 } 1248 kva = 0; 1249 1250 /* 1251 * get a swap pager clean data structure, block until we get it 1252 */ 1253 if (swap_pager_free.tqh_first == NULL || 1254 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1255 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1256 s = splbio(); 1257 if (curproc == pageproc) { 1258 /* 1259 * pageout daemon needs a swap control block 1260 */ 1261 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT|SWAP_FREE_NEEDED; 1262 /* 1263 * if it does not get one within a short time, then 1264 * there is a potential deadlock, so we go-on trying 1265 * to free pages. 1266 */ 1267 tsleep(&swap_pager_free, PVM, "swpfre", hz/10); 1268 swap_pager_sync(); 1269 if (swap_pager_free.tqh_first == NULL || 1270 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1271 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1272 splx(s); 1273 return VM_PAGER_AGAIN; 1274 } 1275 } else 1276 pagedaemon_wakeup(); 1277 while (swap_pager_free.tqh_first == NULL || 1278 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1279 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1280 if (curproc == pageproc) { 1281 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 1282 if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved) 1283 wakeup(&cnt.v_free_count); 1284 } 1285 1286 swap_pager_needflags |= SWAP_FREE_NEEDED; 1287 tsleep(&swap_pager_free, PVM, "swpfre", 0); 1288 if (curproc == pageproc) 1289 swap_pager_sync(); 1290 else 1291 pagedaemon_wakeup(); 1292 } 1293 splx(s); 1294 } 1295 spc = swap_pager_free.tqh_first; 1296 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1297 1298 kva = spc->spc_kva; 1299 1300 /* 1301 * map our page(s) into kva for I/O 1302 */ 1303 pmap_qenter(kva, m, count); 1304 1305 /* 1306 * get the base I/O offset into the swap file 1307 */ 1308 for (i = 0; i < count; i++) { 1309 fidx = m[i]->pindex + paging_pindex; 1310 off = swap_pager_block_offset(fidx); 1311 /* 1312 * set the valid bit 1313 */ 1314 swb[i]->swb_valid |= (1 << off); 1315 /* 1316 * and unlock the data structure 1317 */ 1318 swb[i]->swb_locked--; 1319 } 1320 1321 /* 1322 * Get a swap buffer header and perform the IO 1323 */ 1324 bp = spc->spc_bp; 1325 bzero(bp, sizeof *bp); 1326 bp->b_spc = spc; 1327 bp->b_vnbufs.le_next = NOLIST; 1328 1329 bp->b_flags = B_BUSY | B_PAGING; 1330 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 1331 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1332 if (bp->b_rcred != NOCRED) 1333 crhold(bp->b_rcred); 1334 if (bp->b_wcred != NOCRED) 1335 crhold(bp->b_wcred); 1336 bp->b_data = (caddr_t) kva; 1337 bp->b_blkno = reqaddr[0]; 1338 pbgetvp(swapdev_vp, bp); 1339 1340 bp->b_bcount = PAGE_SIZE * count; 1341 bp->b_bufsize = PAGE_SIZE * count; 1342 swapdev_vp->v_numoutput++; 1343 1344 /* 1345 * If this is an async write we set up additional buffer fields and 1346 * place a "cleaning" entry on the inuse queue. 1347 */ 1348 s = splbio(); 1349 if (sync == FALSE) { 1350 spc->spc_flags = 0; 1351 spc->spc_object = object; 1352 for (i = 0; i < count; i++) 1353 spc->spc_m[i] = m[i]; 1354 spc->spc_count = count; 1355 /* 1356 * the completion routine for async writes 1357 */ 1358 bp->b_flags |= B_CALL; 1359 bp->b_iodone = swap_pager_iodone; 1360 bp->b_dirtyoff = 0; 1361 bp->b_dirtyend = bp->b_bcount; 1362 object->un_pager.swp.swp_poip++; 1363 TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 1364 } else { 1365 object->un_pager.swp.swp_poip++; 1366 bp->b_flags |= B_CALL; 1367 bp->b_iodone = swap_pager_iodone1; 1368 } 1369 1370 cnt.v_swapout++; 1371 cnt.v_swappgsout += count; 1372 /* 1373 * perform the I/O 1374 */ 1375 VOP_STRATEGY(bp); 1376 if (sync == FALSE) { 1377 if ((bp->b_flags & B_DONE) == B_DONE) { 1378 swap_pager_sync(); 1379 } 1380 splx(s); 1381 for (i = 0; i < count; i++) { 1382 rtvals[i] = VM_PAGER_PEND; 1383 } 1384 return VM_PAGER_PEND; 1385 } 1386 /* 1387 * wait for the sync I/O to complete 1388 */ 1389 while ((bp->b_flags & B_DONE) == 0) { 1390 tsleep(bp, PVM, "swwrt", 0); 1391 } 1392 if (bp->b_flags & B_ERROR) { 1393 printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 1394 bp->b_blkno, bp->b_bcount, bp->b_error); 1395 rv = VM_PAGER_ERROR; 1396 } else { 1397 rv = VM_PAGER_OK; 1398 } 1399 1400 object->un_pager.swp.swp_poip--; 1401 if (object->un_pager.swp.swp_poip == 0) 1402 wakeup(object); 1403 1404 if (bp->b_vp) 1405 pbrelvp(bp); 1406 if (bp->b_flags & B_WANTED) 1407 wakeup(bp); 1408 1409 splx(s); 1410 1411 /* 1412 * remove the mapping for kernel virtual 1413 */ 1414 pmap_qremove(kva, count); 1415 1416 /* 1417 * if we have written the page, then indicate that the page is clean. 1418 */ 1419 if (rv == VM_PAGER_OK) { 1420 for (i = 0; i < count; i++) { 1421 if (rtvals[i] == VM_PAGER_OK) { 1422 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1423 m[i]->dirty = 0; 1424 /* 1425 * optimization, if a page has been read 1426 * during the pageout process, we activate it. 1427 */ 1428 if ((m[i]->flags & PG_ACTIVE) == 0 && 1429 ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || 1430 pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { 1431 vm_page_activate(m[i]); 1432 } 1433 } 1434 } 1435 } else { 1436 for (i = 0; i < count; i++) { 1437 rtvals[i] = rv; 1438 } 1439 } 1440 1441 if (bp->b_rcred != NOCRED) 1442 crfree(bp->b_rcred); 1443 if (bp->b_wcred != NOCRED) 1444 crfree(bp->b_wcred); 1445 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1446 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1447 wakeup(&swap_pager_free); 1448 } 1449 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1450 pagedaemon_wakeup(); 1451 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1452 return (rv); 1453} 1454 1455void 1456swap_pager_sync() 1457{ 1458 register swp_clean_t spc, tspc; 1459 register int s; 1460 1461 tspc = NULL; 1462 if (swap_pager_done.tqh_first == NULL) 1463 return; 1464 for (;;) { 1465 s = splbio(); 1466 /* 1467 * Look up and removal from done list must be done at splbio() 1468 * to avoid conflicts with swap_pager_iodone. 1469 */ 1470 while ((spc = swap_pager_done.tqh_first) != 0) { 1471 pmap_qremove(spc->spc_kva, spc->spc_count); 1472 swap_pager_finish(spc); 1473 TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 1474 goto doclean; 1475 } 1476 1477 /* 1478 * No operations done, thats all we can do for now. 1479 */ 1480 1481 splx(s); 1482 break; 1483 1484 /* 1485 * The desired page was found to be busy earlier in the scan 1486 * but has since completed. 1487 */ 1488doclean: 1489 if (tspc && tspc == spc) { 1490 tspc = NULL; 1491 } 1492 spc->spc_flags = 0; 1493 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1494 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1495 wakeup(&swap_pager_free); 1496 } 1497 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1498 pagedaemon_wakeup(); 1499 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1500 splx(s); 1501 } 1502 1503 return; 1504} 1505 1506void 1507swap_pager_finish(spc) 1508 register swp_clean_t spc; 1509{ 1510 vm_object_t object = spc->spc_m[0]->object; 1511 int i; 1512 1513 object->paging_in_progress -= spc->spc_count; 1514 if ((object->paging_in_progress == 0) && 1515 (object->flags & OBJ_PIPWNT)) { 1516 object->flags &= ~OBJ_PIPWNT; 1517 wakeup(object); 1518 } 1519 1520 /* 1521 * If no error, mark as clean and inform the pmap system. If error, 1522 * mark as dirty so we will try again. (XXX could get stuck doing 1523 * this, should give up after awhile) 1524 */ 1525 if (spc->spc_flags & SPC_ERROR) { 1526 for (i = 0; i < spc->spc_count; i++) { 1527 printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 1528 (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); 1529 } 1530 } else { 1531 for (i = 0; i < spc->spc_count; i++) { 1532 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 1533 spc->spc_m[i]->dirty = 0; 1534 if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 && 1535 ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) 1536 vm_page_activate(spc->spc_m[i]); 1537 } 1538 } 1539 1540 1541 for (i = 0; i < spc->spc_count; i++) { 1542 /* 1543 * we wakeup any processes that are waiting on these pages. 1544 */ 1545 PAGE_WAKEUP(spc->spc_m[i]); 1546 } 1547 nswiodone -= spc->spc_count; 1548 1549 return; 1550} 1551 1552/* 1553 * swap_pager_iodone 1554 */ 1555static void 1556swap_pager_iodone(bp) 1557 register struct buf *bp; 1558{ 1559 register swp_clean_t spc; 1560 int s; 1561 1562 s = splbio(); 1563 spc = (swp_clean_t) bp->b_spc; 1564 TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 1565 TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 1566 if (bp->b_flags & B_ERROR) { 1567 spc->spc_flags |= SPC_ERROR; 1568 printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n", 1569 (bp->b_flags & B_READ) ? "pagein" : "pageout", 1570 (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); 1571 } 1572 1573 if (bp->b_vp) 1574 pbrelvp(bp); 1575 1576 if (bp->b_flags & B_WANTED) 1577 wakeup(bp); 1578 1579 if (bp->b_rcred != NOCRED) 1580 crfree(bp->b_rcred); 1581 if (bp->b_wcred != NOCRED) 1582 crfree(bp->b_wcred); 1583 1584 nswiodone += spc->spc_count; 1585 if (--spc->spc_object->un_pager.swp.swp_poip == 0) { 1586 wakeup(spc->spc_object); 1587 } 1588 if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 1589 swap_pager_inuse.tqh_first == 0) { 1590 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1591 wakeup(&swap_pager_free); 1592 } 1593 1594 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) { 1595 swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT; 1596 pagedaemon_wakeup(); 1597 } 1598 1599 if (vm_pageout_pages_needed) { 1600 wakeup(&vm_pageout_pages_needed); 1601 vm_pageout_pages_needed = 0; 1602 } 1603 if ((swap_pager_inuse.tqh_first == NULL) || 1604 ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min && 1605 nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) { 1606 pagedaemon_wakeup(); 1607 } 1608 splx(s); 1609}
| 163static void swap_pager_reclaim __P((void)); 164static void swap_pager_ridpages __P((vm_page_t *m, int count, 165 int reqpage)); 166static void swap_pager_setvalid __P((vm_object_t object, 167 vm_offset_t offset, int valid)); 168static void swapsizecheck __P((void)); 169 170static inline void 171swapsizecheck() 172{ 173 if (vm_swap_size < 128 * btodb(PAGE_SIZE)) { 174 if (swap_pager_full == 0) 175 printf("swap_pager: out of space\n"); 176 swap_pager_full = 1; 177 } else if (vm_swap_size > 192 * btodb(PAGE_SIZE)) 178 swap_pager_full = 0; 179} 180 181static void 182swap_pager_init() 183{ 184 TAILQ_INIT(&swap_pager_object_list); 185 TAILQ_INIT(&swap_pager_un_object_list); 186 187 /* 188 * Initialize clean lists 189 */ 190 TAILQ_INIT(&swap_pager_inuse); 191 TAILQ_INIT(&swap_pager_done); 192 TAILQ_INIT(&swap_pager_free); 193 194 /* 195 * Calculate the swap allocation constants. 196 */ 197 dmmin = CLBYTES / DEV_BSIZE; 198 dmmax = btodb(SWB_NPAGES * PAGE_SIZE) * 2; 199} 200 201void 202swap_pager_swap_init() 203{ 204 swp_clean_t spc; 205 struct buf *bp; 206 int i; 207 208 /* 209 * kva's are allocated here so that we dont need to keep doing 210 * kmem_alloc pageables at runtime 211 */ 212 for (i = 0, spc = swcleanlist; i < npendingio; i++, spc++) { 213 spc->spc_kva = kmem_alloc_pageable(pager_map, PAGE_SIZE * MAX_PAGEOUT_CLUSTER); 214 if (!spc->spc_kva) { 215 break; 216 } 217 spc->spc_bp = malloc(sizeof(*bp), M_TEMP, M_KERNEL); 218 if (!spc->spc_bp) { 219 kmem_free_wakeup(pager_map, spc->spc_kva, PAGE_SIZE); 220 break; 221 } 222 spc->spc_flags = 0; 223 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 224 } 225} 226 227int 228swap_pager_swp_alloc(object, wait) 229 vm_object_t object; 230 int wait; 231{ 232 sw_blk_t swb; 233 int nblocks; 234 int i, j; 235 236 nblocks = (object->size + SWB_NPAGES - 1) / SWB_NPAGES; 237 swb = malloc(nblocks * sizeof(*swb), M_VMPGDATA, wait); 238 if (swb == NULL) 239 return 1; 240 241 for (i = 0; i < nblocks; i++) { 242 swb[i].swb_valid = 0; 243 swb[i].swb_locked = 0; 244 for (j = 0; j < SWB_NPAGES; j++) 245 swb[i].swb_block[j] = SWB_EMPTY; 246 } 247 248 object->un_pager.swp.swp_nblocks = nblocks; 249 object->un_pager.swp.swp_allocsize = 0; 250 object->un_pager.swp.swp_blocks = swb; 251 object->un_pager.swp.swp_poip = 0; 252 253 if (object->handle != NULL) { 254 TAILQ_INSERT_TAIL(&swap_pager_object_list, object, pager_object_list); 255 } else { 256 TAILQ_INSERT_TAIL(&swap_pager_un_object_list, object, pager_object_list); 257 } 258 259 return 0; 260} 261 262/* 263 * Allocate an object and associated resources. 264 * Note that if we are called from the pageout daemon (handle == NULL) 265 * we should not wait for memory as it could resulting in deadlock. 266 */ 267static vm_object_t 268swap_pager_alloc(handle, size, prot, offset) 269 void *handle; 270 register vm_size_t size; 271 vm_prot_t prot; 272 vm_ooffset_t offset; 273{ 274 vm_object_t object; 275 276 /* 277 * If this is a "named" anonymous region, look it up and use the 278 * object if it exists, otherwise allocate a new one. 279 */ 280 if (handle) { 281 object = vm_pager_object_lookup(&swap_pager_object_list, handle); 282 if (object != NULL) { 283 vm_object_reference(object); 284 } else { 285 /* 286 * XXX - there is a race condition here. Two processes 287 * can request the same named object simultaneuously, 288 * and if one blocks for memory, the result is a disaster. 289 * Probably quite rare, but is yet another reason to just 290 * rip support of "named anonymous regions" out altogether. 291 */ 292 object = vm_object_allocate(OBJT_SWAP, 293 OFF_TO_IDX(offset+ PAGE_SIZE - 1 + size)); 294 object->handle = handle; 295 (void) swap_pager_swp_alloc(object, M_WAITOK); 296 } 297 } else { 298 object = vm_object_allocate(OBJT_SWAP, 299 OFF_TO_IDX(offset + PAGE_SIZE - 1 + size)); 300 (void) swap_pager_swp_alloc(object, M_WAITOK); 301 } 302 303 return (object); 304} 305 306/* 307 * returns disk block associated with pager and offset 308 * additionally, as a side effect returns a flag indicating 309 * if the block has been written 310 */ 311 312inline static daddr_t * 313swap_pager_diskaddr(object, pindex, valid) 314 vm_object_t object; 315 vm_pindex_t pindex; 316 int *valid; 317{ 318 register sw_blk_t swb; 319 int ix; 320 321 if (valid) 322 *valid = 0; 323 ix = pindex / SWB_NPAGES; 324 if ((ix >= object->un_pager.swp.swp_nblocks) || 325 (pindex >= object->size)) { 326 return (FALSE); 327 } 328 swb = &object->un_pager.swp.swp_blocks[ix]; 329 ix = pindex % SWB_NPAGES; 330 if (valid) 331 *valid = swb->swb_valid & (1 << ix); 332 return &swb->swb_block[ix]; 333} 334 335/* 336 * Utility routine to set the valid (written) bit for 337 * a block associated with a pager and offset 338 */ 339static void 340swap_pager_setvalid(object, offset, valid) 341 vm_object_t object; 342 vm_offset_t offset; 343 int valid; 344{ 345 register sw_blk_t swb; 346 int ix; 347 348 ix = offset / SWB_NPAGES; 349 if (ix >= object->un_pager.swp.swp_nblocks) 350 return; 351 352 swb = &object->un_pager.swp.swp_blocks[ix]; 353 ix = offset % SWB_NPAGES; 354 if (valid) 355 swb->swb_valid |= (1 << ix); 356 else 357 swb->swb_valid &= ~(1 << ix); 358 return; 359} 360 361/* 362 * this routine allocates swap space with a fragmentation 363 * minimization policy. 364 */ 365static int 366swap_pager_getswapspace(object, amount, rtval) 367 vm_object_t object; 368 unsigned int amount; 369 daddr_t *rtval; 370{ 371 unsigned location; 372 vm_swap_size -= amount; 373 if (!rlist_alloc(&swaplist, amount, &location)) { 374 vm_swap_size += amount; 375 return 0; 376 } else { 377 swapsizecheck(); 378 object->un_pager.swp.swp_allocsize += amount; 379 *rtval = location; 380 return 1; 381 } 382} 383 384/* 385 * this routine frees swap space with a fragmentation 386 * minimization policy. 387 */ 388static void 389swap_pager_freeswapspace(object, from, to) 390 vm_object_t object; 391 unsigned int from; 392 unsigned int to; 393{ 394 rlist_free(&swaplist, from, to); 395 vm_swap_size += (to - from) + 1; 396 object->un_pager.swp.swp_allocsize -= (to - from) + 1; 397 swapsizecheck(); 398} 399/* 400 * this routine frees swap blocks from a specified pager 401 */ 402void 403swap_pager_freespace(object, start, size) 404 vm_object_t object; 405 vm_pindex_t start; 406 vm_size_t size; 407{ 408 vm_pindex_t i; 409 int s; 410 411 s = splbio(); 412 for (i = start; i < start + size; i += 1) { 413 int valid; 414 daddr_t *addr = swap_pager_diskaddr(object, i, &valid); 415 416 if (addr && *addr != SWB_EMPTY) { 417 swap_pager_freeswapspace(object, *addr, *addr + btodb(PAGE_SIZE) - 1); 418 if (valid) { 419 swap_pager_setvalid(object, i, 0); 420 } 421 *addr = SWB_EMPTY; 422 } 423 } 424 splx(s); 425} 426 427static void 428swap_pager_free_swap(object) 429 vm_object_t object; 430{ 431 register int i, j; 432 register sw_blk_t swb; 433 int first_block=0, block_count=0; 434 int s; 435 /* 436 * Free left over swap blocks 437 */ 438 s = splbio(); 439 for (i = 0, swb = object->un_pager.swp.swp_blocks; 440 i < object->un_pager.swp.swp_nblocks; i++, swb++) { 441 for (j = 0; j < SWB_NPAGES; j++) { 442 if (swb->swb_block[j] != SWB_EMPTY) { 443 /* 444 * initially the length of the run is zero 445 */ 446 if (block_count == 0) { 447 first_block = swb->swb_block[j]; 448 block_count = btodb(PAGE_SIZE); 449 swb->swb_block[j] = SWB_EMPTY; 450 /* 451 * if the new block can be included into the current run 452 */ 453 } else if (swb->swb_block[j] == first_block + block_count) { 454 block_count += btodb(PAGE_SIZE); 455 swb->swb_block[j] = SWB_EMPTY; 456 /* 457 * terminate the previous run, and start a new one 458 */ 459 } else { 460 swap_pager_freeswapspace(object, first_block, 461 (unsigned) first_block + block_count - 1); 462 first_block = swb->swb_block[j]; 463 block_count = btodb(PAGE_SIZE); 464 swb->swb_block[j] = SWB_EMPTY; 465 } 466 } 467 } 468 } 469 470 if (block_count) { 471 swap_pager_freeswapspace(object, first_block, 472 (unsigned) first_block + block_count - 1); 473 } 474 splx(s); 475} 476 477 478/* 479 * swap_pager_reclaim frees up over-allocated space from all pagers 480 * this eliminates internal fragmentation due to allocation of space 481 * for segments that are never swapped to. It has been written so that 482 * it does not block until the rlist_free operation occurs; it keeps 483 * the queues consistant. 484 */ 485 486/* 487 * Maximum number of blocks (pages) to reclaim per pass 488 */ 489#define MAXRECLAIM 128 490 491static void 492swap_pager_reclaim() 493{ 494 vm_object_t object; 495 int i, j, k; 496 int s; 497 int reclaimcount; 498 static struct { 499 int address; 500 vm_object_t object; 501 } reclaims[MAXRECLAIM]; 502 static int in_reclaim; 503 504 /* 505 * allow only one process to be in the swap_pager_reclaim subroutine 506 */ 507 s = splbio(); 508 if (in_reclaim) { 509 tsleep(&in_reclaim, PSWP, "swrclm", 0); 510 splx(s); 511 return; 512 } 513 in_reclaim = 1; 514 reclaimcount = 0; 515 516 /* for each pager queue */ 517 for (k = 0; swp_qs[k]; k++) { 518 519 object = swp_qs[k]->tqh_first; 520 while (object && (reclaimcount < MAXRECLAIM)) { 521 522 /* 523 * see if any blocks associated with a pager has been 524 * allocated but not used (written) 525 */ 526 if (object->paging_in_progress == 0) { 527 for (i = 0; i < object->un_pager.swp.swp_nblocks; i++) { 528 sw_blk_t swb = &object->un_pager.swp.swp_blocks[i]; 529 530 if (swb->swb_locked) 531 continue; 532 for (j = 0; j < SWB_NPAGES; j++) { 533 if (swb->swb_block[j] != SWB_EMPTY && 534 (swb->swb_valid & (1 << j)) == 0) { 535 reclaims[reclaimcount].address = swb->swb_block[j]; 536 reclaims[reclaimcount++].object = object; 537 swb->swb_block[j] = SWB_EMPTY; 538 if (reclaimcount >= MAXRECLAIM) 539 goto rfinished; 540 } 541 } 542 } 543 } 544 object = object->pager_object_list.tqe_next; 545 } 546 } 547 548rfinished: 549 550 /* 551 * free the blocks that have been added to the reclaim list 552 */ 553 for (i = 0; i < reclaimcount; i++) { 554 swap_pager_freeswapspace(reclaims[i].object, 555 reclaims[i].address, reclaims[i].address + btodb(PAGE_SIZE) - 1); 556 } 557 splx(s); 558 in_reclaim = 0; 559 wakeup(&in_reclaim); 560} 561 562 563/* 564 * swap_pager_copy copies blocks from one pager to another and 565 * destroys the source pager 566 */ 567 568void 569swap_pager_copy(srcobject, srcoffset, dstobject, dstoffset, offset) 570 vm_object_t srcobject; 571 vm_pindex_t srcoffset; 572 vm_object_t dstobject; 573 vm_pindex_t dstoffset; 574 vm_pindex_t offset; 575{ 576 vm_pindex_t i; 577 int origsize; 578 int s; 579 580 if (vm_swap_size) 581 no_swap_space = 0; 582 583 origsize = srcobject->un_pager.swp.swp_allocsize; 584 585 /* 586 * remove the source object from the swap_pager internal queue 587 */ 588 if (srcobject->handle == NULL) { 589 TAILQ_REMOVE(&swap_pager_un_object_list, srcobject, pager_object_list); 590 } else { 591 TAILQ_REMOVE(&swap_pager_object_list, srcobject, pager_object_list); 592 } 593 594 s = splbio(); 595 while (srcobject->un_pager.swp.swp_poip) { 596 tsleep(srcobject, PVM, "spgout", 0); 597 } 598 splx(s); 599 600 /* 601 * clean all of the pages that are currently active and finished 602 */ 603 swap_pager_sync(); 604 605 s = splbio(); 606 /* 607 * transfer source to destination 608 */ 609 for (i = 0; i < dstobject->size; i += 1) { 610 int srcvalid, dstvalid; 611 daddr_t *srcaddrp = swap_pager_diskaddr(srcobject, i + offset + srcoffset, 612 &srcvalid); 613 daddr_t *dstaddrp; 614 615 /* 616 * see if the source has space allocated 617 */ 618 if (srcaddrp && *srcaddrp != SWB_EMPTY) { 619 /* 620 * if the source is valid and the dest has no space, 621 * then copy the allocation from the srouce to the 622 * dest. 623 */ 624 if (srcvalid) { 625 dstaddrp = swap_pager_diskaddr(dstobject, i + dstoffset, 626 &dstvalid); 627 /* 628 * if the dest already has a valid block, 629 * deallocate the source block without 630 * copying. 631 */ 632 if (!dstvalid && dstaddrp && *dstaddrp != SWB_EMPTY) { 633 swap_pager_freeswapspace(dstobject, *dstaddrp, 634 *dstaddrp + btodb(PAGE_SIZE) - 1); 635 *dstaddrp = SWB_EMPTY; 636 } 637 if (dstaddrp && *dstaddrp == SWB_EMPTY) { 638 *dstaddrp = *srcaddrp; 639 *srcaddrp = SWB_EMPTY; 640 dstobject->un_pager.swp.swp_allocsize += btodb(PAGE_SIZE); 641 srcobject->un_pager.swp.swp_allocsize -= btodb(PAGE_SIZE); 642 swap_pager_setvalid(dstobject, i + dstoffset, 1); 643 } 644 } 645 /* 646 * if the source is not empty at this point, then 647 * deallocate the space. 648 */ 649 if (*srcaddrp != SWB_EMPTY) { 650 swap_pager_freeswapspace(srcobject, *srcaddrp, 651 *srcaddrp + btodb(PAGE_SIZE) - 1); 652 *srcaddrp = SWB_EMPTY; 653 } 654 } 655 } 656 splx(s); 657 658 /* 659 * Free left over swap blocks 660 */ 661 swap_pager_free_swap(srcobject); 662 663 if (srcobject->un_pager.swp.swp_allocsize) { 664 printf("swap_pager_copy: *warning* pager with %d blocks (orig: %d)\n", 665 srcobject->un_pager.swp.swp_allocsize, origsize); 666 } 667 668 free(srcobject->un_pager.swp.swp_blocks, M_VMPGDATA); 669 srcobject->un_pager.swp.swp_blocks = NULL; 670 671 return; 672} 673 674static void 675swap_pager_dealloc(object) 676 vm_object_t object; 677{ 678 int s; 679 680 /* 681 * Remove from list right away so lookups will fail if we block for 682 * pageout completion. 683 */ 684 if (object->handle == NULL) { 685 TAILQ_REMOVE(&swap_pager_un_object_list, object, pager_object_list); 686 } else { 687 TAILQ_REMOVE(&swap_pager_object_list, object, pager_object_list); 688 } 689 690 /* 691 * Wait for all pageouts to finish and remove all entries from 692 * cleaning list. 693 */ 694 695 s = splbio(); 696 while (object->un_pager.swp.swp_poip) { 697 tsleep(object, PVM, "swpout", 0); 698 } 699 splx(s); 700 701 702 swap_pager_sync(); 703 704 /* 705 * Free left over swap blocks 706 */ 707 swap_pager_free_swap(object); 708 709 if (object->un_pager.swp.swp_allocsize) { 710 printf("swap_pager_dealloc: *warning* freeing pager with %d blocks\n", 711 object->un_pager.swp.swp_allocsize); 712 } 713 /* 714 * Free swap management resources 715 */ 716 free(object->un_pager.swp.swp_blocks, M_VMPGDATA); 717 object->un_pager.swp.swp_blocks = NULL; 718} 719 720static inline __pure int 721swap_pager_block_index(pindex) 722 vm_pindex_t pindex; 723{ 724 return (pindex / SWB_NPAGES); 725} 726 727static inline __pure int 728swap_pager_block_offset(pindex) 729 vm_pindex_t pindex; 730{ 731 return (pindex % SWB_NPAGES); 732} 733 734/* 735 * swap_pager_haspage returns TRUE if the pager has data that has 736 * been written out. 737 */ 738static boolean_t 739swap_pager_haspage(object, pindex, before, after) 740 vm_object_t object; 741 vm_pindex_t pindex; 742 int *before; 743 int *after; 744{ 745 register sw_blk_t swb; 746 int ix; 747 748 if (before != NULL) 749 *before = 0; 750 if (after != NULL) 751 *after = 0; 752 ix = pindex / SWB_NPAGES; 753 if (ix >= object->un_pager.swp.swp_nblocks) { 754 return (FALSE); 755 } 756 swb = &object->un_pager.swp.swp_blocks[ix]; 757 ix = pindex % SWB_NPAGES; 758 759 if (swb->swb_block[ix] != SWB_EMPTY) { 760 761 if (swb->swb_valid & (1 << ix)) { 762 int tix; 763 if (before) { 764 for(tix = ix - 1; tix >= 0; --tix) { 765 if ((swb->swb_valid & (1 << tix)) == 0) 766 break; 767 if ((swb->swb_block[tix] + 768 (ix - tix) * (PAGE_SIZE/DEV_BSIZE)) != 769 swb->swb_block[ix]) 770 break; 771 (*before)++; 772 } 773 } 774 775 if (after) { 776 for(tix = ix + 1; tix < SWB_NPAGES; tix++) { 777 if ((swb->swb_valid & (1 << tix)) == 0) 778 break; 779 if ((swb->swb_block[tix] - 780 (tix - ix) * (PAGE_SIZE/DEV_BSIZE)) != 781 swb->swb_block[ix]) 782 break; 783 (*after)++; 784 } 785 } 786 787 return TRUE; 788 } 789 } 790 return (FALSE); 791} 792 793/* 794 * swap_pager_freepage is a convienience routine that clears the busy 795 * bit and deallocates a page. 796 */ 797static void 798swap_pager_freepage(m) 799 vm_page_t m; 800{ 801 PAGE_WAKEUP(m); 802 vm_page_free(m); 803} 804 805/* 806 * swap_pager_ridpages is a convienience routine that deallocates all 807 * but the required page. this is usually used in error returns that 808 * need to invalidate the "extra" readahead pages. 809 */ 810static void 811swap_pager_ridpages(m, count, reqpage) 812 vm_page_t *m; 813 int count; 814 int reqpage; 815{ 816 int i; 817 818 for (i = 0; i < count; i++) 819 if (i != reqpage) 820 swap_pager_freepage(m[i]); 821} 822 823/* 824 * swap_pager_iodone1 is the completion routine for both reads and async writes 825 */ 826static void 827swap_pager_iodone1(bp) 828 struct buf *bp; 829{ 830 bp->b_flags |= B_DONE; 831 bp->b_flags &= ~B_ASYNC; 832 wakeup(bp); 833} 834 835int 836swap_pager_getpages(object, m, count, reqpage) 837 vm_object_t object; 838 vm_page_t *m; 839 int count, reqpage; 840{ 841 register struct buf *bp; 842 sw_blk_t swb[count]; 843 register int s; 844 int i; 845 boolean_t rv; 846 vm_offset_t kva, off[count]; 847 swp_clean_t spc; 848 vm_pindex_t paging_offset; 849 int reqaddr[count]; 850 int sequential; 851 852 int first, last; 853 int failed; 854 int reqdskregion; 855 856 object = m[reqpage]->object; 857 paging_offset = OFF_TO_IDX(object->paging_offset); 858 sequential = (m[reqpage]->pindex == (object->last_read + 1)); 859 860 for (i = 0; i < count; i++) { 861 vm_pindex_t fidx = m[i]->pindex + paging_offset; 862 int ix = swap_pager_block_index(fidx); 863 864 if (ix >= object->un_pager.swp.swp_nblocks) { 865 int j; 866 867 if (i <= reqpage) { 868 swap_pager_ridpages(m, count, reqpage); 869 return (VM_PAGER_FAIL); 870 } 871 for (j = i; j < count; j++) { 872 swap_pager_freepage(m[j]); 873 } 874 count = i; 875 break; 876 } 877 swb[i] = &object->un_pager.swp.swp_blocks[ix]; 878 off[i] = swap_pager_block_offset(fidx); 879 reqaddr[i] = swb[i]->swb_block[off[i]]; 880 } 881 882 /* make sure that our required input request is existant */ 883 884 if (reqaddr[reqpage] == SWB_EMPTY || 885 (swb[reqpage]->swb_valid & (1 << off[reqpage])) == 0) { 886 swap_pager_ridpages(m, count, reqpage); 887 return (VM_PAGER_FAIL); 888 } 889 reqdskregion = reqaddr[reqpage] / dmmax; 890 891 /* 892 * search backwards for the first contiguous page to transfer 893 */ 894 failed = 0; 895 first = 0; 896 for (i = reqpage - 1; i >= 0; --i) { 897 if (sequential || failed || (reqaddr[i] == SWB_EMPTY) || 898 (swb[i]->swb_valid & (1 << off[i])) == 0 || 899 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 900 ((reqaddr[i] / dmmax) != reqdskregion)) { 901 failed = 1; 902 swap_pager_freepage(m[i]); 903 if (first == 0) 904 first = i + 1; 905 } 906 } 907 /* 908 * search forwards for the last contiguous page to transfer 909 */ 910 failed = 0; 911 last = count; 912 for (i = reqpage + 1; i < count; i++) { 913 if (failed || (reqaddr[i] == SWB_EMPTY) || 914 (swb[i]->swb_valid & (1 << off[i])) == 0 || 915 (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || 916 ((reqaddr[i] / dmmax) != reqdskregion)) { 917 failed = 1; 918 swap_pager_freepage(m[i]); 919 if (last == count) 920 last = i; 921 } 922 } 923 924 count = last; 925 if (first != 0) { 926 for (i = first; i < count; i++) { 927 m[i - first] = m[i]; 928 reqaddr[i - first] = reqaddr[i]; 929 off[i - first] = off[i]; 930 } 931 count -= first; 932 reqpage -= first; 933 } 934 ++swb[reqpage]->swb_locked; 935 936 /* 937 * at this point: "m" is a pointer to the array of vm_page_t for 938 * paging I/O "count" is the number of vm_page_t entries represented 939 * by "m" "object" is the vm_object_t for I/O "reqpage" is the index 940 * into "m" for the page actually faulted 941 */ 942 943 spc = NULL; /* we might not use an spc data structure */ 944 945 if ((count == 1) && (swap_pager_free.tqh_first != NULL)) { 946 spc = swap_pager_free.tqh_first; 947 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 948 kva = spc->spc_kva; 949 bp = spc->spc_bp; 950 bzero(bp, sizeof *bp); 951 bp->b_spc = spc; 952 bp->b_vnbufs.le_next = NOLIST; 953 } else { 954 /* 955 * Get a swap buffer header to perform the IO 956 */ 957 bp = getpbuf(); 958 kva = (vm_offset_t) bp->b_data; 959 } 960 961 /* 962 * map our page(s) into kva for input 963 */ 964 pmap_qenter(kva, m, count); 965 966 bp->b_flags = B_BUSY | B_READ | B_CALL | B_PAGING; 967 bp->b_iodone = swap_pager_iodone1; 968 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 969 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 970 crhold(bp->b_rcred); 971 crhold(bp->b_wcred); 972 bp->b_un.b_addr = (caddr_t) kva; 973 bp->b_blkno = reqaddr[0]; 974 bp->b_bcount = PAGE_SIZE * count; 975 bp->b_bufsize = PAGE_SIZE * count; 976 977 pbgetvp(swapdev_vp, bp); 978 979 cnt.v_swapin++; 980 cnt.v_swappgsin += count; 981 /* 982 * perform the I/O 983 */ 984 VOP_STRATEGY(bp); 985 986 /* 987 * wait for the sync I/O to complete 988 */ 989 s = splbio(); 990 while ((bp->b_flags & B_DONE) == 0) { 991 tsleep(bp, PVM, "swread", 0); 992 } 993 994 if (bp->b_flags & B_ERROR) { 995 printf("swap_pager: I/O error - pagein failed; blkno %d, size %d, error %d\n", 996 bp->b_blkno, bp->b_bcount, bp->b_error); 997 rv = VM_PAGER_ERROR; 998 } else { 999 rv = VM_PAGER_OK; 1000 } 1001 1002 /* 1003 * relpbuf does this, but we maintain our own buffer list also... 1004 */ 1005 if (bp->b_vp) 1006 pbrelvp(bp); 1007 1008 splx(s); 1009 swb[reqpage]->swb_locked--; 1010 1011 /* 1012 * remove the mapping for kernel virtual 1013 */ 1014 pmap_qremove(kva, count); 1015 1016 if (spc) { 1017 m[reqpage]->object->last_read = m[reqpage]->pindex; 1018 if (bp->b_flags & B_WANTED) 1019 wakeup(bp); 1020 /* 1021 * if we have used an spc, we need to free it. 1022 */ 1023 if (bp->b_rcred != NOCRED) 1024 crfree(bp->b_rcred); 1025 if (bp->b_wcred != NOCRED) 1026 crfree(bp->b_wcred); 1027 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1028 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1029 wakeup(&swap_pager_free); 1030 } 1031 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1032 pagedaemon_wakeup(); 1033 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1034 } else { 1035 /* 1036 * release the physical I/O buffer 1037 */ 1038 relpbuf(bp); 1039 /* 1040 * finish up input if everything is ok 1041 */ 1042 if (rv == VM_PAGER_OK) { 1043 for (i = 0; i < count; i++) { 1044 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1045 m[i]->dirty = 0; 1046 m[i]->flags &= ~PG_ZERO; 1047 if (i != reqpage) { 1048 /* 1049 * whether or not to leave the page 1050 * activated is up in the air, but we 1051 * should put the page on a page queue 1052 * somewhere. (it already is in the 1053 * object). After some emperical 1054 * results, it is best to deactivate 1055 * the readahead pages. 1056 */ 1057 vm_page_deactivate(m[i]); 1058 1059 /* 1060 * just in case someone was asking for 1061 * this page we now tell them that it 1062 * is ok to use 1063 */ 1064 m[i]->valid = VM_PAGE_BITS_ALL; 1065 PAGE_WAKEUP(m[i]); 1066 } 1067 } 1068 1069 m[reqpage]->object->last_read = m[count-1]->pindex; 1070 1071 /* 1072 * If we're out of swap space, then attempt to free 1073 * some whenever pages are brought in. We must clear 1074 * the clean flag so that the page contents will be 1075 * preserved. 1076 */ 1077 if (swap_pager_full) { 1078 for (i = 0; i < count; i++) { 1079 m[i]->dirty = VM_PAGE_BITS_ALL; 1080 } 1081 swap_pager_freespace(object, m[0]->pindex + paging_offset, count); 1082 } 1083 } else { 1084 swap_pager_ridpages(m, count, reqpage); 1085 } 1086 } 1087 if (rv == VM_PAGER_OK) { 1088 pmap_clear_modify(VM_PAGE_TO_PHYS(m[reqpage])); 1089 m[reqpage]->valid = VM_PAGE_BITS_ALL; 1090 m[reqpage]->dirty = 0; 1091 } 1092 return (rv); 1093} 1094 1095int 1096swap_pager_putpages(object, m, count, sync, rtvals) 1097 vm_object_t object; 1098 vm_page_t *m; 1099 int count; 1100 boolean_t sync; 1101 int *rtvals; 1102{ 1103 register struct buf *bp; 1104 sw_blk_t swb[count]; 1105 register int s; 1106 int i, j, ix; 1107 boolean_t rv; 1108 vm_offset_t kva, off, fidx; 1109 swp_clean_t spc; 1110 vm_pindex_t paging_pindex; 1111 int reqaddr[count]; 1112 int failed; 1113 1114 if (vm_swap_size) 1115 no_swap_space = 0; 1116 if (no_swap_space) { 1117 for (i = 0; i < count; i++) 1118 rtvals[i] = VM_PAGER_FAIL; 1119 return VM_PAGER_FAIL; 1120 } 1121 spc = NULL; 1122 1123 object = m[0]->object; 1124 paging_pindex = OFF_TO_IDX(object->paging_offset); 1125 1126 failed = 0; 1127 for (j = 0; j < count; j++) { 1128 fidx = m[j]->pindex + paging_pindex; 1129 ix = swap_pager_block_index(fidx); 1130 swb[j] = 0; 1131 if (ix >= object->un_pager.swp.swp_nblocks) { 1132 rtvals[j] = VM_PAGER_FAIL; 1133 failed = 1; 1134 continue; 1135 } else { 1136 rtvals[j] = VM_PAGER_OK; 1137 } 1138 swb[j] = &object->un_pager.swp.swp_blocks[ix]; 1139 swb[j]->swb_locked++; 1140 if (failed) { 1141 rtvals[j] = VM_PAGER_FAIL; 1142 continue; 1143 } 1144 off = swap_pager_block_offset(fidx); 1145 reqaddr[j] = swb[j]->swb_block[off]; 1146 if (reqaddr[j] == SWB_EMPTY) { 1147 daddr_t blk; 1148 int tries; 1149 int ntoget; 1150 1151 tries = 0; 1152 s = splbio(); 1153 1154 /* 1155 * if any other pages have been allocated in this 1156 * block, we only try to get one page. 1157 */ 1158 for (i = 0; i < SWB_NPAGES; i++) { 1159 if (swb[j]->swb_block[i] != SWB_EMPTY) 1160 break; 1161 } 1162 1163 ntoget = (i == SWB_NPAGES) ? SWB_NPAGES : 1; 1164 /* 1165 * this code is alittle conservative, but works (the 1166 * intent of this code is to allocate small chunks for 1167 * small objects) 1168 */ 1169 if ((off == 0) && ((fidx + ntoget) > object->size)) { 1170 ntoget = object->size - fidx; 1171 } 1172 retrygetspace: 1173 if (!swap_pager_full && ntoget > 1 && 1174 swap_pager_getswapspace(object, ntoget * btodb(PAGE_SIZE), 1175 &blk)) { 1176 1177 for (i = 0; i < ntoget; i++) { 1178 swb[j]->swb_block[i] = blk + btodb(PAGE_SIZE) * i; 1179 swb[j]->swb_valid = 0; 1180 } 1181 1182 reqaddr[j] = swb[j]->swb_block[off]; 1183 } else if (!swap_pager_getswapspace(object, btodb(PAGE_SIZE), 1184 &swb[j]->swb_block[off])) { 1185 /* 1186 * if the allocation has failed, we try to 1187 * reclaim space and retry. 1188 */ 1189 if (++tries == 1) { 1190 swap_pager_reclaim(); 1191 goto retrygetspace; 1192 } 1193 rtvals[j] = VM_PAGER_AGAIN; 1194 failed = 1; 1195 swap_pager_full = 1; 1196 } else { 1197 reqaddr[j] = swb[j]->swb_block[off]; 1198 swb[j]->swb_valid &= ~(1 << off); 1199 } 1200 splx(s); 1201 } 1202 } 1203 1204 /* 1205 * search forwards for the last contiguous page to transfer 1206 */ 1207 failed = 0; 1208 for (i = 0; i < count; i++) { 1209 if (failed || 1210 (reqaddr[i] != reqaddr[0] + i * btodb(PAGE_SIZE)) || 1211 ((reqaddr[i] / dmmax) != (reqaddr[0] / dmmax)) || 1212 (rtvals[i] != VM_PAGER_OK)) { 1213 failed = 1; 1214 if (rtvals[i] == VM_PAGER_OK) 1215 rtvals[i] = VM_PAGER_AGAIN; 1216 } 1217 } 1218 1219 for (i = 0; i < count; i++) { 1220 if (rtvals[i] != VM_PAGER_OK) { 1221 if (swb[i]) 1222 --swb[i]->swb_locked; 1223 } 1224 } 1225 1226 for (i = 0; i < count; i++) 1227 if (rtvals[i] != VM_PAGER_OK) 1228 break; 1229 1230 if (i == 0) { 1231 return VM_PAGER_AGAIN; 1232 } 1233 count = i; 1234 for (i = 0; i < count; i++) { 1235 if (reqaddr[i] == SWB_EMPTY) { 1236 printf("I/O to empty block???? -- pindex: %d, i: %d\n", 1237 m[i]->pindex, i); 1238 } 1239 } 1240 1241 /* 1242 * For synchronous writes, we clean up all completed async pageouts. 1243 */ 1244 if (sync == TRUE) { 1245 swap_pager_sync(); 1246 } 1247 kva = 0; 1248 1249 /* 1250 * get a swap pager clean data structure, block until we get it 1251 */ 1252 if (swap_pager_free.tqh_first == NULL || 1253 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1254 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1255 s = splbio(); 1256 if (curproc == pageproc) { 1257 /* 1258 * pageout daemon needs a swap control block 1259 */ 1260 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT|SWAP_FREE_NEEDED; 1261 /* 1262 * if it does not get one within a short time, then 1263 * there is a potential deadlock, so we go-on trying 1264 * to free pages. 1265 */ 1266 tsleep(&swap_pager_free, PVM, "swpfre", hz/10); 1267 swap_pager_sync(); 1268 if (swap_pager_free.tqh_first == NULL || 1269 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1270 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1271 splx(s); 1272 return VM_PAGER_AGAIN; 1273 } 1274 } else 1275 pagedaemon_wakeup(); 1276 while (swap_pager_free.tqh_first == NULL || 1277 swap_pager_free.tqh_first->spc_list.tqe_next == NULL || 1278 swap_pager_free.tqh_first->spc_list.tqe_next->spc_list.tqe_next == NULL) { 1279 if (curproc == pageproc) { 1280 swap_pager_needflags |= SWAP_FREE_NEEDED_BY_PAGEOUT; 1281 if((cnt.v_free_count + cnt.v_cache_count) > cnt.v_free_reserved) 1282 wakeup(&cnt.v_free_count); 1283 } 1284 1285 swap_pager_needflags |= SWAP_FREE_NEEDED; 1286 tsleep(&swap_pager_free, PVM, "swpfre", 0); 1287 if (curproc == pageproc) 1288 swap_pager_sync(); 1289 else 1290 pagedaemon_wakeup(); 1291 } 1292 splx(s); 1293 } 1294 spc = swap_pager_free.tqh_first; 1295 TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 1296 1297 kva = spc->spc_kva; 1298 1299 /* 1300 * map our page(s) into kva for I/O 1301 */ 1302 pmap_qenter(kva, m, count); 1303 1304 /* 1305 * get the base I/O offset into the swap file 1306 */ 1307 for (i = 0; i < count; i++) { 1308 fidx = m[i]->pindex + paging_pindex; 1309 off = swap_pager_block_offset(fidx); 1310 /* 1311 * set the valid bit 1312 */ 1313 swb[i]->swb_valid |= (1 << off); 1314 /* 1315 * and unlock the data structure 1316 */ 1317 swb[i]->swb_locked--; 1318 } 1319 1320 /* 1321 * Get a swap buffer header and perform the IO 1322 */ 1323 bp = spc->spc_bp; 1324 bzero(bp, sizeof *bp); 1325 bp->b_spc = spc; 1326 bp->b_vnbufs.le_next = NOLIST; 1327 1328 bp->b_flags = B_BUSY | B_PAGING; 1329 bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 1330 bp->b_rcred = bp->b_wcred = bp->b_proc->p_ucred; 1331 if (bp->b_rcred != NOCRED) 1332 crhold(bp->b_rcred); 1333 if (bp->b_wcred != NOCRED) 1334 crhold(bp->b_wcred); 1335 bp->b_data = (caddr_t) kva; 1336 bp->b_blkno = reqaddr[0]; 1337 pbgetvp(swapdev_vp, bp); 1338 1339 bp->b_bcount = PAGE_SIZE * count; 1340 bp->b_bufsize = PAGE_SIZE * count; 1341 swapdev_vp->v_numoutput++; 1342 1343 /* 1344 * If this is an async write we set up additional buffer fields and 1345 * place a "cleaning" entry on the inuse queue. 1346 */ 1347 s = splbio(); 1348 if (sync == FALSE) { 1349 spc->spc_flags = 0; 1350 spc->spc_object = object; 1351 for (i = 0; i < count; i++) 1352 spc->spc_m[i] = m[i]; 1353 spc->spc_count = count; 1354 /* 1355 * the completion routine for async writes 1356 */ 1357 bp->b_flags |= B_CALL; 1358 bp->b_iodone = swap_pager_iodone; 1359 bp->b_dirtyoff = 0; 1360 bp->b_dirtyend = bp->b_bcount; 1361 object->un_pager.swp.swp_poip++; 1362 TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 1363 } else { 1364 object->un_pager.swp.swp_poip++; 1365 bp->b_flags |= B_CALL; 1366 bp->b_iodone = swap_pager_iodone1; 1367 } 1368 1369 cnt.v_swapout++; 1370 cnt.v_swappgsout += count; 1371 /* 1372 * perform the I/O 1373 */ 1374 VOP_STRATEGY(bp); 1375 if (sync == FALSE) { 1376 if ((bp->b_flags & B_DONE) == B_DONE) { 1377 swap_pager_sync(); 1378 } 1379 splx(s); 1380 for (i = 0; i < count; i++) { 1381 rtvals[i] = VM_PAGER_PEND; 1382 } 1383 return VM_PAGER_PEND; 1384 } 1385 /* 1386 * wait for the sync I/O to complete 1387 */ 1388 while ((bp->b_flags & B_DONE) == 0) { 1389 tsleep(bp, PVM, "swwrt", 0); 1390 } 1391 if (bp->b_flags & B_ERROR) { 1392 printf("swap_pager: I/O error - pageout failed; blkno %d, size %d, error %d\n", 1393 bp->b_blkno, bp->b_bcount, bp->b_error); 1394 rv = VM_PAGER_ERROR; 1395 } else { 1396 rv = VM_PAGER_OK; 1397 } 1398 1399 object->un_pager.swp.swp_poip--; 1400 if (object->un_pager.swp.swp_poip == 0) 1401 wakeup(object); 1402 1403 if (bp->b_vp) 1404 pbrelvp(bp); 1405 if (bp->b_flags & B_WANTED) 1406 wakeup(bp); 1407 1408 splx(s); 1409 1410 /* 1411 * remove the mapping for kernel virtual 1412 */ 1413 pmap_qremove(kva, count); 1414 1415 /* 1416 * if we have written the page, then indicate that the page is clean. 1417 */ 1418 if (rv == VM_PAGER_OK) { 1419 for (i = 0; i < count; i++) { 1420 if (rtvals[i] == VM_PAGER_OK) { 1421 pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); 1422 m[i]->dirty = 0; 1423 /* 1424 * optimization, if a page has been read 1425 * during the pageout process, we activate it. 1426 */ 1427 if ((m[i]->flags & PG_ACTIVE) == 0 && 1428 ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || 1429 pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { 1430 vm_page_activate(m[i]); 1431 } 1432 } 1433 } 1434 } else { 1435 for (i = 0; i < count; i++) { 1436 rtvals[i] = rv; 1437 } 1438 } 1439 1440 if (bp->b_rcred != NOCRED) 1441 crfree(bp->b_rcred); 1442 if (bp->b_wcred != NOCRED) 1443 crfree(bp->b_wcred); 1444 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1445 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1446 wakeup(&swap_pager_free); 1447 } 1448 if (swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1449 pagedaemon_wakeup(); 1450 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1451 return (rv); 1452} 1453 1454void 1455swap_pager_sync() 1456{ 1457 register swp_clean_t spc, tspc; 1458 register int s; 1459 1460 tspc = NULL; 1461 if (swap_pager_done.tqh_first == NULL) 1462 return; 1463 for (;;) { 1464 s = splbio(); 1465 /* 1466 * Look up and removal from done list must be done at splbio() 1467 * to avoid conflicts with swap_pager_iodone. 1468 */ 1469 while ((spc = swap_pager_done.tqh_first) != 0) { 1470 pmap_qremove(spc->spc_kva, spc->spc_count); 1471 swap_pager_finish(spc); 1472 TAILQ_REMOVE(&swap_pager_done, spc, spc_list); 1473 goto doclean; 1474 } 1475 1476 /* 1477 * No operations done, thats all we can do for now. 1478 */ 1479 1480 splx(s); 1481 break; 1482 1483 /* 1484 * The desired page was found to be busy earlier in the scan 1485 * but has since completed. 1486 */ 1487doclean: 1488 if (tspc && tspc == spc) { 1489 tspc = NULL; 1490 } 1491 spc->spc_flags = 0; 1492 TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 1493 if (swap_pager_needflags & SWAP_FREE_NEEDED) { 1494 wakeup(&swap_pager_free); 1495 } 1496 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) 1497 pagedaemon_wakeup(); 1498 swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); 1499 splx(s); 1500 } 1501 1502 return; 1503} 1504 1505void 1506swap_pager_finish(spc) 1507 register swp_clean_t spc; 1508{ 1509 vm_object_t object = spc->spc_m[0]->object; 1510 int i; 1511 1512 object->paging_in_progress -= spc->spc_count; 1513 if ((object->paging_in_progress == 0) && 1514 (object->flags & OBJ_PIPWNT)) { 1515 object->flags &= ~OBJ_PIPWNT; 1516 wakeup(object); 1517 } 1518 1519 /* 1520 * If no error, mark as clean and inform the pmap system. If error, 1521 * mark as dirty so we will try again. (XXX could get stuck doing 1522 * this, should give up after awhile) 1523 */ 1524 if (spc->spc_flags & SPC_ERROR) { 1525 for (i = 0; i < spc->spc_count; i++) { 1526 printf("swap_pager_finish: I/O error, clean of page %lx failed\n", 1527 (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); 1528 } 1529 } else { 1530 for (i = 0; i < spc->spc_count; i++) { 1531 pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); 1532 spc->spc_m[i]->dirty = 0; 1533 if ((spc->spc_m[i]->flags & PG_ACTIVE) == 0 && 1534 ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) 1535 vm_page_activate(spc->spc_m[i]); 1536 } 1537 } 1538 1539 1540 for (i = 0; i < spc->spc_count; i++) { 1541 /* 1542 * we wakeup any processes that are waiting on these pages. 1543 */ 1544 PAGE_WAKEUP(spc->spc_m[i]); 1545 } 1546 nswiodone -= spc->spc_count; 1547 1548 return; 1549} 1550 1551/* 1552 * swap_pager_iodone 1553 */ 1554static void 1555swap_pager_iodone(bp) 1556 register struct buf *bp; 1557{ 1558 register swp_clean_t spc; 1559 int s; 1560 1561 s = splbio(); 1562 spc = (swp_clean_t) bp->b_spc; 1563 TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 1564 TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); 1565 if (bp->b_flags & B_ERROR) { 1566 spc->spc_flags |= SPC_ERROR; 1567 printf("swap_pager: I/O error - async %s failed; blkno %lu, size %ld, error %d\n", 1568 (bp->b_flags & B_READ) ? "pagein" : "pageout", 1569 (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); 1570 } 1571 1572 if (bp->b_vp) 1573 pbrelvp(bp); 1574 1575 if (bp->b_flags & B_WANTED) 1576 wakeup(bp); 1577 1578 if (bp->b_rcred != NOCRED) 1579 crfree(bp->b_rcred); 1580 if (bp->b_wcred != NOCRED) 1581 crfree(bp->b_wcred); 1582 1583 nswiodone += spc->spc_count; 1584 if (--spc->spc_object->un_pager.swp.swp_poip == 0) { 1585 wakeup(spc->spc_object); 1586 } 1587 if ((swap_pager_needflags & SWAP_FREE_NEEDED) || 1588 swap_pager_inuse.tqh_first == 0) { 1589 swap_pager_needflags &= ~SWAP_FREE_NEEDED; 1590 wakeup(&swap_pager_free); 1591 } 1592 1593 if( swap_pager_needflags & SWAP_FREE_NEEDED_BY_PAGEOUT) { 1594 swap_pager_needflags &= ~SWAP_FREE_NEEDED_BY_PAGEOUT; 1595 pagedaemon_wakeup(); 1596 } 1597 1598 if (vm_pageout_pages_needed) { 1599 wakeup(&vm_pageout_pages_needed); 1600 vm_pageout_pages_needed = 0; 1601 } 1602 if ((swap_pager_inuse.tqh_first == NULL) || 1603 ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_free_min && 1604 nswiodone + cnt.v_free_count + cnt.v_cache_count >= cnt.v_free_min)) { 1605 pagedaemon_wakeup(); 1606 } 1607 splx(s); 1608}
|