uvm_anon.c revision 1.20
1/* $NetBSD: uvm_anon.c,v 1.20 2001/11/06 08:07:49 chs Exp $ */ 2 3/* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35/* 36 * uvm_anon.c: uvm anon ops 37 */ 38 39#include "opt_uvmhist.h" 40 41#include <sys/param.h> 42#include <sys/systm.h> 43#include <sys/proc.h> 44#include <sys/malloc.h> 45#include <sys/pool.h> 46#include <sys/kernel.h> 47 48#include <uvm/uvm.h> 49#include <uvm/uvm_swap.h> 50 51/* 52 * anonblock_list: global list of anon blocks, 53 * locked by swap_syscall_lock (since we never remove 54 * anything from this list and we only add to it via swapctl(2)). 55 */ 56 57struct uvm_anonblock { 58 LIST_ENTRY(uvm_anonblock) list; 59 int count; 60 struct vm_anon *anons; 61}; 62static LIST_HEAD(anonlist, uvm_anonblock) anonblock_list; 63 64 65static boolean_t anon_pagein __P((struct vm_anon *)); 66 67 68/* 69 * allocate anons 70 */ 71void 72uvm_anon_init() 73{ 74 int nanon = uvmexp.free - (uvmexp.free / 16); /* XXXCDC ??? */ 75 76 simple_lock_init(&uvm.afreelock); 77 LIST_INIT(&anonblock_list); 78 79 /* 80 * Allocate the initial anons. 81 */ 82 uvm_anon_add(nanon); 83} 84 85/* 86 * add some more anons to the free pool. called when we add 87 * more swap space. 88 * 89 * => swap_syscall_lock should be held (protects anonblock_list). 90 */ 91int 92uvm_anon_add(count) 93 int count; 94{ 95 struct uvm_anonblock *anonblock; 96 struct vm_anon *anon; 97 int lcv, needed; 98 99 simple_lock(&uvm.afreelock); 100 uvmexp.nanonneeded += count; 101 needed = uvmexp.nanonneeded - uvmexp.nanon; 102 simple_unlock(&uvm.afreelock); 103 104 if (needed <= 0) { 105 return 0; 106 } 107 anon = (void *)uvm_km_alloc(kernel_map, sizeof(*anon) * needed); 108 if (anon == NULL) { 109 simple_lock(&uvm.afreelock); 110 uvmexp.nanonneeded -= count; 111 simple_unlock(&uvm.afreelock); 112 return ENOMEM; 113 } 114 MALLOC(anonblock, void *, sizeof(*anonblock), M_UVMAMAP, M_WAITOK); 115 116 anonblock->count = needed; 117 anonblock->anons = anon; 118 LIST_INSERT_HEAD(&anonblock_list, anonblock, list); 119 memset(anon, 0, sizeof(*anon) * needed); 120 121 simple_lock(&uvm.afreelock); 122 uvmexp.nanon += needed; 123 uvmexp.nfreeanon += needed; 124 for (lcv = 0; lcv < needed; lcv++) { 125 simple_lock_init(&anon->an_lock); 126 anon[lcv].u.an_nxt = uvm.afree; 127 uvm.afree = &anon[lcv]; 128 simple_lock_init(&uvm.afree->an_lock); 129 } 130 simple_unlock(&uvm.afreelock); 131 return 0; 132} 133 134/* 135 * remove anons from the free pool. 136 */ 137void 138uvm_anon_remove(count) 139 int count; 140{ 141 /* 142 * we never actually free any anons, to avoid allocation overhead. 143 * XXX someday we might want to try to free anons. 144 */ 145 146 simple_lock(&uvm.afreelock); 147 uvmexp.nanonneeded -= count; 148 simple_unlock(&uvm.afreelock); 149} 150 151/* 152 * allocate an anon 153 * 154 * => new anon is returned locked! 155 */ 156struct vm_anon * 157uvm_analloc() 158{ 159 struct vm_anon *a; 160 161 simple_lock(&uvm.afreelock); 162 a = uvm.afree; 163 if (a) { 164 uvm.afree = a->u.an_nxt; 165 uvmexp.nfreeanon--; 166 a->an_ref = 1; 167 a->an_swslot = 0; 168 a->u.an_page = NULL; /* so we can free quickly */ 169 LOCK_ASSERT(simple_lock_held(&a->an_lock) == 0); 170 simple_lock(&a->an_lock); 171 } 172 simple_unlock(&uvm.afreelock); 173 return(a); 174} 175 176/* 177 * uvm_anfree: free a single anon structure 178 * 179 * => caller must remove anon from its amap before calling (if it was in 180 * an amap). 181 * => anon must be unlocked and have a zero reference count. 182 * => we may lock the pageq's. 183 */ 184 185void 186uvm_anfree(anon) 187 struct vm_anon *anon; 188{ 189 struct vm_page *pg; 190 UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist); 191 UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0); 192 193 KASSERT(anon->an_ref == 0); 194 LOCK_ASSERT(!simple_lock_held(&anon->an_lock)); 195 196 /* 197 * get page 198 */ 199 200 pg = anon->u.an_page; 201 202 /* 203 * if there is a resident page and it is loaned, then anon may not 204 * own it. call out to uvm_anon_lockpage() to ensure the real owner 205 * of the page has been identified and locked. 206 */ 207 208 if (pg && pg->loan_count) 209 pg = uvm_anon_lockloanpg(anon); 210 211 /* 212 * if we have a resident page, we must dispose of it before freeing 213 * the anon. 214 */ 215 216 if (pg) { 217 218 /* 219 * if the page is owned by a uobject (now locked), then we must 220 * kill the loan on the page rather than free it. 221 */ 222 223 if (pg->uobject) { 224 uvm_lock_pageq(); 225 KASSERT(pg->loan_count > 0); 226 pg->loan_count--; 227 pg->uanon = NULL; 228 uvm_unlock_pageq(); 229 simple_unlock(&pg->uobject->vmobjlock); 230 } else { 231 232 /* 233 * page has no uobject, so we must be the owner of it. 234 * if page is busy then we wait until it is not busy, 235 * and then free it. 236 */ 237 238 KASSERT((pg->flags & PG_RELEASED) == 0); 239 simple_lock(&anon->an_lock); 240 pmap_page_protect(pg, VM_PROT_NONE); 241 while ((pg = anon->u.an_page) && 242 (pg->flags & PG_BUSY) != 0) { 243 pg->flags |= PG_WANTED; 244 UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, 0, 245 "anfree", 0); 246 simple_lock(&anon->an_lock); 247 } 248 if (pg) { 249 uvm_lock_pageq(); 250 uvm_pagefree(pg); 251 uvm_unlock_pageq(); 252 } 253 simple_unlock(&anon->an_lock); 254 UVMHIST_LOG(maphist, "anon 0x%x, page 0x%x: " 255 "freed now!", anon, pg, 0, 0); 256 } 257 } 258 if (pg == NULL && anon->an_swslot != 0) { 259 /* this page is no longer only in swap. */ 260 simple_lock(&uvm.swap_data_lock); 261 KASSERT(uvmexp.swpgonly > 0); 262 uvmexp.swpgonly--; 263 simple_unlock(&uvm.swap_data_lock); 264 } 265 266 /* 267 * free any swap resources. 268 */ 269 270 uvm_anon_dropswap(anon); 271 272 /* 273 * now that we've stripped the data areas from the anon, 274 * free the anon itself. 275 */ 276 277 simple_lock(&uvm.afreelock); 278 anon->u.an_nxt = uvm.afree; 279 uvm.afree = anon; 280 uvmexp.nfreeanon++; 281 simple_unlock(&uvm.afreelock); 282 UVMHIST_LOG(maphist,"<- done!",0,0,0,0); 283} 284 285/* 286 * uvm_anon_dropswap: release any swap resources from this anon. 287 * 288 * => anon must be locked or have a reference count of 0. 289 */ 290void 291uvm_anon_dropswap(anon) 292 struct vm_anon *anon; 293{ 294 UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist); 295 296 if (anon->an_swslot == 0) 297 return; 298 299 UVMHIST_LOG(maphist,"freeing swap for anon %p, paged to swslot 0x%x", 300 anon, anon->an_swslot, 0, 0); 301 uvm_swap_free(anon->an_swslot, 1); 302 anon->an_swslot = 0; 303} 304 305/* 306 * uvm_anon_lockloanpg: given a locked anon, lock its resident page 307 * 308 * => anon is locked by caller 309 * => on return: anon is locked 310 * if there is a resident page: 311 * if it has a uobject, it is locked by us 312 * if it is ownerless, we take over as owner 313 * we return the resident page (it can change during 314 * this function) 315 * => note that the only time an anon has an ownerless resident page 316 * is if the page was loaned from a uvm_object and the uvm_object 317 * disowned it 318 * => this only needs to be called when you want to do an operation 319 * on an anon's resident page and that page has a non-zero loan 320 * count. 321 */ 322struct vm_page * 323uvm_anon_lockloanpg(anon) 324 struct vm_anon *anon; 325{ 326 struct vm_page *pg; 327 boolean_t locked = FALSE; 328 329 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 330 331 /* 332 * loop while we have a resident page that has a non-zero loan count. 333 * if we successfully get our lock, we will "break" the loop. 334 * note that the test for pg->loan_count is not protected -- this 335 * may produce false positive results. note that a false positive 336 * result may cause us to do more work than we need to, but it will 337 * not produce an incorrect result. 338 */ 339 340 while (((pg = anon->u.an_page) != NULL) && pg->loan_count != 0) { 341 342 /* 343 * quickly check to see if the page has an object before 344 * bothering to lock the page queues. this may also produce 345 * a false positive result, but that's ok because we do a real 346 * check after that. 347 */ 348 349 if (pg->uobject) { 350 uvm_lock_pageq(); 351 if (pg->uobject) { 352 locked = 353 simple_lock_try(&pg->uobject->vmobjlock); 354 } else { 355 /* object disowned before we got PQ lock */ 356 locked = TRUE; 357 } 358 uvm_unlock_pageq(); 359 360 /* 361 * if we didn't get a lock (try lock failed), then we 362 * toggle our anon lock and try again 363 */ 364 365 if (!locked) { 366 simple_unlock(&anon->an_lock); 367 368 /* 369 * someone locking the object has a chance to 370 * lock us right now 371 */ 372 373 simple_lock(&anon->an_lock); 374 continue; 375 } 376 } 377 378 /* 379 * if page is un-owned [i.e. the object dropped its ownership], 380 * then we can take over as owner! 381 */ 382 383 if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) { 384 uvm_lock_pageq(); 385 pg->pqflags |= PQ_ANON; 386 pg->loan_count--; 387 uvm_unlock_pageq(); 388 } 389 break; 390 } 391 return(pg); 392} 393 394 395 396/* 397 * page in every anon that is paged out to a range of swslots. 398 * 399 * swap_syscall_lock should be held (protects anonblock_list). 400 */ 401 402boolean_t 403anon_swap_off(startslot, endslot) 404 int startslot, endslot; 405{ 406 struct uvm_anonblock *anonblock; 407 408 LIST_FOREACH(anonblock, &anonblock_list, list) { 409 int i; 410 411 /* 412 * loop thru all the anons in the anonblock, 413 * paging in where needed. 414 */ 415 416 for (i = 0; i < anonblock->count; i++) { 417 struct vm_anon *anon = &anonblock->anons[i]; 418 int slot; 419 420 /* 421 * lock anon to work on it. 422 */ 423 424 simple_lock(&anon->an_lock); 425 426 /* 427 * is this anon's swap slot in range? 428 */ 429 430 slot = anon->an_swslot; 431 if (slot >= startslot && slot < endslot) { 432 boolean_t rv; 433 434 /* 435 * yup, page it in. 436 */ 437 438 /* locked: anon */ 439 rv = anon_pagein(anon); 440 /* unlocked: anon */ 441 442 if (rv) { 443 return rv; 444 } 445 } else { 446 447 /* 448 * nope, unlock and proceed. 449 */ 450 451 simple_unlock(&anon->an_lock); 452 } 453 } 454 } 455 return FALSE; 456} 457 458 459/* 460 * fetch an anon's page. 461 * 462 * => anon must be locked, and is unlocked upon return. 463 * => returns TRUE if pagein was aborted due to lack of memory. 464 */ 465 466static boolean_t 467anon_pagein(anon) 468 struct vm_anon *anon; 469{ 470 struct vm_page *pg; 471 struct uvm_object *uobj; 472 int rv; 473 474 /* locked: anon */ 475 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 476 477 rv = uvmfault_anonget(NULL, NULL, anon); 478 479 /* 480 * if rv == 0, anon is still locked, else anon 481 * is unlocked 482 */ 483 484 switch (rv) { 485 case 0: 486 break; 487 488 case EIO: 489 case ERESTART: 490 491 /* 492 * nothing more to do on errors. 493 * ERESTART can only mean that the anon was freed, 494 * so again there's nothing to do. 495 */ 496 497 return FALSE; 498 } 499 500 /* 501 * ok, we've got the page now. 502 * mark it as dirty, clear its swslot and un-busy it. 503 */ 504 505 pg = anon->u.an_page; 506 uobj = pg->uobject; 507 uvm_swap_free(anon->an_swslot, 1); 508 anon->an_swslot = 0; 509 pg->flags &= ~(PG_CLEAN); 510 511 /* 512 * deactivate the page (to put it on a page queue) 513 */ 514 515 pmap_clear_reference(pg); 516 uvm_lock_pageq(); 517 uvm_pagedeactivate(pg); 518 uvm_unlock_pageq(); 519 520 /* 521 * unlock the anon and we're done. 522 */ 523 524 simple_unlock(&anon->an_lock); 525 if (uobj) { 526 simple_unlock(&uobj->vmobjlock); 527 } 528 return FALSE; 529} 530