uvm_anon.c revision 1.66
1/* $NetBSD: uvm_anon.c,v 1.66 2019/12/01 17:02:50 ad Exp $ */ 2 3/* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28/* 29 * uvm_anon.c: uvm anon ops 30 */ 31 32#include <sys/cdefs.h> 33__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.66 2019/12/01 17:02:50 ad Exp $"); 34 35#include "opt_uvmhist.h" 36 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/pool.h> 40#include <sys/kernel.h> 41 42#include <uvm/uvm.h> 43#include <uvm/uvm_swap.h> 44#include <uvm/uvm_pdpolicy.h> 45 46static struct pool_cache uvm_anon_cache; 47 48static int uvm_anon_ctor(void *, void *, int); 49 50void 51uvm_anon_init(void) 52{ 53 54 pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0, 55 PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor, 56 NULL, NULL); 57} 58 59static int 60uvm_anon_ctor(void *arg, void *object, int flags) 61{ 62 struct vm_anon *anon = object; 63 64 anon->an_ref = 0; 65 anon->an_lock = NULL; 66 anon->an_page = NULL; 67#if defined(VMSWAP) 68 anon->an_swslot = 0; 69#endif 70 return 0; 71} 72 73/* 74 * uvm_analloc: allocate a new anon. 75 * 76 * => anon will have no lock associated. 77 */ 78struct vm_anon * 79uvm_analloc(void) 80{ 81 struct vm_anon *anon; 82 83 anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT); 84 if (anon) { 85 KASSERT(anon->an_ref == 0); 86 KASSERT(anon->an_lock == NULL); 87 KASSERT(anon->an_page == NULL); 88#if defined(VMSWAP) 89 KASSERT(anon->an_swslot == 0); 90#endif 91 anon->an_ref = 1; 92 } 93 return anon; 94} 95 96/* 97 * uvm_anon_dispose: break loans and remove pmap mapping 98 * 99 * => anon must be removed from the amap (if anon was in an amap). 100 * => amap must be locked; we may drop and re-acquire the lock here. 101 */ 102static bool 103uvm_anon_dispose(struct vm_anon *anon) 104{ 105 struct vm_page *pg = anon->an_page; 106 107 UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist); 108 UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0); 109 110 KASSERT(mutex_owned(anon->an_lock)); 111 112 /* 113 * Dispose the page, if it is resident. 114 */ 115 116 if (pg) { 117 KASSERT(anon->an_lock != NULL); 118 119 /* 120 * If there is a resident page and it is loaned, then anon 121 * may not own it. Call out to uvm_anon_lockloanpg() to 122 * identify and lock the real owner of the page. 123 */ 124 125 if (pg->loan_count) { 126 pg = uvm_anon_lockloanpg(anon); 127 } 128 129 /* 130 * If the page is owned by a UVM object (now locked), 131 * then kill the loan on the page rather than free it, 132 * and release the object lock. 133 */ 134 135 if (pg->uobject) { 136 mutex_enter(&uvm_pageqlock); 137 KASSERT(pg->loan_count > 0); 138 pg->loan_count--; 139 pg->uanon = NULL; 140 mutex_exit(&uvm_pageqlock); 141 mutex_exit(pg->uobject->vmobjlock); 142 } else { 143 144 /* 145 * If page has no UVM object, then anon is the owner, 146 * and it is already locked. 147 */ 148 149 KASSERT((pg->flags & PG_RELEASED) == 0); 150 pmap_page_protect(pg, VM_PROT_NONE); 151 152 /* 153 * If the page is busy, mark it as PG_RELEASED, so 154 * that uvm_anon_release(9) would release it later. 155 */ 156 157 if (pg->flags & PG_BUSY) { 158 pg->flags |= PG_RELEASED; 159 mutex_obj_hold(anon->an_lock); 160 return false; 161 } 162 } 163 } 164 165#if defined(VMSWAP) 166 if (pg == NULL && anon->an_swslot > 0) { 167 /* This page is no longer only in swap. */ 168 KASSERT(uvmexp.swpgonly > 0); 169 atomic_dec_uint(&uvmexp.swpgonly); 170 } 171#endif 172 173 UVMHIST_LOG(maphist,"<- done!",0,0,0,0); 174 return true; 175} 176 177/* 178 * uvm_anon_free: free a single anon. 179 * 180 * => anon must be already disposed. 181 */ 182void 183uvm_anon_free(struct vm_anon *anon) 184{ 185 186#if defined(VMSWAP) 187 /* Free any dangling swap slot. */ 188 uvm_anon_dropswap(anon); 189#endif 190 KASSERT(anon->an_ref == 0); 191 KASSERT(anon->an_lock == NULL); 192 KASSERT(anon->an_page == NULL); 193#if defined(VMSWAP) 194 KASSERT(anon->an_swslot == 0); 195#endif 196 pool_cache_put(&uvm_anon_cache, anon); 197} 198 199/* 200 * uvm_anon_freelst: free a linked list of anon structures. 201 * 202 * => amap must be locked, we will unlock it. 203 */ 204void 205uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst) 206{ 207 struct vm_anon *anon; 208 struct vm_anon **anonp = &anonlst; 209 struct vm_page *pg; 210 211 KASSERT(mutex_owned(amap->am_lock)); 212 213 if (anonlst == NULL) { 214 amap_unlock(amap); 215 return; 216 } 217 218 /* Break loans and hardware mappings. Defer release of busy pages. */ 219 while ((anon = *anonp) != NULL) { 220 if (!uvm_anon_dispose(anon)) { 221 /* Do not free this anon. */ 222 *anonp = anon->an_link; 223 /* Note: clears an_ref as well. */ 224 anon->an_link = NULL; 225 } else { 226 anonp = &anon->an_link; 227 } 228 } 229 230 /* Free pages and leave a page replacement hint. */ 231 mutex_enter(&uvm_pageqlock); 232 for (anon = anonlst; anon != NULL; anon = anon->an_link) { 233 UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: " 234 "releasing now!", (uintptr_t)anon, 235 (uintptr_t)anon->an_page, 0, 0); 236 if ((pg = anon->an_page) != NULL) { 237 uvm_pagefree(pg); 238 } 239 uvmpdpol_anfree(anon); 240 } 241 mutex_exit(&uvm_pageqlock); 242 amap_unlock(amap); 243 244 /* Free swap space, pages and vm_anon. */ 245 while (anonlst) { 246 anon = anonlst->an_link; 247 /* Note: clears an_ref as well. */ 248 anonlst->an_link = NULL; 249 anonlst->an_lock = NULL; 250 uvm_anon_free(anonlst); 251 anonlst = anon; 252 } 253} 254 255/* 256 * uvm_anon_lockloanpg: given a locked anon, lock its resident page owner. 257 * 258 * => anon is locked by caller 259 * => on return: anon is locked 260 * if there is a resident page: 261 * if it has a uobject, it is locked by us 262 * if it is ownerless, we take over as owner 263 * we return the resident page (it can change during 264 * this function) 265 * => note that the only time an anon has an ownerless resident page 266 * is if the page was loaned from a uvm_object and the uvm_object 267 * disowned it 268 * => this only needs to be called when you want to do an operation 269 * on an anon's resident page and that page has a non-zero loan 270 * count. 271 */ 272struct vm_page * 273uvm_anon_lockloanpg(struct vm_anon *anon) 274{ 275 struct vm_page *pg; 276 bool locked = false; 277 278 KASSERT(mutex_owned(anon->an_lock)); 279 280 /* 281 * loop while we have a resident page that has a non-zero loan count. 282 * if we successfully get our lock, we will "break" the loop. 283 * note that the test for pg->loan_count is not protected -- this 284 * may produce false positive results. note that a false positive 285 * result may cause us to do more work than we need to, but it will 286 * not produce an incorrect result. 287 */ 288 289 while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) { 290 291 /* 292 * quickly check to see if the page has an object before 293 * bothering to lock the page queues. this may also produce 294 * a false positive result, but that's ok because we do a real 295 * check after that. 296 */ 297 298 if (pg->uobject) { 299 mutex_enter(&uvm_pageqlock); 300 if (pg->uobject) { 301 locked = 302 mutex_tryenter(pg->uobject->vmobjlock); 303 } else { 304 /* object disowned before we got PQ lock */ 305 locked = true; 306 } 307 mutex_exit(&uvm_pageqlock); 308 309 /* 310 * if we didn't get a lock (try lock failed), then we 311 * toggle our anon lock and try again 312 */ 313 314 if (!locked) { 315 /* 316 * someone locking the object has a chance to 317 * lock us right now 318 * 319 * XXX Better than yielding but inadequate. 320 */ 321 kpause("livelock", false, 1, anon->an_lock); 322 continue; 323 } 324 } 325 326 /* 327 * If page is un-owned i.e. the object dropped its ownership, 328 * then we have to take the ownership. 329 */ 330 331 if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) { 332 mutex_enter(&uvm_pageqlock); 333 pg->pqflags |= PQ_ANON; 334 pg->loan_count--; 335 mutex_exit(&uvm_pageqlock); 336 } 337 break; 338 } 339 return pg; 340} 341 342#if defined(VMSWAP) 343 344/* 345 * uvm_anon_pagein: fetch an anon's page. 346 * 347 * => anon must be locked, and is unlocked upon return. 348 * => returns true if pagein was aborted due to lack of memory. 349 */ 350 351bool 352uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon) 353{ 354 struct vm_page *pg; 355 struct uvm_object *uobj; 356 357 KASSERT(mutex_owned(anon->an_lock)); 358 KASSERT(anon->an_lock == amap->am_lock); 359 360 /* 361 * Get the page of the anon. 362 */ 363 364 switch (uvmfault_anonget(NULL, amap, anon)) { 365 case 0: 366 /* Success - we have the page. */ 367 KASSERT(mutex_owned(anon->an_lock)); 368 break; 369 case EIO: 370 case ERESTART: 371 /* 372 * Nothing more to do on errors. ERESTART means that the 373 * anon was freed. 374 */ 375 return false; 376 default: 377 return true; 378 } 379 380 /* 381 * Mark the page as dirty, clear its swslot and un-busy it. 382 */ 383 384 pg = anon->an_page; 385 uobj = pg->uobject; 386 if (anon->an_swslot > 0) { 387 uvm_swap_free(anon->an_swslot, 1); 388 } 389 anon->an_swslot = 0; 390 pg->flags &= ~PG_CLEAN; 391 392 /* 393 * Deactivate the page (to put it on a page queue). 394 */ 395 396 mutex_enter(&uvm_pageqlock); 397 if (pg->wire_count == 0) { 398 uvm_pagedeactivate(pg); 399 } 400 mutex_exit(&uvm_pageqlock); 401 402 if (pg->flags & PG_WANTED) { 403 pg->flags &= ~PG_WANTED; 404 wakeup(pg); 405 } 406 407 mutex_exit(anon->an_lock); 408 if (uobj) { 409 mutex_exit(uobj->vmobjlock); 410 } 411 return false; 412} 413 414/* 415 * uvm_anon_dropswap: release any swap resources from this anon. 416 * 417 * => anon must be locked or have a reference count of 0. 418 */ 419void 420uvm_anon_dropswap(struct vm_anon *anon) 421{ 422 UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist); 423 424 if (anon->an_swslot == 0) 425 return; 426 427 UVMHIST_LOG(maphist,"freeing swap for anon %#jx, paged to swslot 0x%jx", 428 (uintptr_t)anon, anon->an_swslot, 0, 0); 429 uvm_swap_free(anon->an_swslot, 1); 430 anon->an_swslot = 0; 431} 432 433#endif 434 435/* 436 * uvm_anon_release: release an anon and its page. 437 * 438 * => anon should not have any references. 439 * => anon must be locked. 440 */ 441 442void 443uvm_anon_release(struct vm_anon *anon) 444{ 445 struct vm_page *pg = anon->an_page; 446 bool success __diagused; 447 448 KASSERT(mutex_owned(anon->an_lock)); 449 KASSERT(pg != NULL); 450 KASSERT((pg->flags & PG_RELEASED) != 0); 451 KASSERT((pg->flags & PG_BUSY) != 0); 452 KASSERT(pg->uobject == NULL); 453 KASSERT(pg->uanon == anon); 454 KASSERT(pg->loan_count == 0); 455 KASSERT(anon->an_ref == 0); 456 457 mutex_enter(&uvm_pageqlock); 458 uvm_pagefree(pg); 459 mutex_exit(&uvm_pageqlock); 460 KASSERT(anon->an_page == NULL); 461 /* dispose should succeed as no one can reach this anon anymore. */ 462 success = uvm_anon_dispose(anon); 463 KASSERT(success); 464 mutex_exit(anon->an_lock); 465 /* Note: extra reference is held for PG_RELEASED case. */ 466 mutex_obj_free(anon->an_lock); 467 anon->an_lock = NULL; 468 uvm_anon_free(anon); 469} 470