sys/vm/vm_reserv.c

168 static void		vm_reserv_depopulate(vm_reserv_t rv);
170 static boolean_t	vm_reserv_has_pindex(vm_reserv_t rv,
172 static void		vm_reserv_populate(vm_reserv_t rv);
173 static void		vm_reserv_reclaim(vm_reserv_t rv);
182 	vm_reserv_t rv;
194 		TAILQ_FOREACH(rv, &vm_rvq_partpop/*[level]*/, partpopq) {
196 			unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt;
216 vm_reserv_depopulate(vm_reserv_t rv)
220 	KASSERT(rv->object != NULL,
221 	    ("vm_reserv_depopulate: reserv %p is free", rv));
222 	KASSERT(rv->popcnt > 0,
223 	    ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv));
224 	if (rv->inpartpopq) {
225 		TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
226 		rv->inpartpopq = FALSE;
228 	rv->popcnt--;
229 	if (rv->popcnt == 0) {
230 		LIST_REMOVE(rv, objq);
231 		rv->object = NULL;
232 		vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER);
235 		rv->inpartpopq = TRUE;
236 		TAILQ_INSERT_TAIL(&vm_rvq_partpop, rv, partpopq);
255 vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex)
258 	return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0);
268 vm_reserv_populate(vm_reserv_t rv)
272 	KASSERT(rv->object != NULL,
273 	    ("vm_reserv_populate: reserv %p is free", rv));
274 	KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES,
275 	    ("vm_reserv_populate: reserv %p is already full", rv));
276 	if (rv->inpartpopq) {
277 		TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
278 		rv->inpartpopq = FALSE;
280 	rv->popcnt++;
281 	if (rv->popcnt < VM_LEVEL_0_NPAGES) {
282 		rv->inpartpopq = TRUE;
283 		TAILQ_INSERT_TAIL(&vm_rvq_partpop, rv, partpopq);
297 	vm_reserv_t rv;
317 		rv = vm_reserv_from_page(mpred);
318 		if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) {
319 			m = &rv->pages[VM_RESERV_INDEX(object, pindex)];
323 			vm_reserv_populate(rv);
331 			rv = vm_reserv_from_page(msucc);
332 			if (rv->object == object &&
333 			    vm_reserv_has_pindex(rv, pindex)) {
334 				m = &rv->pages[VM_RESERV_INDEX(object, pindex)];
338 				vm_reserv_populate(rv);
356 	else if ((rv = vm_reserv_from_page(mpred))->object != object)
359 		leftcap = rv->pindex + VM_LEVEL_0_NPAGES;
366 	else if ((rv = vm_reserv_from_page(msucc))->object != object)
369 		rightcap = rv->pindex;
400 		rv = vm_reserv_from_page(m);
401 		KASSERT(rv->pages == m,
403 		    rv));
404 		KASSERT(rv->object == NULL,
405 		    ("vm_reserv_alloc_page: reserv %p isn't free", rv));
406 		LIST_INSERT_HEAD(&object->rvq, rv, objq);
407 		rv->object = object;
408 		rv->pindex = first;
409 		KASSERT(rv->popcnt == 0,
411 		    rv));
412 		KASSERT(!rv->inpartpopq,
414 		    rv));
415 		vm_reserv_populate(rv);
416 		m = &rv->pages[VM_RESERV_INDEX(object, pindex)];
427 	vm_reserv_t rv;
431 	while ((rv = LIST_FIRST(&object->rvq)) != NULL) {
432 		KASSERT(rv->object == object,
433 		    ("vm_reserv_break_all: reserv %p is corrupted", rv));
434 		if (rv->inpartpopq) {
435 			TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
436 			rv->inpartpopq = FALSE;
438 		LIST_REMOVE(rv, objq);
439 		rv->object = NULL;
441 			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
442 				vm_phys_free_pages(&rv->pages[i], 0);
444 				rv->popcnt--;
446 		KASSERT(rv->popcnt == 0,
448 		    rv));
463 	vm_reserv_t rv;
466 	rv = vm_reserv_from_page(m);
467 	if (rv->object == NULL)
470 		vm_phys_set_pool(VM_FREEPOOL_CACHE, rv->pages,
472 	vm_reserv_depopulate(rv);
509 	vm_reserv_t rv;
511 	rv = vm_reserv_from_page(m);
512 	return (rv->popcnt == VM_LEVEL_0_NPAGES ? 0 : -1);
529 	vm_reserv_t rv;
533 	rv = vm_reserv_from_page(m);
534 	if (rv->object == NULL)
538 	if (m->object == rv->object &&
539 	    m->pindex - rv->pindex == VM_RESERV_INDEX(m->object, m->pindex))
540 		vm_reserv_populate(rv);
542 		KASSERT(rv->inpartpopq,
544 		    rv));
545 		TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
546 		rv->inpartpopq = FALSE;
547 		LIST_REMOVE(rv, objq);
548 		rv->object = NULL;
550 		m_index = m - rv->pages;
552 			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
553 				vm_phys_free_pages(&rv->pages[i], 0);
555 				rv->popcnt--;
558 			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
559 				vm_phys_free_pages(&rv->pages[i], 0);
561 				rv->popcnt--;
563 		KASSERT(rv->popcnt == 0,
565 		    rv));
578 vm_reserv_reclaim(vm_reserv_t rv)
583 	KASSERT(rv->inpartpopq,
584 	    ("vm_reserv_reclaim: reserv %p's inpartpopq is corrupted", rv));
585 	TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
586 	rv->inpartpopq = FALSE;
587 	KASSERT(rv->object != NULL,
588 	    ("vm_reserv_reclaim: reserv %p is free", rv));
589 	LIST_REMOVE(rv, objq);
590 	rv->object = NULL;
592 		if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
593 			vm_phys_free_pages(&rv->pages[i], 0);
595 			rv->popcnt--;
597 	KASSERT(rv->popcnt == 0,
598 	    ("vm_reserv_reclaim: reserv %p's popcnt is corrupted", rv));
612 	vm_reserv_t rv;
615 	if ((rv = TAILQ_FIRST(&vm_rvq_partpop)) != NULL) {
616 		vm_reserv_reclaim(rv);
636 	vm_reserv_t rv;
642 	TAILQ_FOREACH(rv, &vm_rvq_partpop, partpopq) {
643 		pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]);
650 			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) {
653 					pa = VM_PAGE_TO_PHYS(&rv->pages[i]);
664 					vm_reserv_reclaim(rv);
682 	vm_reserv_t rv;
685 	rv = vm_reserv_from_page(m);
686 	if (rv->object == old_object) {
688 		if (rv->object == old_object) {
689 			LIST_REMOVE(rv, objq);
690 			LIST_INSERT_HEAD(&new_object->rvq, rv, objq);
691 			rv->object = new_object;
692 			rv->pindex -= old_object_offset;