sys/kern/subr_vmem.c

63 #include <vm/uma.h>
64 #include <vm/vm.h>
65 #include <vm/pmap.h>
66 #include <vm/vm_map.h>
67 #include <vm/vm_object.h>
68 #include <vm/vm_kern.h>
69 #include <vm/vm_extern.h>
70 #include <vm/vm_param.h>
71 #include <vm/vm_pageout.h>
188 #define	VMEM_CONDVAR_INIT(vm, wchan)	cv_init(&vm->vm_cv, wchan)
189 #define	VMEM_CONDVAR_DESTROY(vm)	cv_destroy(&vm->vm_cv)
190 #define	VMEM_CONDVAR_WAIT(vm)		cv_wait(&vm->vm_cv, &vm->vm_lock)
191 #define	VMEM_CONDVAR_BROADCAST(vm)	cv_broadcast(&vm->vm_cv)
194 #define	VMEM_LOCK(vm)		mtx_lock(&vm->vm_lock)
195 #define	VMEM_TRYLOCK(vm)	mtx_trylock(&vm->vm_lock)
196 #define	VMEM_UNLOCK(vm)		mtx_unlock(&vm->vm_lock)
197 #define	VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF)
198 #define	VMEM_LOCK_DESTROY(vm)	mtx_destroy(&vm->vm_lock)
199 #define	VMEM_ASSERT_LOCKED(vm)	mtx_assert(&vm->vm_lock, MA_OWNED);
248 bt_fill(vmem_t *vm, int flags)
252 	VMEM_ASSERT_LOCKED(vm);
259 	if (vm != kmem_arena)
268 	while (vm->vm_nfreetags < BT_MAXALLOC) {
272 			VMEM_UNLOCK(vm);
274 			VMEM_LOCK(vm);
278 		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
279 		vm->vm_nfreetags++;
282 	if (vm->vm_nfreetags < BT_MAXALLOC)
292 bt_alloc(vmem_t *vm)
296 	VMEM_ASSERT_LOCKED(vm);
297 	bt = LIST_FIRST(&vm->vm_freetags);
300 	vm->vm_nfreetags--;
310 bt_freetrim(vmem_t *vm, int freelimit)
316 	VMEM_ASSERT_LOCKED(vm);
317 	while (vm->vm_nfreetags > freelimit) {
318 		bt = LIST_FIRST(&vm->vm_freetags);
320 		vm->vm_nfreetags--;
323 	VMEM_UNLOCK(vm);
331 bt_free(vmem_t *vm, bt_t *bt)
334 	VMEM_ASSERT_LOCKED(vm);
335 	MPASS(LIST_FIRST(&vm->vm_freetags) != bt);
336 	LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
337 	vm->vm_nfreetags++;
354 bt_freehead_tofree(vmem_t *vm, vmem_size_t size)
356 	const vmem_size_t qsize = size >> vm->vm_quantum_shift;
360 	MPASS((size & vm->vm_quantum_mask) == 0);
364 	return &vm->vm_freelist[idx];
376 bt_freehead_toalloc(vmem_t *vm, vmem_size_t size, int strat)
378 	const vmem_size_t qsize = size >> vm->vm_quantum_shift;
382 	MPASS((size & vm->vm_quantum_mask) == 0);
391 	return &vm->vm_freelist[idx];
397 bt_hashhead(vmem_t *vm, vmem_addr_t addr)
403 	list = &vm->vm_hashlist[hash % vm->vm_hashsize];
409 bt_lookupbusy(vmem_t *vm, vmem_addr_t addr)
414 	VMEM_ASSERT_LOCKED(vm);
415 	list = bt_hashhead(vm, addr);
426 bt_rembusy(vmem_t *vm, bt_t *bt)
429 	VMEM_ASSERT_LOCKED(vm);
430 	MPASS(vm->vm_nbusytag > 0);
431 	vm->vm_inuse -= bt->bt_size;
432 	vm->vm_nbusytag--;
437 bt_insbusy(vmem_t *vm, bt_t *bt)
441 	VMEM_ASSERT_LOCKED(vm);
444 	list = bt_hashhead(vm, bt->bt_start);
446 	vm->vm_nbusytag++;
447 	vm->vm_inuse += bt->bt_size;
453 bt_remseg(vmem_t *vm, bt_t *bt)
456 	TAILQ_REMOVE(&vm->vm_seglist, bt, bt_seglist);
457 	bt_free(vm, bt);
461 bt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev)
464 	TAILQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist);
468 bt_insseg_tail(vmem_t *vm, bt_t *bt)
471 	TAILQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist);
475 bt_remfree(vmem_t *vm, bt_t *bt)
484 bt_insfree(vmem_t *vm, bt_t *bt)
488 	list = bt_freehead_tofree(vm, bt->bt_size);
534 qc_init(vmem_t *vm, vmem_size_t qcache_max)
541 	MPASS((qcache_max & vm->vm_quantum_mask) == 0);
542 	qcache_idx_max = MIN(qcache_max >> vm->vm_quantum_shift,
544 	vm->vm_qcache_max = qcache_idx_max << vm->vm_quantum_shift;
546 		qc = &vm->vm_qcache[i];
547 		size = (i + 1) << vm->vm_quantum_shift;
549 		    vm->vm_name, size);
550 		qc->qc_vmem = vm;
560 qc_destroy(vmem_t *vm)
565 	qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
567 		uma_zdestroy(vm->vm_qcache[i].qc_cache);
571 qc_drain(vmem_t *vm)
576 	qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
578 		zone_drain(vm->vm_qcache[i].qc_cache);
676 vmem_rehash(vmem_t *vm, vmem_size_t newhashsize)
694 	VMEM_LOCK(vm);
695 	oldhashlist = vm->vm_hashlist;
696 	oldhashsize = vm->vm_hashsize;
697 	vm->vm_hashlist = newhashlist;
698 	vm->vm_hashsize = newhashsize;
700 		VMEM_UNLOCK(vm);
705 			bt_rembusy(vm, bt);
706 			bt_insbusy(vm, bt);
709 	VMEM_UNLOCK(vm);
711 	if (oldhashlist != vm->vm_hash0) {
728 	vmem_t *vm;
733 	LIST_FOREACH(vm, &vmem_list, vm_alllist) {
737 			VMEM_LOCK(vm);
738 			vmem_check(vm);
739 			VMEM_UNLOCK(vm);
742 		desired = 1 << flsl(vm->vm_nbusytag);
745 		current = vm->vm_hashsize;
749 			vmem_rehash(vm, desired);
755 		VMEM_CONDVAR_BROADCAST(vm);
776 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type)
782 	MPASS((size & vm->vm_quantum_mask) == 0);
784 	btspan = bt_alloc(vm);
788 	bt_insseg_tail(vm, btspan);
790 	btfree = bt_alloc(vm);
794 	bt_insseg(vm, btfree, btspan);
795 	bt_insfree(vm, btfree);
797 	vm->vm_size += size;
801 vmem_destroy1(vmem_t *vm)
808 	qc_destroy(vm);
813 	VMEM_LOCK(vm);
814 	MPASS(vm->vm_nbusytag == 0);
816 	while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL)
817 		bt_remseg(vm, bt);
819 	if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0)
820 		free(vm->vm_hashlist, M_VMEM);
822 	bt_freetrim(vm, 0);
824 	VMEM_CONDVAR_DESTROY(vm);
825 	VMEM_LOCK_DESTROY(vm);
826 	free(vm, M_VMEM);
830 vmem_import(vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)
835 	if (vm->vm_importfn == NULL)
842 	if (align != vm->vm_quantum_mask + 1)
844 	size = roundup(size, vm->vm_import_quantum);
850 	MPASS(vm->vm_nfreetags >= BT_MAXALLOC);
851 	vm->vm_nfreetags -= BT_MAXALLOC;
852 	VMEM_UNLOCK(vm);
853 	error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
854 	VMEM_LOCK(vm);
855 	vm->vm_nfreetags += BT_MAXALLOC;
859 	vmem_add1(vm, addr, size, BT_TYPE_SPAN);
921 vmem_clip(vmem_t *vm, bt_t *bt, vmem_addr_t start, vmem_size_t size)
926 	VMEM_ASSERT_LOCKED(vm);
929 	bt_remfree(vm, bt);
931 		btprev = bt_alloc(vm);
937 		bt_insfree(vm, btprev);
938 		bt_insseg(vm, btprev,
942 	if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) {
944 		btnew = bt_alloc(vm);
950 		bt_insfree(vm, bt);
951 		bt_insseg(vm, btnew,
953 		bt_insbusy(vm, btnew);
957 		bt_insbusy(vm, bt);
966 vmem_set_import(vmem_t *vm, vmem_import_t *importfn,
970 	VMEM_LOCK(vm);
971 	vm->vm_importfn = importfn;
972 	vm->vm_releasefn = releasefn;
973 	vm->vm_arg = arg;
974 	vm->vm_import_quantum = import_quantum;
975 	VMEM_UNLOCK(vm);
979 vmem_set_reclaim(vmem_t *vm, vmem_reclaim_t *reclaimfn)
982 	VMEM_LOCK(vm);
983 	vm->vm_reclaimfn = reclaimfn;
984 	VMEM_UNLOCK(vm);
991 vmem_init(vmem_t *vm, const char *name, vmem_addr_t base, vmem_size_t size,
999 	bzero(vm, sizeof(*vm));
1001 	VMEM_CONDVAR_INIT(vm, name);
1002 	VMEM_LOCK_INIT(vm, name);
1003 	vm->vm_nfreetags = 0;
1004 	LIST_INIT(&vm->vm_freetags);
1005 	strlcpy(vm->vm_name, name, sizeof(vm->vm_name));
1006 	vm->vm_quantum_mask = quantum - 1;
1007 	vm->vm_quantum_shift = flsl(quantum) - 1;
1008 	vm->vm_nbusytag = 0;
1009 	vm->vm_size = 0;
1010 	vm->vm_inuse = 0;
1011 	qc_init(vm, qcache_max);
1013 	TAILQ_INIT(&vm->vm_seglist);
1015 		LIST_INIT(&vm->vm_freelist[i]);
1017 	memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));
1018 	vm->vm_hashsize = VMEM_HASHSIZE_MIN;
1019 	vm->vm_hashlist = vm->vm_hash0;
1022 		if (vmem_add(vm, base, size, flags) != 0) {
1023 			vmem_destroy1(vm);
1029 	LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist);
1032 	return vm;
1043 	vmem_t *vm;
1045 	vm = malloc(sizeof(*vm), M_VMEM, flags & (M_WAITOK|M_NOWAIT));
1046 	if (vm == NULL)
1048 	if (vmem_init(vm, name, base, size, quantum, qcache_max,
1051 	return (vm);
1055 vmem_destroy(vmem_t *vm)
1059 	LIST_REMOVE(vm, vm_alllist);
1062 	vmem_destroy1(vm);
1066 vmem_roundup_size(vmem_t *vm, vmem_size_t size)
1069 	return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask;
1076 vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp)
1087 	if (size <= vm->vm_qcache_max) {
1088 		qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
1095 	return vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX,
1100 vmem_xalloc(vmem_t *vm, const vmem_size_t size0, vmem_size_t align,
1105 	const vmem_size_t size = vmem_roundup_size(vm, size0);
1122 	MPASS((align & vm->vm_quantum_mask) == 0);
1124 	MPASS((phase & vm->vm_quantum_mask) == 0);
1125 	MPASS((nocross & vm->vm_quantum_mask) == 0);
1133 		align = vm->vm_quantum_mask + 1;
1136 	end = &vm->vm_freelist[VMEM_MAXORDER];
1140 	first = bt_freehead_toalloc(vm, size, strat);
1141 	VMEM_LOCK(vm);
1147 		if (vm->vm_nfreetags < BT_MAXALLOC &&
1148 		    bt_fill(vm, flags) != 0) {
1164 						vmem_clip(vm, bt, *addrp, size);
1178 			first = bt_freehead_toalloc(vm, size, strat);
1186 		if (vmem_import(vm, size, align, flags) == 0)
1193 		if (vm->vm_qcache_max != 0 || vm->vm_reclaimfn != NULL) {
1194 			avail = vm->vm_size - vm->vm_inuse;
1195 			VMEM_UNLOCK(vm);
1196 			if (vm->vm_qcache_max != 0)
1197 				qc_drain(vm);
1198 			if (vm->vm_reclaimfn != NULL)
1199 				vm->vm_reclaimfn(vm, flags);
1200 			VMEM_LOCK(vm);
1202 			if (vm->vm_size - vm->vm_inuse > avail)
1209 		VMEM_CONDVAR_WAIT(vm);
1212 	VMEM_UNLOCK(vm);
1223 vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
1228 	if (size <= vm->vm_qcache_max) {
1229 		qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift];
1232 		vmem_xfree(vm, addr, size);
1236 vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
1243 	VMEM_LOCK(vm);
1244 	bt = bt_lookupbusy(vm, addr);
1247 	MPASS(bt->bt_size == vmem_roundup_size(vm, size) ||
1248 	    bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask);
1250 	bt_rembusy(vm, bt);
1258 		bt_remfree(vm, t);
1259 		bt_remseg(vm, t);
1266 		bt_remfree(vm, t);
1267 		bt_remseg(vm, t);
1273 	if (vm->vm_releasefn != NULL && t->bt_type == BT_TYPE_SPAN &&
1281 		bt_remseg(vm, bt);
1282 		bt_remseg(vm, t);
1283 		vm->vm_size -= spansize;
1284 		VMEM_CONDVAR_BROADCAST(vm);
1285 		bt_freetrim(vm, BT_MAXFREE);
1286 		(*vm->vm_releasefn)(vm->vm_arg, spanaddr, spansize);
1288 		bt_insfree(vm, bt);
1289 		VMEM_CONDVAR_BROADCAST(vm);
1290 		bt_freetrim(vm, BT_MAXFREE);
1299 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int flags)
1305 	VMEM_LOCK(vm);
1306 	if (vm->vm_nfreetags >= BT_MAXALLOC || bt_fill(vm, flags) == 0)
1307 		vmem_add1(vm, addr, size, BT_TYPE_SPAN_STATIC);
1310 	VMEM_UNLOCK(vm);
1319 vmem_size(vmem_t *vm, int typemask)
1325 		return vm->vm_inuse;
1327 		return vm->vm_size - vm->vm_inuse;
1329 		return vm->vm_size;
1331 		VMEM_LOCK(vm);
1333 			if (LIST_EMPTY(&vm->vm_freelist[i]))
1335 			VMEM_UNLOCK(vm);
1337 			    vm->vm_quantum_shift);
1339 		VMEM_UNLOCK(vm);
1382 vmem_dump(const vmem_t *vm , int (*pr)(const char *, ...) __printflike(1, 2))
1387 	(*pr)("vmem %p '%s'\n", vm, vm->vm_name);
1388 	TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1393 		const struct vmem_freelist *fl = &vm->vm_freelist[i];
1412 vmem_whatis_lookup(vmem_t *vm, vmem_addr_t addr)
1416 	TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1431 	vmem_t *vm;
1433 	LIST_FOREACH(vm, &vmem_list, vm_alllist) {
1436 		bt = vmem_whatis_lookup(vm, addr);
1442 		    (vmem_size_t)(addr - bt->bt_start), vm->vm_name,
1450 	const vmem_t *vm;
1452 	LIST_FOREACH(vm, &vmem_list, vm_alllist) {
1453 		vmem_dump(vm, pr);
1460 	const vmem_t *vm = (const void *)addr;
1462 	vmem_dump(vm, pr);
1478 	const vmem_t *vm;
1480 	LIST_FOREACH(vm, &vmem_list, vm_alllist)
1481 		vmem_dump(vm, db_printf);
1486 	const vmem_t *vm = (const void *)addr;
1497 	db_printf("vmem %p '%s'\n", vm, vm->vm_name);
1498 	db_printf("\tquantum:\t%zu\n", vm->vm_quantum_mask + 1);
1499 	db_printf("\tsize:\t%zu\n", vm->vm_size);
1500 	db_printf("\tinuse:\t%zu\n", vm->vm_inuse);
1501 	db_printf("\tfree:\t%zu\n", vm->vm_size - vm->vm_inuse);
1502 	db_printf("\tbusy tags:\t%d\n", vm->vm_nbusytag);
1503 	db_printf("\tfree tags:\t%d\n", vm->vm_nfreetags);
1509 	TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1510 		ord = SIZE2ORDER(bt->bt_size >> vm->vm_quantum_shift);
1524 		    ORDER2SIZE(ord) << vm->vm_quantum_shift,
1531 	const vmem_t *vm;
1533 	LIST_FOREACH(vm, &vmem_list, vm_alllist)
1534 		vmem_summ((db_expr_t)vm, TRUE, count, modif);
1543 vmem_check_sanity(vmem_t *vm)
1547 	MPASS(vm != NULL);
1549 	TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1556 	TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) {
1557 		TAILQ_FOREACH(bt2, &vm->vm_seglist, bt_seglist) {
1578 vmem_check(vmem_t *vm)
1581 	if (!vmem_check_sanity(vm)) {
1582 		panic("insanity vmem %p", vm);