subr_vmem.c revision 254025
1252330Sjeff/*- 2252330Sjeff * Copyright (c)2006,2007,2008,2009 YAMAMOTO Takashi, 3252330Sjeff * Copyright (c) 2013 EMC Corp. 4252330Sjeff * All rights reserved. 5252330Sjeff * 6252330Sjeff * Redistribution and use in source and binary forms, with or without 7252330Sjeff * modification, are permitted provided that the following conditions 8252330Sjeff * are met: 9252330Sjeff * 1. Redistributions of source code must retain the above copyright 10252330Sjeff * notice, this list of conditions and the following disclaimer. 11252330Sjeff * 2. Redistributions in binary form must reproduce the above copyright 12252330Sjeff * notice, this list of conditions and the following disclaimer in the 13252330Sjeff * documentation and/or other materials provided with the distribution. 14252330Sjeff * 15252330Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16252330Sjeff * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17252330Sjeff * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18252330Sjeff * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19252330Sjeff * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20252330Sjeff * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21252330Sjeff * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22252330Sjeff * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23252330Sjeff * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24252330Sjeff * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25252330Sjeff * SUCH DAMAGE. 26252330Sjeff */ 27252330Sjeff 28252330Sjeff/* 29252330Sjeff * From: 30252330Sjeff * $NetBSD: vmem_impl.h,v 1.2 2013/01/29 21:26:24 para Exp $ 31252330Sjeff * $NetBSD: subr_vmem.c,v 1.83 2013/03/06 11:20:10 yamt Exp $ 32252330Sjeff */ 33252330Sjeff 34252330Sjeff/* 35252330Sjeff * reference: 36252330Sjeff * - Magazines and Vmem: Extending the Slab Allocator 37252330Sjeff * to Many CPUs and Arbitrary Resources 38252330Sjeff * http://www.usenix.org/event/usenix01/bonwick.html 39252330Sjeff */ 40252330Sjeff 41252330Sjeff#include <sys/cdefs.h> 42252330Sjeff__FBSDID("$FreeBSD: head/sys/kern/subr_vmem.c 254025 2013-08-07 06:21:20Z jeff $"); 43252330Sjeff 44252330Sjeff#include "opt_ddb.h" 45252330Sjeff 46252330Sjeff#include <sys/param.h> 47252330Sjeff#include <sys/systm.h> 48252330Sjeff#include <sys/kernel.h> 49252330Sjeff#include <sys/queue.h> 50252330Sjeff#include <sys/callout.h> 51252330Sjeff#include <sys/hash.h> 52252330Sjeff#include <sys/lock.h> 53252330Sjeff#include <sys/malloc.h> 54252330Sjeff#include <sys/mutex.h> 55252330Sjeff#include <sys/smp.h> 56252330Sjeff#include <sys/condvar.h> 57252330Sjeff#include <sys/taskqueue.h> 58252330Sjeff#include <sys/vmem.h> 59252330Sjeff 60252330Sjeff#include <vm/uma.h> 61252330Sjeff#include <vm/vm.h> 62252330Sjeff#include <vm/pmap.h> 63252330Sjeff#include <vm/vm_map.h> 64254025Sjeff#include <vm/vm_object.h> 65252330Sjeff#include <vm/vm_kern.h> 66252330Sjeff#include <vm/vm_extern.h> 67252330Sjeff#include <vm/vm_param.h> 68252330Sjeff#include <vm/vm_pageout.h> 69252330Sjeff 70252330Sjeff#define VMEM_MAXORDER (sizeof(vmem_size_t) * NBBY) 71252330Sjeff 72252330Sjeff#define VMEM_HASHSIZE_MIN 16 73252330Sjeff#define VMEM_HASHSIZE_MAX 131072 74252330Sjeff 75252330Sjeff#define VMEM_QCACHE_IDX_MAX 16 76252330Sjeff 77252330Sjeff#define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT) 78252330Sjeff 79252330Sjeff#define VMEM_FLAGS \ 80252330Sjeff (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | M_FIRSTFIT) 81252330Sjeff 82252330Sjeff#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) 83252330Sjeff 84252330Sjeff#define QC_NAME_MAX 16 85252330Sjeff 86252330Sjeff/* 87252330Sjeff * Data structures private to vmem. 88252330Sjeff */ 89252330SjeffMALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures"); 90252330Sjeff 91252330Sjefftypedef struct vmem_btag bt_t; 92252330Sjeff 93252330SjeffTAILQ_HEAD(vmem_seglist, vmem_btag); 94252330SjeffLIST_HEAD(vmem_freelist, vmem_btag); 95252330SjeffLIST_HEAD(vmem_hashlist, vmem_btag); 96252330Sjeff 97252330Sjeffstruct qcache { 98252330Sjeff uma_zone_t qc_cache; 99252330Sjeff vmem_t *qc_vmem; 100252330Sjeff vmem_size_t qc_size; 101252330Sjeff char qc_name[QC_NAME_MAX]; 102252330Sjeff}; 103252330Sjefftypedef struct qcache qcache_t; 104252330Sjeff#define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache)) 105252330Sjeff 106252330Sjeff#define VMEM_NAME_MAX 16 107252330Sjeff 108252330Sjeff/* vmem arena */ 109252330Sjeffstruct vmem { 110252330Sjeff struct mtx_padalign vm_lock; 111252330Sjeff struct cv vm_cv; 112252330Sjeff char vm_name[VMEM_NAME_MAX+1]; 113252330Sjeff LIST_ENTRY(vmem) vm_alllist; 114252330Sjeff struct vmem_hashlist vm_hash0[VMEM_HASHSIZE_MIN]; 115252330Sjeff struct vmem_freelist vm_freelist[VMEM_MAXORDER]; 116252330Sjeff struct vmem_seglist vm_seglist; 117252330Sjeff struct vmem_hashlist *vm_hashlist; 118252330Sjeff vmem_size_t vm_hashsize; 119252330Sjeff 120252330Sjeff /* Constant after init */ 121252330Sjeff vmem_size_t vm_qcache_max; 122252330Sjeff vmem_size_t vm_quantum_mask; 123252330Sjeff vmem_size_t vm_import_quantum; 124252330Sjeff int vm_quantum_shift; 125252330Sjeff 126252330Sjeff /* Written on alloc/free */ 127252330Sjeff LIST_HEAD(, vmem_btag) vm_freetags; 128252330Sjeff int vm_nfreetags; 129252330Sjeff int vm_nbusytag; 130252330Sjeff vmem_size_t vm_inuse; 131252330Sjeff vmem_size_t vm_size; 132252330Sjeff 133252330Sjeff /* Used on import. */ 134252330Sjeff vmem_import_t *vm_importfn; 135252330Sjeff vmem_release_t *vm_releasefn; 136252330Sjeff void *vm_arg; 137252330Sjeff 138252330Sjeff /* Space exhaustion callback. */ 139252330Sjeff vmem_reclaim_t *vm_reclaimfn; 140252330Sjeff 141252330Sjeff /* quantum cache */ 142252330Sjeff qcache_t vm_qcache[VMEM_QCACHE_IDX_MAX]; 143252330Sjeff}; 144252330Sjeff 145252330Sjeff/* boundary tag */ 146252330Sjeffstruct vmem_btag { 147252330Sjeff TAILQ_ENTRY(vmem_btag) bt_seglist; 148252330Sjeff union { 149252330Sjeff LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */ 150252330Sjeff LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */ 151252330Sjeff } bt_u; 152252330Sjeff#define bt_hashlist bt_u.u_hashlist 153252330Sjeff#define bt_freelist bt_u.u_freelist 154252330Sjeff vmem_addr_t bt_start; 155252330Sjeff vmem_size_t bt_size; 156252330Sjeff int bt_type; 157252330Sjeff}; 158252330Sjeff 159252330Sjeff#define BT_TYPE_SPAN 1 /* Allocated from importfn */ 160252330Sjeff#define BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */ 161252330Sjeff#define BT_TYPE_FREE 3 /* Available space. */ 162252330Sjeff#define BT_TYPE_BUSY 4 /* Used space. */ 163252330Sjeff#define BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC) 164252330Sjeff 165252330Sjeff#define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1) 166252330Sjeff 167252330Sjeff#if defined(DIAGNOSTIC) 168252330Sjeffstatic void vmem_check(vmem_t *); 169252330Sjeff#endif 170252330Sjeff 171252330Sjeffstatic struct callout vmem_periodic_ch; 172252330Sjeffstatic int vmem_periodic_interval; 173252330Sjeffstatic struct task vmem_periodic_wk; 174252330Sjeff 175252330Sjeffstatic struct mtx_padalign vmem_list_lock; 176252330Sjeffstatic LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); 177252330Sjeff 178252330Sjeff/* ---- misc */ 179252330Sjeff#define VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan) 180252330Sjeff#define VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv) 181252330Sjeff#define VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock) 182252330Sjeff#define VMEM_CONDVAR_BROADCAST(vm) cv_broadcast(&vm->vm_cv) 183252330Sjeff 184252330Sjeff 185252330Sjeff#define VMEM_LOCK(vm) mtx_lock(&vm->vm_lock) 186252330Sjeff#define VMEM_TRYLOCK(vm) mtx_trylock(&vm->vm_lock) 187252330Sjeff#define VMEM_UNLOCK(vm) mtx_unlock(&vm->vm_lock) 188252330Sjeff#define VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF) 189252330Sjeff#define VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock) 190252330Sjeff#define VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED); 191252330Sjeff 192252330Sjeff#define VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align))) 193252330Sjeff 194252330Sjeff#define VMEM_CROSS_P(addr1, addr2, boundary) \ 195252330Sjeff ((((addr1) ^ (addr2)) & -(boundary)) != 0) 196252330Sjeff 197252330Sjeff#define ORDER2SIZE(order) ((vmem_size_t)1 << (order)) 198252330Sjeff#define SIZE2ORDER(size) ((int)flsl(size) - 1) 199252330Sjeff 200252330Sjeff/* 201252330Sjeff * Maximum number of boundary tags that may be required to satisfy an 202252330Sjeff * allocation. Two may be required to import. Another two may be 203252330Sjeff * required to clip edges. 204252330Sjeff */ 205252330Sjeff#define BT_MAXALLOC 4 206252330Sjeff 207252330Sjeff/* 208252330Sjeff * Max free limits the number of locally cached boundary tags. We 209252330Sjeff * just want to avoid hitting the zone allocator for every call. 210252330Sjeff */ 211252330Sjeff#define BT_MAXFREE (BT_MAXALLOC * 8) 212252330Sjeff 213252330Sjeff/* Allocator for boundary tags. */ 214252330Sjeffstatic uma_zone_t vmem_bt_zone; 215252330Sjeff 216252330Sjeff/* boot time arena storage. */ 217254025Sjeffstatic struct vmem kernel_arena_storage; 218254025Sjeffstatic struct vmem kmem_arena_storage; 219252330Sjeffstatic struct vmem buffer_arena_storage; 220252330Sjeffstatic struct vmem transient_arena_storage; 221254025Sjeffvmem_t *kernel_arena = &kernel_arena_storage; 222254025Sjeffvmem_t *kmem_arena = &kmem_arena_storage; 223252330Sjeffvmem_t *buffer_arena = &buffer_arena_storage; 224252330Sjeffvmem_t *transient_arena = &transient_arena_storage; 225252330Sjeff 226252330Sjeff/* 227252330Sjeff * Fill the vmem's boundary tag cache. We guarantee that boundary tag 228252330Sjeff * allocation will not fail once bt_fill() passes. To do so we cache 229252330Sjeff * at least the maximum possible tag allocations in the arena. 230252330Sjeff */ 231252330Sjeffstatic int 232252330Sjeffbt_fill(vmem_t *vm, int flags) 233252330Sjeff{ 234252330Sjeff bt_t *bt; 235252330Sjeff 236252330Sjeff VMEM_ASSERT_LOCKED(vm); 237252330Sjeff 238252330Sjeff /* 239254025Sjeff * Only allow the kmem arena to dip into reserve tags. It is the 240254025Sjeff * vmem where new tags come from. 241254025Sjeff */ 242254025Sjeff flags &= BT_FLAGS; 243254025Sjeff if (vm != kmem_arena) 244254025Sjeff flags &= ~M_USE_RESERVE; 245254025Sjeff 246254025Sjeff /* 247252330Sjeff * Loop until we meet the reserve. To minimize the lock shuffle 248252330Sjeff * and prevent simultaneous fills we first try a NOWAIT regardless 249252330Sjeff * of the caller's flags. Specify M_NOVM so we don't recurse while 250252330Sjeff * holding a vmem lock. 251252330Sjeff */ 252252330Sjeff while (vm->vm_nfreetags < BT_MAXALLOC) { 253252330Sjeff bt = uma_zalloc(vmem_bt_zone, 254252330Sjeff (flags & M_USE_RESERVE) | M_NOWAIT | M_NOVM); 255252330Sjeff if (bt == NULL) { 256252330Sjeff VMEM_UNLOCK(vm); 257252330Sjeff bt = uma_zalloc(vmem_bt_zone, flags); 258252330Sjeff VMEM_LOCK(vm); 259252330Sjeff if (bt == NULL && (flags & M_NOWAIT) != 0) 260252330Sjeff break; 261252330Sjeff } 262252330Sjeff LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist); 263252330Sjeff vm->vm_nfreetags++; 264252330Sjeff } 265252330Sjeff 266252330Sjeff if (vm->vm_nfreetags < BT_MAXALLOC) 267252330Sjeff return ENOMEM; 268252330Sjeff 269252330Sjeff return 0; 270252330Sjeff} 271252330Sjeff 272252330Sjeff/* 273252330Sjeff * Pop a tag off of the freetag stack. 274252330Sjeff */ 275252330Sjeffstatic bt_t * 276252330Sjeffbt_alloc(vmem_t *vm) 277252330Sjeff{ 278252330Sjeff bt_t *bt; 279252330Sjeff 280252330Sjeff VMEM_ASSERT_LOCKED(vm); 281252330Sjeff bt = LIST_FIRST(&vm->vm_freetags); 282252330Sjeff MPASS(bt != NULL); 283252330Sjeff LIST_REMOVE(bt, bt_freelist); 284252330Sjeff vm->vm_nfreetags--; 285252330Sjeff 286252330Sjeff return bt; 287252330Sjeff} 288252330Sjeff 289252330Sjeff/* 290252330Sjeff * Trim the per-vmem free list. Returns with the lock released to 291252330Sjeff * avoid allocator recursions. 292252330Sjeff */ 293252330Sjeffstatic void 294252330Sjeffbt_freetrim(vmem_t *vm, int freelimit) 295252330Sjeff{ 296252330Sjeff LIST_HEAD(, vmem_btag) freetags; 297252330Sjeff bt_t *bt; 298252330Sjeff 299252330Sjeff LIST_INIT(&freetags); 300252330Sjeff VMEM_ASSERT_LOCKED(vm); 301252330Sjeff while (vm->vm_nfreetags > freelimit) { 302252330Sjeff bt = LIST_FIRST(&vm->vm_freetags); 303252330Sjeff LIST_REMOVE(bt, bt_freelist); 304252330Sjeff vm->vm_nfreetags--; 305252330Sjeff LIST_INSERT_HEAD(&freetags, bt, bt_freelist); 306252330Sjeff } 307252330Sjeff VMEM_UNLOCK(vm); 308252330Sjeff while ((bt = LIST_FIRST(&freetags)) != NULL) { 309252330Sjeff LIST_REMOVE(bt, bt_freelist); 310252330Sjeff uma_zfree(vmem_bt_zone, bt); 311252330Sjeff } 312252330Sjeff} 313252330Sjeff 314252330Sjeffstatic inline void 315252330Sjeffbt_free(vmem_t *vm, bt_t *bt) 316252330Sjeff{ 317252330Sjeff 318252330Sjeff VMEM_ASSERT_LOCKED(vm); 319252330Sjeff MPASS(LIST_FIRST(&vm->vm_freetags) != bt); 320252330Sjeff LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist); 321252330Sjeff vm->vm_nfreetags++; 322252330Sjeff} 323252330Sjeff 324252330Sjeff/* 325252330Sjeff * freelist[0] ... [1, 1] 326252330Sjeff * freelist[1] ... [2, 3] 327252330Sjeff * freelist[2] ... [4, 7] 328252330Sjeff * freelist[3] ... [8, 15] 329252330Sjeff * : 330252330Sjeff * freelist[n] ... [(1 << n), (1 << (n + 1)) - 1] 331252330Sjeff * : 332252330Sjeff */ 333252330Sjeff 334252330Sjeffstatic struct vmem_freelist * 335252330Sjeffbt_freehead_tofree(vmem_t *vm, vmem_size_t size) 336252330Sjeff{ 337252330Sjeff const vmem_size_t qsize = size >> vm->vm_quantum_shift; 338252330Sjeff const int idx = SIZE2ORDER(qsize); 339252330Sjeff 340252330Sjeff MPASS(size != 0 && qsize != 0); 341252330Sjeff MPASS((size & vm->vm_quantum_mask) == 0); 342252330Sjeff MPASS(idx >= 0); 343252330Sjeff MPASS(idx < VMEM_MAXORDER); 344252330Sjeff 345252330Sjeff return &vm->vm_freelist[idx]; 346252330Sjeff} 347252330Sjeff 348252330Sjeff/* 349252330Sjeff * bt_freehead_toalloc: return the freelist for the given size and allocation 350252330Sjeff * strategy. 351252330Sjeff * 352252330Sjeff * For M_FIRSTFIT, return the list in which any blocks are large enough 353252330Sjeff * for the requested size. otherwise, return the list which can have blocks 354252330Sjeff * large enough for the requested size. 355252330Sjeff */ 356252330Sjeffstatic struct vmem_freelist * 357252330Sjeffbt_freehead_toalloc(vmem_t *vm, vmem_size_t size, int strat) 358252330Sjeff{ 359252330Sjeff const vmem_size_t qsize = size >> vm->vm_quantum_shift; 360252330Sjeff int idx = SIZE2ORDER(qsize); 361252330Sjeff 362252330Sjeff MPASS(size != 0 && qsize != 0); 363252330Sjeff MPASS((size & vm->vm_quantum_mask) == 0); 364252330Sjeff 365252330Sjeff if (strat == M_FIRSTFIT && ORDER2SIZE(idx) != qsize) { 366252330Sjeff idx++; 367252330Sjeff /* check too large request? */ 368252330Sjeff } 369252330Sjeff MPASS(idx >= 0); 370252330Sjeff MPASS(idx < VMEM_MAXORDER); 371252330Sjeff 372252330Sjeff return &vm->vm_freelist[idx]; 373252330Sjeff} 374252330Sjeff 375252330Sjeff/* ---- boundary tag hash */ 376252330Sjeff 377252330Sjeffstatic struct vmem_hashlist * 378252330Sjeffbt_hashhead(vmem_t *vm, vmem_addr_t addr) 379252330Sjeff{ 380252330Sjeff struct vmem_hashlist *list; 381252330Sjeff unsigned int hash; 382252330Sjeff 383252330Sjeff hash = hash32_buf(&addr, sizeof(addr), 0); 384252330Sjeff list = &vm->vm_hashlist[hash % vm->vm_hashsize]; 385252330Sjeff 386252330Sjeff return list; 387252330Sjeff} 388252330Sjeff 389252330Sjeffstatic bt_t * 390252330Sjeffbt_lookupbusy(vmem_t *vm, vmem_addr_t addr) 391252330Sjeff{ 392252330Sjeff struct vmem_hashlist *list; 393252330Sjeff bt_t *bt; 394252330Sjeff 395252330Sjeff VMEM_ASSERT_LOCKED(vm); 396252330Sjeff list = bt_hashhead(vm, addr); 397252330Sjeff LIST_FOREACH(bt, list, bt_hashlist) { 398252330Sjeff if (bt->bt_start == addr) { 399252330Sjeff break; 400252330Sjeff } 401252330Sjeff } 402252330Sjeff 403252330Sjeff return bt; 404252330Sjeff} 405252330Sjeff 406252330Sjeffstatic void 407252330Sjeffbt_rembusy(vmem_t *vm, bt_t *bt) 408252330Sjeff{ 409252330Sjeff 410252330Sjeff VMEM_ASSERT_LOCKED(vm); 411252330Sjeff MPASS(vm->vm_nbusytag > 0); 412252330Sjeff vm->vm_inuse -= bt->bt_size; 413252330Sjeff vm->vm_nbusytag--; 414252330Sjeff LIST_REMOVE(bt, bt_hashlist); 415252330Sjeff} 416252330Sjeff 417252330Sjeffstatic void 418252330Sjeffbt_insbusy(vmem_t *vm, bt_t *bt) 419252330Sjeff{ 420252330Sjeff struct vmem_hashlist *list; 421252330Sjeff 422252330Sjeff VMEM_ASSERT_LOCKED(vm); 423252330Sjeff MPASS(bt->bt_type == BT_TYPE_BUSY); 424252330Sjeff 425252330Sjeff list = bt_hashhead(vm, bt->bt_start); 426252330Sjeff LIST_INSERT_HEAD(list, bt, bt_hashlist); 427252330Sjeff vm->vm_nbusytag++; 428252330Sjeff vm->vm_inuse += bt->bt_size; 429252330Sjeff} 430252330Sjeff 431252330Sjeff/* ---- boundary tag list */ 432252330Sjeff 433252330Sjeffstatic void 434252330Sjeffbt_remseg(vmem_t *vm, bt_t *bt) 435252330Sjeff{ 436252330Sjeff 437252330Sjeff TAILQ_REMOVE(&vm->vm_seglist, bt, bt_seglist); 438252330Sjeff bt_free(vm, bt); 439252330Sjeff} 440252330Sjeff 441252330Sjeffstatic void 442252330Sjeffbt_insseg(vmem_t *vm, bt_t *bt, bt_t *prev) 443252330Sjeff{ 444252330Sjeff 445252330Sjeff TAILQ_INSERT_AFTER(&vm->vm_seglist, prev, bt, bt_seglist); 446252330Sjeff} 447252330Sjeff 448252330Sjeffstatic void 449252330Sjeffbt_insseg_tail(vmem_t *vm, bt_t *bt) 450252330Sjeff{ 451252330Sjeff 452252330Sjeff TAILQ_INSERT_TAIL(&vm->vm_seglist, bt, bt_seglist); 453252330Sjeff} 454252330Sjeff 455252330Sjeffstatic void 456252330Sjeffbt_remfree(vmem_t *vm, bt_t *bt) 457252330Sjeff{ 458252330Sjeff 459252330Sjeff MPASS(bt->bt_type == BT_TYPE_FREE); 460252330Sjeff 461252330Sjeff LIST_REMOVE(bt, bt_freelist); 462252330Sjeff} 463252330Sjeff 464252330Sjeffstatic void 465252330Sjeffbt_insfree(vmem_t *vm, bt_t *bt) 466252330Sjeff{ 467252330Sjeff struct vmem_freelist *list; 468252330Sjeff 469252330Sjeff list = bt_freehead_tofree(vm, bt->bt_size); 470252330Sjeff LIST_INSERT_HEAD(list, bt, bt_freelist); 471252330Sjeff} 472252330Sjeff 473252330Sjeff/* ---- vmem internal functions */ 474252330Sjeff 475252330Sjeff/* 476252330Sjeff * Import from the arena into the quantum cache in UMA. 477252330Sjeff */ 478252330Sjeffstatic int 479252330Sjeffqc_import(void *arg, void **store, int cnt, int flags) 480252330Sjeff{ 481252330Sjeff qcache_t *qc; 482252330Sjeff vmem_addr_t addr; 483252330Sjeff int i; 484252330Sjeff 485252330Sjeff qc = arg; 486252330Sjeff flags |= M_BESTFIT; 487252330Sjeff for (i = 0; i < cnt; i++) { 488252330Sjeff if (vmem_xalloc(qc->qc_vmem, qc->qc_size, 0, 0, 0, 489252330Sjeff VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags, &addr) != 0) 490252330Sjeff break; 491252330Sjeff store[i] = (void *)addr; 492252330Sjeff /* Only guarantee one allocation. */ 493252330Sjeff flags &= ~M_WAITOK; 494252330Sjeff flags |= M_NOWAIT; 495252330Sjeff } 496252330Sjeff return i; 497252330Sjeff} 498252330Sjeff 499252330Sjeff/* 500252330Sjeff * Release memory from the UMA cache to the arena. 501252330Sjeff */ 502252330Sjeffstatic void 503252330Sjeffqc_release(void *arg, void **store, int cnt) 504252330Sjeff{ 505252330Sjeff qcache_t *qc; 506252330Sjeff int i; 507252330Sjeff 508252330Sjeff qc = arg; 509252330Sjeff for (i = 0; i < cnt; i++) 510252330Sjeff vmem_xfree(qc->qc_vmem, (vmem_addr_t)store[i], qc->qc_size); 511252330Sjeff} 512252330Sjeff 513252330Sjeffstatic void 514252330Sjeffqc_init(vmem_t *vm, vmem_size_t qcache_max) 515252330Sjeff{ 516252330Sjeff qcache_t *qc; 517252330Sjeff vmem_size_t size; 518252330Sjeff int qcache_idx_max; 519252330Sjeff int i; 520252330Sjeff 521252330Sjeff MPASS((qcache_max & vm->vm_quantum_mask) == 0); 522252330Sjeff qcache_idx_max = MIN(qcache_max >> vm->vm_quantum_shift, 523252330Sjeff VMEM_QCACHE_IDX_MAX); 524252330Sjeff vm->vm_qcache_max = qcache_idx_max << vm->vm_quantum_shift; 525252330Sjeff for (i = 0; i < qcache_idx_max; i++) { 526252330Sjeff qc = &vm->vm_qcache[i]; 527252330Sjeff size = (i + 1) << vm->vm_quantum_shift; 528252330Sjeff snprintf(qc->qc_name, sizeof(qc->qc_name), "%s-%zu", 529252330Sjeff vm->vm_name, size); 530252330Sjeff qc->qc_vmem = vm; 531252330Sjeff qc->qc_size = size; 532252330Sjeff qc->qc_cache = uma_zcache_create(qc->qc_name, size, 533252330Sjeff NULL, NULL, NULL, NULL, qc_import, qc_release, qc, 534252330Sjeff UMA_ZONE_VM); 535252330Sjeff MPASS(qc->qc_cache); 536252330Sjeff } 537252330Sjeff} 538252330Sjeff 539252330Sjeffstatic void 540252330Sjeffqc_destroy(vmem_t *vm) 541252330Sjeff{ 542252330Sjeff int qcache_idx_max; 543252330Sjeff int i; 544252330Sjeff 545252330Sjeff qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 546252330Sjeff for (i = 0; i < qcache_idx_max; i++) 547252330Sjeff uma_zdestroy(vm->vm_qcache[i].qc_cache); 548252330Sjeff} 549252330Sjeff 550252330Sjeffstatic void 551252330Sjeffqc_drain(vmem_t *vm) 552252330Sjeff{ 553252330Sjeff int qcache_idx_max; 554252330Sjeff int i; 555252330Sjeff 556252330Sjeff qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift; 557252330Sjeff for (i = 0; i < qcache_idx_max; i++) 558252330Sjeff zone_drain(vm->vm_qcache[i].qc_cache); 559252330Sjeff} 560252330Sjeff 561254025Sjeff#ifndef UMA_MD_SMALL_ALLOC 562254025Sjeff 563254025Sjeffstatic struct mtx_padalign vmem_bt_lock; 564254025Sjeff 565254025Sjeff/* 566254025Sjeff * vmem_bt_alloc: Allocate a new page of boundary tags. 567254025Sjeff * 568254025Sjeff * On architectures with uma_small_alloc there is no recursion; no address 569254025Sjeff * space need be allocated to allocate boundary tags. For the others, we 570254025Sjeff * must handle recursion. Boundary tags are necessary to allocate new 571254025Sjeff * boundary tags. 572254025Sjeff * 573254025Sjeff * UMA guarantees that enough tags are held in reserve to allocate a new 574254025Sjeff * page of kva. We dip into this reserve by specifying M_USE_RESERVE only 575254025Sjeff * when allocating the page to hold new boundary tags. In this way the 576254025Sjeff * reserve is automatically filled by the allocation that uses the reserve. 577254025Sjeff * 578254025Sjeff * We still have to guarantee that the new tags are allocated atomically since 579254025Sjeff * many threads may try concurrently. The bt_lock provides this guarantee. 580254025Sjeff * We convert WAITOK allocations to NOWAIT and then handle the blocking here 581254025Sjeff * on failure. It's ok to return NULL for a WAITOK allocation as UMA will 582254025Sjeff * loop again after checking to see if we lost the race to allocate. 583254025Sjeff * 584254025Sjeff * There is a small race between vmem_bt_alloc() returning the page and the 585254025Sjeff * zone lock being acquired to add the page to the zone. For WAITOK 586254025Sjeff * allocations we just pause briefly. NOWAIT may experience a transient 587254025Sjeff * failure. To alleviate this we permit a small number of simultaneous 588254025Sjeff * fills to proceed concurrently so NOWAIT is less likely to fail unless 589254025Sjeff * we are really out of KVA. 590254025Sjeff */ 591254025Sjeffstatic void * 592254025Sjeffvmem_bt_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait) 593254025Sjeff{ 594254025Sjeff vmem_addr_t addr; 595254025Sjeff 596254025Sjeff *pflag = UMA_SLAB_KMEM; 597254025Sjeff 598254025Sjeff /* 599254025Sjeff * Single thread boundary tag allocation so that the address space 600254025Sjeff * and memory are added in one atomic operation. 601254025Sjeff */ 602254025Sjeff mtx_lock(&vmem_bt_lock); 603254025Sjeff if (vmem_xalloc(kmem_arena, bytes, 0, 0, 0, VMEM_ADDR_MIN, 604254025Sjeff VMEM_ADDR_MAX, M_NOWAIT | M_NOVM | M_USE_RESERVE | M_BESTFIT, 605254025Sjeff &addr) == 0) { 606254025Sjeff if (kmem_back(kmem_object, addr, bytes, 607254025Sjeff M_NOWAIT | M_USE_RESERVE) == 0) { 608254025Sjeff mtx_unlock(&vmem_bt_lock); 609254025Sjeff return ((void *)addr); 610254025Sjeff } 611254025Sjeff vmem_xfree(kmem_arena, addr, bytes); 612254025Sjeff mtx_unlock(&vmem_bt_lock); 613254025Sjeff /* 614254025Sjeff * Out of memory, not address space. This may not even be 615254025Sjeff * possible due to M_USE_RESERVE page allocation. 616254025Sjeff */ 617254025Sjeff if (wait & M_WAITOK) 618254025Sjeff VM_WAIT; 619254025Sjeff return (NULL); 620254025Sjeff } 621254025Sjeff mtx_unlock(&vmem_bt_lock); 622254025Sjeff /* 623254025Sjeff * We're either out of address space or lost a fill race. 624254025Sjeff */ 625254025Sjeff if (wait & M_WAITOK) 626254025Sjeff pause("btalloc", 1); 627254025Sjeff 628254025Sjeff return (NULL); 629254025Sjeff} 630254025Sjeff#endif 631254025Sjeff 632252330Sjeffvoid 633252330Sjeffvmem_startup(void) 634252330Sjeff{ 635252330Sjeff 636252330Sjeff mtx_init(&vmem_list_lock, "vmem list lock", NULL, MTX_DEF); 637252330Sjeff vmem_bt_zone = uma_zcreate("vmem btag", 638252330Sjeff sizeof(struct vmem_btag), NULL, NULL, NULL, NULL, 639252330Sjeff UMA_ALIGN_PTR, UMA_ZONE_VM); 640254025Sjeff#ifndef UMA_MD_SMALL_ALLOC 641254025Sjeff mtx_init(&vmem_bt_lock, "btag lock", NULL, MTX_DEF); 642254025Sjeff uma_prealloc(vmem_bt_zone, BT_MAXALLOC); 643254025Sjeff /* 644254025Sjeff * Reserve enough tags to allocate new tags. We allow multiple 645254025Sjeff * CPUs to attempt to allocate new tags concurrently to limit 646254025Sjeff * false restarts in UMA. 647254025Sjeff */ 648254025Sjeff uma_zone_reserve(vmem_bt_zone, BT_MAXALLOC * (mp_ncpus + 1) / 2); 649254025Sjeff uma_zone_set_allocf(vmem_bt_zone, vmem_bt_alloc); 650254025Sjeff#endif 651252330Sjeff} 652252330Sjeff 653252330Sjeff/* ---- rehash */ 654252330Sjeff 655252330Sjeffstatic int 656252330Sjeffvmem_rehash(vmem_t *vm, vmem_size_t newhashsize) 657252330Sjeff{ 658252330Sjeff bt_t *bt; 659252330Sjeff int i; 660252330Sjeff struct vmem_hashlist *newhashlist; 661252330Sjeff struct vmem_hashlist *oldhashlist; 662252330Sjeff vmem_size_t oldhashsize; 663252330Sjeff 664252330Sjeff MPASS(newhashsize > 0); 665252330Sjeff 666252330Sjeff newhashlist = malloc(sizeof(struct vmem_hashlist) * newhashsize, 667252330Sjeff M_VMEM, M_NOWAIT); 668252330Sjeff if (newhashlist == NULL) 669252330Sjeff return ENOMEM; 670252330Sjeff for (i = 0; i < newhashsize; i++) { 671252330Sjeff LIST_INIT(&newhashlist[i]); 672252330Sjeff } 673252330Sjeff 674252330Sjeff VMEM_LOCK(vm); 675252330Sjeff oldhashlist = vm->vm_hashlist; 676252330Sjeff oldhashsize = vm->vm_hashsize; 677252330Sjeff vm->vm_hashlist = newhashlist; 678252330Sjeff vm->vm_hashsize = newhashsize; 679252330Sjeff if (oldhashlist == NULL) { 680252330Sjeff VMEM_UNLOCK(vm); 681252330Sjeff return 0; 682252330Sjeff } 683252330Sjeff for (i = 0; i < oldhashsize; i++) { 684252330Sjeff while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) { 685252330Sjeff bt_rembusy(vm, bt); 686252330Sjeff bt_insbusy(vm, bt); 687252330Sjeff } 688252330Sjeff } 689252330Sjeff VMEM_UNLOCK(vm); 690252330Sjeff 691252330Sjeff if (oldhashlist != vm->vm_hash0) { 692252330Sjeff free(oldhashlist, M_VMEM); 693252330Sjeff } 694252330Sjeff 695252330Sjeff return 0; 696252330Sjeff} 697252330Sjeff 698252330Sjeffstatic void 699252330Sjeffvmem_periodic_kick(void *dummy) 700252330Sjeff{ 701252330Sjeff 702252330Sjeff taskqueue_enqueue(taskqueue_thread, &vmem_periodic_wk); 703252330Sjeff} 704252330Sjeff 705252330Sjeffstatic void 706252330Sjeffvmem_periodic(void *unused, int pending) 707252330Sjeff{ 708252330Sjeff vmem_t *vm; 709252330Sjeff vmem_size_t desired; 710252330Sjeff vmem_size_t current; 711252330Sjeff 712252330Sjeff mtx_lock(&vmem_list_lock); 713252330Sjeff LIST_FOREACH(vm, &vmem_list, vm_alllist) { 714252330Sjeff#ifdef DIAGNOSTIC 715252330Sjeff /* Convenient time to verify vmem state. */ 716252330Sjeff VMEM_LOCK(vm); 717252330Sjeff vmem_check(vm); 718252330Sjeff VMEM_UNLOCK(vm); 719252330Sjeff#endif 720252330Sjeff desired = 1 << flsl(vm->vm_nbusytag); 721252330Sjeff desired = MIN(MAX(desired, VMEM_HASHSIZE_MIN), 722252330Sjeff VMEM_HASHSIZE_MAX); 723252330Sjeff current = vm->vm_hashsize; 724252330Sjeff 725252330Sjeff /* Grow in powers of two. Shrink less aggressively. */ 726252330Sjeff if (desired >= current * 2 || desired * 4 <= current) 727252330Sjeff vmem_rehash(vm, desired); 728252330Sjeff } 729252330Sjeff mtx_unlock(&vmem_list_lock); 730252330Sjeff 731252330Sjeff callout_reset(&vmem_periodic_ch, vmem_periodic_interval, 732252330Sjeff vmem_periodic_kick, NULL); 733252330Sjeff} 734252330Sjeff 735252330Sjeffstatic void 736252330Sjeffvmem_start_callout(void *unused) 737252330Sjeff{ 738252330Sjeff 739252330Sjeff TASK_INIT(&vmem_periodic_wk, 0, vmem_periodic, NULL); 740252330Sjeff vmem_periodic_interval = hz * 10; 741252330Sjeff callout_init(&vmem_periodic_ch, CALLOUT_MPSAFE); 742252330Sjeff callout_reset(&vmem_periodic_ch, vmem_periodic_interval, 743252330Sjeff vmem_periodic_kick, NULL); 744252330Sjeff} 745252330SjeffSYSINIT(vfs, SI_SUB_CONFIGURE, SI_ORDER_ANY, vmem_start_callout, NULL); 746252330Sjeff 747252330Sjeffstatic void 748253596Sglebiusvmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type) 749252330Sjeff{ 750252330Sjeff bt_t *btspan; 751252330Sjeff bt_t *btfree; 752252330Sjeff 753252330Sjeff MPASS(type == BT_TYPE_SPAN || type == BT_TYPE_SPAN_STATIC); 754252330Sjeff 755252330Sjeff btspan = bt_alloc(vm); 756252330Sjeff btspan->bt_type = type; 757252330Sjeff btspan->bt_start = addr; 758252330Sjeff btspan->bt_size = size; 759254025Sjeff bt_insseg_tail(vm, btspan); 760252330Sjeff 761252330Sjeff btfree = bt_alloc(vm); 762252330Sjeff btfree->bt_type = BT_TYPE_FREE; 763252330Sjeff btfree->bt_start = addr; 764252330Sjeff btfree->bt_size = size; 765252330Sjeff bt_insseg(vm, btfree, btspan); 766252330Sjeff bt_insfree(vm, btfree); 767254025Sjeff 768252330Sjeff vm->vm_size += size; 769252330Sjeff} 770252330Sjeff 771252330Sjeffstatic void 772252330Sjeffvmem_destroy1(vmem_t *vm) 773252330Sjeff{ 774252330Sjeff bt_t *bt; 775252330Sjeff 776252330Sjeff /* 777252330Sjeff * Drain per-cpu quantum caches. 778252330Sjeff */ 779252330Sjeff qc_destroy(vm); 780252330Sjeff 781252330Sjeff /* 782252330Sjeff * The vmem should now only contain empty segments. 783252330Sjeff */ 784252330Sjeff VMEM_LOCK(vm); 785252330Sjeff MPASS(vm->vm_nbusytag == 0); 786252330Sjeff 787252330Sjeff while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL) 788252330Sjeff bt_remseg(vm, bt); 789252330Sjeff 790252330Sjeff if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0) 791252330Sjeff free(vm->vm_hashlist, M_VMEM); 792252330Sjeff 793252330Sjeff bt_freetrim(vm, 0); 794252330Sjeff 795252330Sjeff VMEM_CONDVAR_DESTROY(vm); 796252330Sjeff VMEM_LOCK_DESTROY(vm); 797252330Sjeff free(vm, M_VMEM); 798252330Sjeff} 799252330Sjeff 800252330Sjeffstatic int 801252330Sjeffvmem_import(vmem_t *vm, vmem_size_t size, int flags) 802252330Sjeff{ 803252330Sjeff vmem_addr_t addr; 804252330Sjeff int error; 805252330Sjeff 806252330Sjeff if (vm->vm_importfn == NULL) 807252330Sjeff return EINVAL; 808252330Sjeff 809252330Sjeff size = roundup(size, vm->vm_import_quantum); 810252330Sjeff 811252330Sjeff /* 812252330Sjeff * Hide MAXALLOC tags so we're guaranteed to be able to add this 813252330Sjeff * span and the tag we want to allocate from it. 814252330Sjeff */ 815252330Sjeff MPASS(vm->vm_nfreetags >= BT_MAXALLOC); 816252330Sjeff vm->vm_nfreetags -= BT_MAXALLOC; 817252330Sjeff VMEM_UNLOCK(vm); 818252330Sjeff error = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr); 819252330Sjeff VMEM_LOCK(vm); 820252330Sjeff vm->vm_nfreetags += BT_MAXALLOC; 821252330Sjeff if (error) 822252330Sjeff return ENOMEM; 823252330Sjeff 824253596Sglebius vmem_add1(vm, addr, size, BT_TYPE_SPAN); 825252330Sjeff 826252330Sjeff return 0; 827252330Sjeff} 828252330Sjeff 829252330Sjeff/* 830252330Sjeff * vmem_fit: check if a bt can satisfy the given restrictions. 831252330Sjeff * 832252330Sjeff * it's a caller's responsibility to ensure the region is big enough 833252330Sjeff * before calling us. 834252330Sjeff */ 835252330Sjeffstatic int 836252330Sjeffvmem_fit(const bt_t *bt, vmem_size_t size, vmem_size_t align, 837252330Sjeff vmem_size_t phase, vmem_size_t nocross, vmem_addr_t minaddr, 838252330Sjeff vmem_addr_t maxaddr, vmem_addr_t *addrp) 839252330Sjeff{ 840252330Sjeff vmem_addr_t start; 841252330Sjeff vmem_addr_t end; 842252330Sjeff 843252330Sjeff MPASS(size > 0); 844252330Sjeff MPASS(bt->bt_size >= size); /* caller's responsibility */ 845252330Sjeff 846252330Sjeff /* 847252330Sjeff * XXX assumption: vmem_addr_t and vmem_size_t are 848252330Sjeff * unsigned integer of the same size. 849252330Sjeff */ 850252330Sjeff 851252330Sjeff start = bt->bt_start; 852252330Sjeff if (start < minaddr) { 853252330Sjeff start = minaddr; 854252330Sjeff } 855252330Sjeff end = BT_END(bt); 856252330Sjeff if (end > maxaddr) 857252330Sjeff end = maxaddr; 858252330Sjeff if (start > end) 859252330Sjeff return (ENOMEM); 860252330Sjeff 861252330Sjeff start = VMEM_ALIGNUP(start - phase, align) + phase; 862252330Sjeff if (start < bt->bt_start) 863252330Sjeff start += align; 864252330Sjeff if (VMEM_CROSS_P(start, start + size - 1, nocross)) { 865252330Sjeff MPASS(align < nocross); 866252330Sjeff start = VMEM_ALIGNUP(start - phase, nocross) + phase; 867252330Sjeff } 868252330Sjeff if (start <= end && end - start >= size - 1) { 869252330Sjeff MPASS((start & (align - 1)) == phase); 870252330Sjeff MPASS(!VMEM_CROSS_P(start, start + size - 1, nocross)); 871252330Sjeff MPASS(minaddr <= start); 872252330Sjeff MPASS(maxaddr == 0 || start + size - 1 <= maxaddr); 873252330Sjeff MPASS(bt->bt_start <= start); 874252330Sjeff MPASS(BT_END(bt) - start >= size - 1); 875252330Sjeff *addrp = start; 876252330Sjeff 877252330Sjeff return (0); 878252330Sjeff } 879252330Sjeff return (ENOMEM); 880252330Sjeff} 881252330Sjeff 882252330Sjeff/* 883252330Sjeff * vmem_clip: Trim the boundary tag edges to the requested start and size. 884252330Sjeff */ 885252330Sjeffstatic void 886252330Sjeffvmem_clip(vmem_t *vm, bt_t *bt, vmem_addr_t start, vmem_size_t size) 887252330Sjeff{ 888252330Sjeff bt_t *btnew; 889252330Sjeff bt_t *btprev; 890252330Sjeff 891252330Sjeff VMEM_ASSERT_LOCKED(vm); 892252330Sjeff MPASS(bt->bt_type == BT_TYPE_FREE); 893252330Sjeff MPASS(bt->bt_size >= size); 894252330Sjeff bt_remfree(vm, bt); 895252330Sjeff if (bt->bt_start != start) { 896252330Sjeff btprev = bt_alloc(vm); 897252330Sjeff btprev->bt_type = BT_TYPE_FREE; 898252330Sjeff btprev->bt_start = bt->bt_start; 899252330Sjeff btprev->bt_size = start - bt->bt_start; 900252330Sjeff bt->bt_start = start; 901252330Sjeff bt->bt_size -= btprev->bt_size; 902252330Sjeff bt_insfree(vm, btprev); 903252330Sjeff bt_insseg(vm, btprev, 904252330Sjeff TAILQ_PREV(bt, vmem_seglist, bt_seglist)); 905252330Sjeff } 906252330Sjeff MPASS(bt->bt_start == start); 907252330Sjeff if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) { 908252330Sjeff /* split */ 909252330Sjeff btnew = bt_alloc(vm); 910252330Sjeff btnew->bt_type = BT_TYPE_BUSY; 911252330Sjeff btnew->bt_start = bt->bt_start; 912252330Sjeff btnew->bt_size = size; 913252330Sjeff bt->bt_start = bt->bt_start + size; 914252330Sjeff bt->bt_size -= size; 915252330Sjeff bt_insfree(vm, bt); 916252330Sjeff bt_insseg(vm, btnew, 917252330Sjeff TAILQ_PREV(bt, vmem_seglist, bt_seglist)); 918252330Sjeff bt_insbusy(vm, btnew); 919252330Sjeff bt = btnew; 920252330Sjeff } else { 921252330Sjeff bt->bt_type = BT_TYPE_BUSY; 922252330Sjeff bt_insbusy(vm, bt); 923252330Sjeff } 924252330Sjeff MPASS(bt->bt_size >= size); 925252330Sjeff bt->bt_type = BT_TYPE_BUSY; 926252330Sjeff} 927252330Sjeff 928252330Sjeff/* ---- vmem API */ 929252330Sjeff 930252330Sjeffvoid 931252330Sjeffvmem_set_import(vmem_t *vm, vmem_import_t *importfn, 932252330Sjeff vmem_release_t *releasefn, void *arg, vmem_size_t import_quantum) 933252330Sjeff{ 934252330Sjeff 935252330Sjeff VMEM_LOCK(vm); 936252330Sjeff vm->vm_importfn = importfn; 937252330Sjeff vm->vm_releasefn = releasefn; 938252330Sjeff vm->vm_arg = arg; 939252330Sjeff vm->vm_import_quantum = import_quantum; 940252330Sjeff VMEM_UNLOCK(vm); 941252330Sjeff} 942252330Sjeff 943252330Sjeffvoid 944252330Sjeffvmem_set_reclaim(vmem_t *vm, vmem_reclaim_t *reclaimfn) 945252330Sjeff{ 946252330Sjeff 947252330Sjeff VMEM_LOCK(vm); 948252330Sjeff vm->vm_reclaimfn = reclaimfn; 949252330Sjeff VMEM_UNLOCK(vm); 950252330Sjeff} 951252330Sjeff 952252330Sjeff/* 953252330Sjeff * vmem_init: Initializes vmem arena. 954252330Sjeff */ 955252330Sjeffvmem_t * 956252330Sjeffvmem_init(vmem_t *vm, const char *name, vmem_addr_t base, vmem_size_t size, 957252330Sjeff vmem_size_t quantum, vmem_size_t qcache_max, int flags) 958252330Sjeff{ 959252330Sjeff int i; 960252330Sjeff 961252330Sjeff MPASS(quantum > 0); 962252330Sjeff 963252330Sjeff bzero(vm, sizeof(*vm)); 964252330Sjeff 965252330Sjeff VMEM_CONDVAR_INIT(vm, name); 966252330Sjeff VMEM_LOCK_INIT(vm, name); 967252330Sjeff vm->vm_nfreetags = 0; 968252330Sjeff LIST_INIT(&vm->vm_freetags); 969252330Sjeff strlcpy(vm->vm_name, name, sizeof(vm->vm_name)); 970252330Sjeff vm->vm_quantum_mask = quantum - 1; 971252330Sjeff vm->vm_quantum_shift = SIZE2ORDER(quantum); 972252330Sjeff MPASS(ORDER2SIZE(vm->vm_quantum_shift) == quantum); 973252330Sjeff vm->vm_nbusytag = 0; 974252330Sjeff vm->vm_size = 0; 975252330Sjeff vm->vm_inuse = 0; 976252330Sjeff qc_init(vm, qcache_max); 977252330Sjeff 978252330Sjeff TAILQ_INIT(&vm->vm_seglist); 979252330Sjeff for (i = 0; i < VMEM_MAXORDER; i++) { 980252330Sjeff LIST_INIT(&vm->vm_freelist[i]); 981252330Sjeff } 982252330Sjeff memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0)); 983252330Sjeff vm->vm_hashsize = VMEM_HASHSIZE_MIN; 984252330Sjeff vm->vm_hashlist = vm->vm_hash0; 985252330Sjeff 986252330Sjeff if (size != 0) { 987252330Sjeff if (vmem_add(vm, base, size, flags) != 0) { 988252330Sjeff vmem_destroy1(vm); 989252330Sjeff return NULL; 990252330Sjeff } 991252330Sjeff } 992252330Sjeff 993252330Sjeff mtx_lock(&vmem_list_lock); 994252330Sjeff LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); 995252330Sjeff mtx_unlock(&vmem_list_lock); 996252330Sjeff 997252330Sjeff return vm; 998252330Sjeff} 999252330Sjeff 1000252330Sjeff/* 1001252330Sjeff * vmem_create: create an arena. 1002252330Sjeff */ 1003252330Sjeffvmem_t * 1004252330Sjeffvmem_create(const char *name, vmem_addr_t base, vmem_size_t size, 1005252330Sjeff vmem_size_t quantum, vmem_size_t qcache_max, int flags) 1006252330Sjeff{ 1007252330Sjeff 1008252330Sjeff vmem_t *vm; 1009252330Sjeff 1010252330Sjeff vm = malloc(sizeof(*vm), M_VMEM, flags & (M_WAITOK|M_NOWAIT)); 1011252330Sjeff if (vm == NULL) 1012252330Sjeff return (NULL); 1013252330Sjeff if (vmem_init(vm, name, base, size, quantum, qcache_max, 1014252330Sjeff flags) == NULL) { 1015252330Sjeff free(vm, M_VMEM); 1016252330Sjeff return (NULL); 1017252330Sjeff } 1018252330Sjeff return (vm); 1019252330Sjeff} 1020252330Sjeff 1021252330Sjeffvoid 1022252330Sjeffvmem_destroy(vmem_t *vm) 1023252330Sjeff{ 1024252330Sjeff 1025252330Sjeff mtx_lock(&vmem_list_lock); 1026252330Sjeff LIST_REMOVE(vm, vm_alllist); 1027252330Sjeff mtx_unlock(&vmem_list_lock); 1028252330Sjeff 1029252330Sjeff vmem_destroy1(vm); 1030252330Sjeff} 1031252330Sjeff 1032252330Sjeffvmem_size_t 1033252330Sjeffvmem_roundup_size(vmem_t *vm, vmem_size_t size) 1034252330Sjeff{ 1035252330Sjeff 1036252330Sjeff return (size + vm->vm_quantum_mask) & ~vm->vm_quantum_mask; 1037252330Sjeff} 1038252330Sjeff 1039252330Sjeff/* 1040252330Sjeff * vmem_alloc: allocate resource from the arena. 1041252330Sjeff */ 1042252330Sjeffint 1043252330Sjeffvmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp) 1044252330Sjeff{ 1045252330Sjeff const int strat __unused = flags & VMEM_FITMASK; 1046252330Sjeff qcache_t *qc; 1047252330Sjeff 1048252330Sjeff flags &= VMEM_FLAGS; 1049252330Sjeff MPASS(size > 0); 1050252330Sjeff MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT); 1051252330Sjeff if ((flags & M_NOWAIT) == 0) 1052252330Sjeff WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_alloc"); 1053252330Sjeff 1054252330Sjeff if (size <= vm->vm_qcache_max) { 1055252330Sjeff qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift]; 1056252330Sjeff *addrp = (vmem_addr_t)uma_zalloc(qc->qc_cache, flags); 1057252330Sjeff if (*addrp == 0) 1058252330Sjeff return (ENOMEM); 1059252330Sjeff return (0); 1060252330Sjeff } 1061252330Sjeff 1062252330Sjeff return vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, 1063252330Sjeff flags, addrp); 1064252330Sjeff} 1065252330Sjeff 1066252330Sjeffint 1067252330Sjeffvmem_xalloc(vmem_t *vm, const vmem_size_t size0, vmem_size_t align, 1068252330Sjeff const vmem_size_t phase, const vmem_size_t nocross, 1069252330Sjeff const vmem_addr_t minaddr, const vmem_addr_t maxaddr, int flags, 1070252330Sjeff vmem_addr_t *addrp) 1071252330Sjeff{ 1072252330Sjeff const vmem_size_t size = vmem_roundup_size(vm, size0); 1073252330Sjeff struct vmem_freelist *list; 1074252330Sjeff struct vmem_freelist *first; 1075252330Sjeff struct vmem_freelist *end; 1076252330Sjeff vmem_size_t avail; 1077252330Sjeff bt_t *bt; 1078252330Sjeff int error; 1079252330Sjeff int strat; 1080252330Sjeff 1081252330Sjeff flags &= VMEM_FLAGS; 1082252330Sjeff strat = flags & VMEM_FITMASK; 1083252330Sjeff MPASS(size0 > 0); 1084252330Sjeff MPASS(size > 0); 1085252330Sjeff MPASS(strat == M_BESTFIT || strat == M_FIRSTFIT); 1086252330Sjeff MPASS((flags & (M_NOWAIT|M_WAITOK)) != (M_NOWAIT|M_WAITOK)); 1087252330Sjeff if ((flags & M_NOWAIT) == 0) 1088252330Sjeff WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_xalloc"); 1089252330Sjeff MPASS((align & vm->vm_quantum_mask) == 0); 1090252330Sjeff MPASS((align & (align - 1)) == 0); 1091252330Sjeff MPASS((phase & vm->vm_quantum_mask) == 0); 1092252330Sjeff MPASS((nocross & vm->vm_quantum_mask) == 0); 1093252330Sjeff MPASS((nocross & (nocross - 1)) == 0); 1094252330Sjeff MPASS((align == 0 && phase == 0) || phase < align); 1095252330Sjeff MPASS(nocross == 0 || nocross >= size); 1096252330Sjeff MPASS(minaddr <= maxaddr); 1097252330Sjeff MPASS(!VMEM_CROSS_P(phase, phase + size - 1, nocross)); 1098252330Sjeff 1099252330Sjeff if (align == 0) 1100252330Sjeff align = vm->vm_quantum_mask + 1; 1101252330Sjeff 1102252330Sjeff *addrp = 0; 1103252330Sjeff end = &vm->vm_freelist[VMEM_MAXORDER]; 1104252330Sjeff /* 1105252330Sjeff * choose a free block from which we allocate. 1106252330Sjeff */ 1107252330Sjeff first = bt_freehead_toalloc(vm, size, strat); 1108252330Sjeff VMEM_LOCK(vm); 1109252330Sjeff for (;;) { 1110252330Sjeff /* 1111252330Sjeff * Make sure we have enough tags to complete the 1112252330Sjeff * operation. 1113252330Sjeff */ 1114252330Sjeff if (vm->vm_nfreetags < BT_MAXALLOC && 1115252330Sjeff bt_fill(vm, flags) != 0) { 1116252330Sjeff error = ENOMEM; 1117252330Sjeff break; 1118252330Sjeff } 1119252330Sjeff /* 1120252330Sjeff * Scan freelists looking for a tag that satisfies the 1121252330Sjeff * allocation. If we're doing BESTFIT we may encounter 1122252330Sjeff * sizes below the request. If we're doing FIRSTFIT we 1123252330Sjeff * inspect only the first element from each list. 1124252330Sjeff */ 1125252330Sjeff for (list = first; list < end; list++) { 1126252330Sjeff LIST_FOREACH(bt, list, bt_freelist) { 1127252330Sjeff if (bt->bt_size >= size) { 1128252330Sjeff error = vmem_fit(bt, size, align, phase, 1129252330Sjeff nocross, minaddr, maxaddr, addrp); 1130252330Sjeff if (error == 0) { 1131252330Sjeff vmem_clip(vm, bt, *addrp, size); 1132252330Sjeff goto out; 1133252330Sjeff } 1134252330Sjeff } 1135252330Sjeff /* FIRST skips to the next list. */ 1136252330Sjeff if (strat == M_FIRSTFIT) 1137252330Sjeff break; 1138252330Sjeff } 1139252330Sjeff } 1140252330Sjeff /* 1141252330Sjeff * Retry if the fast algorithm failed. 1142252330Sjeff */ 1143252330Sjeff if (strat == M_FIRSTFIT) { 1144252330Sjeff strat = M_BESTFIT; 1145252330Sjeff first = bt_freehead_toalloc(vm, size, strat); 1146252330Sjeff continue; 1147252330Sjeff } 1148252330Sjeff /* 1149252330Sjeff * XXX it is possible to fail to meet restrictions with the 1150252330Sjeff * imported region. It is up to the user to specify the 1151252330Sjeff * import quantum such that it can satisfy any allocation. 1152252330Sjeff */ 1153252330Sjeff if (vmem_import(vm, size, flags) == 0) 1154252330Sjeff continue; 1155252330Sjeff 1156252330Sjeff /* 1157252330Sjeff * Try to free some space from the quantum cache or reclaim 1158252330Sjeff * functions if available. 1159252330Sjeff */ 1160252330Sjeff if (vm->vm_qcache_max != 0 || vm->vm_reclaimfn != NULL) { 1161252330Sjeff avail = vm->vm_size - vm->vm_inuse; 1162252330Sjeff VMEM_UNLOCK(vm); 1163252330Sjeff if (vm->vm_qcache_max != 0) 1164252330Sjeff qc_drain(vm); 1165252330Sjeff if (vm->vm_reclaimfn != NULL) 1166252330Sjeff vm->vm_reclaimfn(vm, flags); 1167252330Sjeff VMEM_LOCK(vm); 1168252330Sjeff /* If we were successful retry even NOWAIT. */ 1169252330Sjeff if (vm->vm_size - vm->vm_inuse > avail) 1170252330Sjeff continue; 1171252330Sjeff } 1172252330Sjeff if ((flags & M_NOWAIT) != 0) { 1173252330Sjeff error = ENOMEM; 1174252330Sjeff break; 1175252330Sjeff } 1176252330Sjeff VMEM_CONDVAR_WAIT(vm); 1177252330Sjeff } 1178252330Sjeffout: 1179252330Sjeff VMEM_UNLOCK(vm); 1180252330Sjeff if (error != 0 && (flags & M_NOWAIT) == 0) 1181252330Sjeff panic("failed to allocate waiting allocation\n"); 1182252330Sjeff 1183252330Sjeff return (error); 1184252330Sjeff} 1185252330Sjeff 1186252330Sjeff/* 1187252330Sjeff * vmem_free: free the resource to the arena. 1188252330Sjeff */ 1189252330Sjeffvoid 1190252330Sjeffvmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1191252330Sjeff{ 1192252330Sjeff qcache_t *qc; 1193252330Sjeff MPASS(size > 0); 1194252330Sjeff 1195252330Sjeff if (size <= vm->vm_qcache_max) { 1196252330Sjeff qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift]; 1197252330Sjeff uma_zfree(qc->qc_cache, (void *)addr); 1198252330Sjeff } else 1199252330Sjeff vmem_xfree(vm, addr, size); 1200252330Sjeff} 1201252330Sjeff 1202252330Sjeffvoid 1203252330Sjeffvmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) 1204252330Sjeff{ 1205252330Sjeff bt_t *bt; 1206252330Sjeff bt_t *t; 1207252330Sjeff 1208252330Sjeff MPASS(size > 0); 1209252330Sjeff 1210252330Sjeff VMEM_LOCK(vm); 1211252330Sjeff bt = bt_lookupbusy(vm, addr); 1212252330Sjeff MPASS(bt != NULL); 1213252330Sjeff MPASS(bt->bt_start == addr); 1214252330Sjeff MPASS(bt->bt_size == vmem_roundup_size(vm, size) || 1215252330Sjeff bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask); 1216252330Sjeff MPASS(bt->bt_type == BT_TYPE_BUSY); 1217252330Sjeff bt_rembusy(vm, bt); 1218252330Sjeff bt->bt_type = BT_TYPE_FREE; 1219252330Sjeff 1220252330Sjeff /* coalesce */ 1221252330Sjeff t = TAILQ_NEXT(bt, bt_seglist); 1222252330Sjeff if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1223252330Sjeff MPASS(BT_END(bt) < t->bt_start); /* YYY */ 1224252330Sjeff bt->bt_size += t->bt_size; 1225252330Sjeff bt_remfree(vm, t); 1226252330Sjeff bt_remseg(vm, t); 1227252330Sjeff } 1228252330Sjeff t = TAILQ_PREV(bt, vmem_seglist, bt_seglist); 1229252330Sjeff if (t != NULL && t->bt_type == BT_TYPE_FREE) { 1230252330Sjeff MPASS(BT_END(t) < bt->bt_start); /* YYY */ 1231252330Sjeff bt->bt_size += t->bt_size; 1232252330Sjeff bt->bt_start = t->bt_start; 1233252330Sjeff bt_remfree(vm, t); 1234252330Sjeff bt_remseg(vm, t); 1235252330Sjeff } 1236252330Sjeff 1237252330Sjeff t = TAILQ_PREV(bt, vmem_seglist, bt_seglist); 1238252330Sjeff MPASS(t != NULL); 1239252330Sjeff MPASS(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY); 1240252330Sjeff if (vm->vm_releasefn != NULL && t->bt_type == BT_TYPE_SPAN && 1241252330Sjeff t->bt_size == bt->bt_size) { 1242252330Sjeff vmem_addr_t spanaddr; 1243252330Sjeff vmem_size_t spansize; 1244252330Sjeff 1245252330Sjeff MPASS(t->bt_start == bt->bt_start); 1246252330Sjeff spanaddr = bt->bt_start; 1247252330Sjeff spansize = bt->bt_size; 1248252330Sjeff bt_remseg(vm, bt); 1249252330Sjeff bt_remseg(vm, t); 1250252330Sjeff vm->vm_size -= spansize; 1251252330Sjeff VMEM_CONDVAR_BROADCAST(vm); 1252252330Sjeff bt_freetrim(vm, BT_MAXFREE); 1253252330Sjeff (*vm->vm_releasefn)(vm->vm_arg, spanaddr, spansize); 1254252330Sjeff } else { 1255252330Sjeff bt_insfree(vm, bt); 1256252330Sjeff VMEM_CONDVAR_BROADCAST(vm); 1257252330Sjeff bt_freetrim(vm, BT_MAXFREE); 1258252330Sjeff } 1259252330Sjeff} 1260252330Sjeff 1261252330Sjeff/* 1262252330Sjeff * vmem_add: 1263252330Sjeff * 1264252330Sjeff */ 1265252330Sjeffint 1266252330Sjeffvmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int flags) 1267252330Sjeff{ 1268252330Sjeff int error; 1269252330Sjeff 1270252330Sjeff error = 0; 1271252330Sjeff flags &= VMEM_FLAGS; 1272252330Sjeff VMEM_LOCK(vm); 1273252330Sjeff if (vm->vm_nfreetags >= BT_MAXALLOC || bt_fill(vm, flags) == 0) 1274253596Sglebius vmem_add1(vm, addr, size, BT_TYPE_SPAN_STATIC); 1275252330Sjeff else 1276252330Sjeff error = ENOMEM; 1277252330Sjeff VMEM_UNLOCK(vm); 1278252330Sjeff 1279252330Sjeff return (error); 1280252330Sjeff} 1281252330Sjeff 1282252330Sjeff/* 1283252330Sjeff * vmem_size: information about arenas size 1284252330Sjeff */ 1285252330Sjeffvmem_size_t 1286252330Sjeffvmem_size(vmem_t *vm, int typemask) 1287252330Sjeff{ 1288252330Sjeff 1289252330Sjeff switch (typemask) { 1290252330Sjeff case VMEM_ALLOC: 1291252330Sjeff return vm->vm_inuse; 1292252330Sjeff case VMEM_FREE: 1293252330Sjeff return vm->vm_size - vm->vm_inuse; 1294252330Sjeff case VMEM_FREE|VMEM_ALLOC: 1295252330Sjeff return vm->vm_size; 1296252330Sjeff default: 1297252330Sjeff panic("vmem_size"); 1298252330Sjeff } 1299252330Sjeff} 1300252330Sjeff 1301252330Sjeff/* ---- debug */ 1302252330Sjeff 1303252330Sjeff#if defined(DDB) || defined(DIAGNOSTIC) 1304252330Sjeff 1305252330Sjeffstatic void bt_dump(const bt_t *, int (*)(const char *, ...) 1306252330Sjeff __printflike(1, 2)); 1307252330Sjeff 1308252330Sjeffstatic const char * 1309252330Sjeffbt_type_string(int type) 1310252330Sjeff{ 1311252330Sjeff 1312252330Sjeff switch (type) { 1313252330Sjeff case BT_TYPE_BUSY: 1314252330Sjeff return "busy"; 1315252330Sjeff case BT_TYPE_FREE: 1316252330Sjeff return "free"; 1317252330Sjeff case BT_TYPE_SPAN: 1318252330Sjeff return "span"; 1319252330Sjeff case BT_TYPE_SPAN_STATIC: 1320252330Sjeff return "static span"; 1321252330Sjeff default: 1322252330Sjeff break; 1323252330Sjeff } 1324252330Sjeff return "BOGUS"; 1325252330Sjeff} 1326252330Sjeff 1327252330Sjeffstatic void 1328252330Sjeffbt_dump(const bt_t *bt, int (*pr)(const char *, ...)) 1329252330Sjeff{ 1330252330Sjeff 1331252330Sjeff (*pr)("\t%p: %jx %jx, %d(%s)\n", 1332252330Sjeff bt, (intmax_t)bt->bt_start, (intmax_t)bt->bt_size, 1333252330Sjeff bt->bt_type, bt_type_string(bt->bt_type)); 1334252330Sjeff} 1335252330Sjeff 1336252330Sjeffstatic void 1337252330Sjeffvmem_dump(const vmem_t *vm , int (*pr)(const char *, ...) __printflike(1, 2)) 1338252330Sjeff{ 1339252330Sjeff const bt_t *bt; 1340252330Sjeff int i; 1341252330Sjeff 1342252330Sjeff (*pr)("vmem %p '%s'\n", vm, vm->vm_name); 1343252330Sjeff TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1344252330Sjeff bt_dump(bt, pr); 1345252330Sjeff } 1346252330Sjeff 1347252330Sjeff for (i = 0; i < VMEM_MAXORDER; i++) { 1348252330Sjeff const struct vmem_freelist *fl = &vm->vm_freelist[i]; 1349252330Sjeff 1350252330Sjeff if (LIST_EMPTY(fl)) { 1351252330Sjeff continue; 1352252330Sjeff } 1353252330Sjeff 1354252330Sjeff (*pr)("freelist[%d]\n", i); 1355252330Sjeff LIST_FOREACH(bt, fl, bt_freelist) { 1356252330Sjeff bt_dump(bt, pr); 1357252330Sjeff } 1358252330Sjeff } 1359252330Sjeff} 1360252330Sjeff 1361252330Sjeff#endif /* defined(DDB) || defined(DIAGNOSTIC) */ 1362252330Sjeff 1363252330Sjeff#if defined(DDB) 1364252330Sjeffstatic bt_t * 1365252330Sjeffvmem_whatis_lookup(vmem_t *vm, vmem_addr_t addr) 1366252330Sjeff{ 1367252330Sjeff bt_t *bt; 1368252330Sjeff 1369252330Sjeff TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1370252330Sjeff if (BT_ISSPAN_P(bt)) { 1371252330Sjeff continue; 1372252330Sjeff } 1373252330Sjeff if (bt->bt_start <= addr && addr <= BT_END(bt)) { 1374252330Sjeff return bt; 1375252330Sjeff } 1376252330Sjeff } 1377252330Sjeff 1378252330Sjeff return NULL; 1379252330Sjeff} 1380252330Sjeff 1381252330Sjeffvoid 1382252330Sjeffvmem_whatis(vmem_addr_t addr, int (*pr)(const char *, ...)) 1383252330Sjeff{ 1384252330Sjeff vmem_t *vm; 1385252330Sjeff 1386252330Sjeff LIST_FOREACH(vm, &vmem_list, vm_alllist) { 1387252330Sjeff bt_t *bt; 1388252330Sjeff 1389252330Sjeff bt = vmem_whatis_lookup(vm, addr); 1390252330Sjeff if (bt == NULL) { 1391252330Sjeff continue; 1392252330Sjeff } 1393252330Sjeff (*pr)("%p is %p+%zu in VMEM '%s' (%s)\n", 1394252330Sjeff (void *)addr, (void *)bt->bt_start, 1395252330Sjeff (vmem_size_t)(addr - bt->bt_start), vm->vm_name, 1396252330Sjeff (bt->bt_type == BT_TYPE_BUSY) ? "allocated" : "free"); 1397252330Sjeff } 1398252330Sjeff} 1399252330Sjeff 1400252330Sjeffvoid 1401252330Sjeffvmem_printall(const char *modif, int (*pr)(const char *, ...)) 1402252330Sjeff{ 1403252330Sjeff const vmem_t *vm; 1404252330Sjeff 1405252330Sjeff LIST_FOREACH(vm, &vmem_list, vm_alllist) { 1406252330Sjeff vmem_dump(vm, pr); 1407252330Sjeff } 1408252330Sjeff} 1409252330Sjeff 1410252330Sjeffvoid 1411252330Sjeffvmem_print(vmem_addr_t addr, const char *modif, int (*pr)(const char *, ...)) 1412252330Sjeff{ 1413252330Sjeff const vmem_t *vm = (const void *)addr; 1414252330Sjeff 1415252330Sjeff vmem_dump(vm, pr); 1416252330Sjeff} 1417252330Sjeff#endif /* defined(DDB) */ 1418252330Sjeff 1419252330Sjeff#define vmem_printf printf 1420252330Sjeff 1421252330Sjeff#if defined(DIAGNOSTIC) 1422252330Sjeff 1423252330Sjeffstatic bool 1424252330Sjeffvmem_check_sanity(vmem_t *vm) 1425252330Sjeff{ 1426252330Sjeff const bt_t *bt, *bt2; 1427252330Sjeff 1428252330Sjeff MPASS(vm != NULL); 1429252330Sjeff 1430252330Sjeff TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1431252330Sjeff if (bt->bt_start > BT_END(bt)) { 1432252330Sjeff printf("corrupted tag\n"); 1433252330Sjeff bt_dump(bt, vmem_printf); 1434252330Sjeff return false; 1435252330Sjeff } 1436252330Sjeff } 1437252330Sjeff TAILQ_FOREACH(bt, &vm->vm_seglist, bt_seglist) { 1438252330Sjeff TAILQ_FOREACH(bt2, &vm->vm_seglist, bt_seglist) { 1439252330Sjeff if (bt == bt2) { 1440252330Sjeff continue; 1441252330Sjeff } 1442252330Sjeff if (BT_ISSPAN_P(bt) != BT_ISSPAN_P(bt2)) { 1443252330Sjeff continue; 1444252330Sjeff } 1445252330Sjeff if (bt->bt_start <= BT_END(bt2) && 1446252330Sjeff bt2->bt_start <= BT_END(bt)) { 1447252330Sjeff printf("overwrapped tags\n"); 1448252330Sjeff bt_dump(bt, vmem_printf); 1449252330Sjeff bt_dump(bt2, vmem_printf); 1450252330Sjeff return false; 1451252330Sjeff } 1452252330Sjeff } 1453252330Sjeff } 1454252330Sjeff 1455252330Sjeff return true; 1456252330Sjeff} 1457252330Sjeff 1458252330Sjeffstatic void 1459252330Sjeffvmem_check(vmem_t *vm) 1460252330Sjeff{ 1461252330Sjeff 1462252330Sjeff if (!vmem_check_sanity(vm)) { 1463252330Sjeff panic("insanity vmem %p", vm); 1464252330Sjeff } 1465252330Sjeff} 1466252330Sjeff 1467252330Sjeff#endif /* defined(DIAGNOSTIC) */ 1468