vfs_bio.c revision 1.120
1/* $NetBSD: vfs_bio.c,v 1.120 2004/03/25 08:22:31 simonb Exp $ */ 2 3/*- 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 37 */ 38 39/*- 40 * Copyright (c) 1994 Christopher G. Demetriou 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 71 */ 72 73/* 74 * Some references: 75 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 76 * Leffler, et al.: The Design and Implementation of the 4.3BSD 77 * UNIX Operating System (Addison Welley, 1989) 78 */ 79 80#include "opt_bufcache.h" 81#include "opt_softdep.h" 82 83#include <sys/cdefs.h> 84__KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.120 2004/03/25 08:22:31 simonb Exp $"); 85 86#include <sys/param.h> 87#include <sys/systm.h> 88#include <sys/kernel.h> 89#include <sys/proc.h> 90#include <sys/buf.h> 91#include <sys/vnode.h> 92#include <sys/mount.h> 93#include <sys/malloc.h> 94#include <sys/resourcevar.h> 95#include <sys/sysctl.h> 96#include <sys/conf.h> 97 98#include <uvm/uvm.h> 99 100#include <miscfs/specfs/specdev.h> 101 102#ifndef BUFPAGES 103# define BUFPAGES 0 104#endif 105 106#ifdef BUFCACHE 107# if (BUFCACHE < 5) || (BUFCACHE > 95) 108# error BUFCACHE is not between 5 and 95 109# endif 110#else 111# define BUFCACHE 15 112#endif 113 114u_int nbuf; /* XXX - for softdep_lockedbufs */ 115u_int bufpages = BUFPAGES; /* optional hardwired count */ 116u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */ 117 118 119/* Macros to clear/set/test flags. */ 120#define SET(t, f) (t) |= (f) 121#define CLR(t, f) (t) &= ~(f) 122#define ISSET(t, f) ((t) & (f)) 123 124/* 125 * Definitions for the buffer hash lists. 126 */ 127#define BUFHASH(dvp, lbn) \ 128 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) 129LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; 130u_long bufhash; 131#ifndef SOFTDEP 132struct bio_ops bioops; /* I/O operation notification */ 133#endif 134 135/* 136 * Insq/Remq for the buffer hash lists. 137 */ 138#define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash) 139#define bremhash(bp) LIST_REMOVE(bp, b_hash) 140 141/* 142 * Definitions for the buffer free lists. 143 */ 144#define BQUEUES 3 /* number of free buffer queues */ 145 146#define BQ_LOCKED 0 /* super-blocks &c */ 147#define BQ_LRU 1 /* lru, useful buffers */ 148#define BQ_AGE 2 /* rubbish */ 149 150TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES]; 151int needbuffer; 152 153/* 154 * Buffer queue lock. 155 * Take this lock first if also taking some buffer's b_interlock. 156 */ 157struct simplelock bqueue_slock = SIMPLELOCK_INITIALIZER; 158 159/* 160 * Buffer pool for I/O buffers. 161 */ 162struct pool bufpool; 163 164/* XXX - somewhat gross.. */ 165#if MAXBSIZE == 0x2000 166#define NMEMPOOLS 4 167#elif MAXBSIZE == 0x4000 168#define NMEMPOOLS 5 169#elif MAXBSIZE == 0x8000 170#define NMEMPOOLS 6 171#else 172#define NMEMPOOLS 7 173#endif 174 175#define MEMPOOL_INDEX_OFFSET 10 /* smallest pool is 1k */ 176#if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE 177#error update vfs_bio buffer memory parameters 178#endif 179 180/* Buffer memory pools */ 181static struct pool bmempools[NMEMPOOLS]; 182 183struct vm_map *buf_map; 184 185/* 186 * Buffer memory pool allocator. 187 */ 188static void * 189bufpool_page_alloc(struct pool *pp, int flags) 190{ 191 192 return (void *)uvm_km_kmemalloc1(buf_map, 193 uvm.kernel_object, MAXBSIZE, MAXBSIZE, UVM_UNKNOWN_OFFSET, 194 (flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK); 195} 196 197static void 198bufpool_page_free(struct pool *pp, void *v) 199{ 200 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE); 201} 202 203static struct pool_allocator bufmempool_allocator = { 204 bufpool_page_alloc, bufpool_page_free, MAXBSIZE, 205}; 206 207/* Buffer memory management variables */ 208u_long bufmem_valimit; 209u_long bufmem_hiwater; 210u_long bufmem_lowater; 211u_long bufmem; 212 213/* 214 * MD code can call this to set a hard limit on the amount 215 * of virtual memory used by the buffer cache. 216 */ 217int 218buf_setvalimit(vsize_t sz) 219{ 220 221 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */ 222 if (sz < NMEMPOOLS * MAXBSIZE) 223 return EINVAL; 224 225 bufmem_valimit = sz; 226 return 0; 227} 228 229static int buf_trim(void); 230 231/* 232 * bread()/breadn() helper. 233 */ 234static __inline struct buf *bio_doread(struct vnode *, daddr_t, int, 235 struct ucred *, int); 236int count_lock_queue(void); 237 238/* 239 * Insq/Remq for the buffer free lists. 240 * Call with buffer queue locked. 241 */ 242#define binsheadfree(bp, dp) TAILQ_INSERT_HEAD(dp, bp, b_freelist) 243#define binstailfree(bp, dp) TAILQ_INSERT_TAIL(dp, bp, b_freelist) 244 245#ifdef DEBUG 246int debug_verify_freelist = 0; 247static int checkfreelist(struct buf *bp, struct bqueues *dp) 248{ 249 struct buf *b; 250 TAILQ_FOREACH(b, dp, b_freelist) { 251 if (b == bp) 252 return 1; 253 } 254 return 0; 255} 256#endif 257 258void 259bremfree(struct buf *bp) 260{ 261 struct bqueues *dp = NULL; 262 263 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 264 265 KDASSERT(!debug_verify_freelist || 266 checkfreelist(bp, &bufqueues[BQ_AGE]) || 267 checkfreelist(bp, &bufqueues[BQ_LRU]) || 268 checkfreelist(bp, &bufqueues[BQ_LOCKED]) ); 269 270 /* 271 * We only calculate the head of the freelist when removing 272 * the last element of the list as that is the only time that 273 * it is needed (e.g. to reset the tail pointer). 274 * 275 * NB: This makes an assumption about how tailq's are implemented. 276 * 277 * We break the TAILQ abstraction in order to efficiently remove a 278 * buffer from its freelist without having to know exactly which 279 * freelist it is on. 280 */ 281 if (TAILQ_NEXT(bp, b_freelist) == NULL) { 282 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 283 if (dp->tqh_last == &bp->b_freelist.tqe_next) 284 break; 285 if (dp == &bufqueues[BQUEUES]) 286 panic("bremfree: lost tail"); 287 } 288 TAILQ_REMOVE(dp, bp, b_freelist); 289} 290 291u_long 292buf_memcalc(void) 293{ 294 u_long n; 295 296 /* 297 * Determine the upper bound of memory to use for buffers. 298 * 299 * - If bufpages is specified, use that as the number 300 * pages. 301 * 302 * - Otherwise, use bufcache as the percentage of 303 * physical memory. 304 */ 305 if (bufpages != 0) { 306 n = bufpages; 307 } else { 308 if (bufcache < 5) { 309 printf("forcing bufcache %d -> 5", bufcache); 310 bufcache = 5; 311 } 312 if (bufcache > 95) { 313 printf("forcing bufcache %d -> 95", bufcache); 314 bufcache = 95; 315 } 316 n = physmem / 100 * bufcache; 317 } 318 319 n <<= PAGE_SHIFT; 320 if (bufmem_valimit != 0 && n > bufmem_valimit) 321 n = bufmem_valimit; 322 323 return (n); 324} 325 326/* 327 * Initialize buffers and hash links for buffers. 328 */ 329void 330bufinit(void) 331{ 332 struct bqueues *dp; 333 int smallmem; 334 u_int i; 335 336 /* 337 * Initialize buffer cache memory parameters. 338 */ 339 bufmem = 0; 340 bufmem_hiwater = buf_memcalc(); 341 /* lowater is approx. 2% of memory (with bufcache=15) */ 342 bufmem_lowater = (bufmem_hiwater >> 3); 343 if (bufmem_lowater < 64 * 1024) 344 /* Ensure a reasonable minimum value */ 345 bufmem_lowater = 64 * 1024; 346 347 if (bufmem_valimit != 0) { 348 vaddr_t minaddr = 0, maxaddr; 349 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 350 bufmem_valimit, VM_MAP_PAGEABLE, 351 FALSE, 0); 352 if (buf_map == NULL) 353 panic("bufinit: cannot allocate submap"); 354 } else 355 buf_map = kernel_map; 356 357 /* 358 * Initialize the buffer pools. 359 */ 360 pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL); 361 362 /* On "small" machines use small pool page sizes where possible */ 363 smallmem = (physmem < atop(16*1024*1024)); 364 365 for (i = 0; i < NMEMPOOLS; i++) { 366 struct pool_allocator *pa; 367 struct pool *pp = &bmempools[i]; 368 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET); 369 char *name = malloc(8, M_TEMP, M_WAITOK); 370 snprintf(name, 8, "buf%dk", 1 << i); 371 pa = (size <= PAGE_SIZE && smallmem) 372 ? &pool_allocator_nointr 373 : &bufmempool_allocator; 374 pool_init(pp, size, 0, 0, PR_IMMEDRELEASE, name, pa); 375 pool_setlowat(pp, 1); 376 } 377 378 /* Initialize the buffer queues */ 379 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 380 TAILQ_INIT(dp); 381 382 /* 383 * Estimate hash table size based on the amount of memory we 384 * intend to use for the buffer cache. The average buffer 385 * size is dependent on our clients (i.e. filesystems). 386 * 387 * For now, use an empirical 3K per buffer. 388 */ 389 nbuf = (bufmem_hiwater / 1024) / 3; 390 bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash); 391} 392 393static int 394buf_lotsfree(void) 395{ 396 int try, thresh; 397 398 if (bufmem < bufmem_lowater) { 399 return 1; 400 } 401 402 /* If there's anything on the AGE list, it should be eaten. */ 403 404 if (TAILQ_FIRST(&bufqueues[BQ_AGE]) != NULL) 405 return 0; 406 407 try = random() & 0x0000000fL; 408 409 thresh = (16 * bufmem) / bufmem_hiwater; 410 411 if ((try > thresh) && (uvmexp.free > ( 2 * uvmexp.freetarg))) { 412 return 1; 413 } 414 415 return 0; 416} 417 418/* 419 * Return estimate of bytes we think need to be 420 * released to help resolve low memory conditions. 421 * 422 * => called at splbio. 423 * => called with bqueue_slock held. 424 */ 425static int 426buf_canrelease(void) 427{ 428 int pagedemand, ninvalid = 0; 429 struct buf *bp; 430 431 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 432 433 if (bufmem < bufmem_lowater) 434 return 0; 435 436 TAILQ_FOREACH(bp, &bufqueues[BQ_AGE], b_freelist) 437 ninvalid += bp->b_bufsize; 438 439 pagedemand = uvmexp.freetarg - uvmexp.free; 440 if (pagedemand < 0) 441 return ninvalid; 442 return MAX(ninvalid, MIN(2 * MAXBSIZE, 443 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE))); 444} 445 446/* 447 * Buffer memory allocation helper functions 448 */ 449static __inline u_long 450buf_mempoolidx(u_long size) 451{ 452 u_int n = 0; 453 454 size -= 1; 455 size >>= MEMPOOL_INDEX_OFFSET; 456 while (size) { 457 size >>= 1; 458 n += 1; 459 } 460 if (n >= NMEMPOOLS) 461 panic("buf mem pool index %d", n); 462 return n; 463} 464 465static __inline u_long 466buf_roundsize(u_long size) 467{ 468 /* Round up to nearest power of 2 */ 469 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET)); 470} 471 472static __inline caddr_t 473buf_malloc(size_t size) 474{ 475 u_int n = buf_mempoolidx(size); 476 caddr_t addr; 477 int s; 478 479 while (1) { 480 addr = pool_get(&bmempools[n], PR_NOWAIT); 481 if (addr != NULL) 482 break; 483 484 /* No memory, see if we can free some. If so, try again */ 485 if (buf_drain(1) > 0) 486 continue; 487 488 /* Wait for buffers to arrive on the LRU queue */ 489 s = splbio(); 490 simple_lock(&bqueue_slock); 491 needbuffer = 1; 492 ltsleep(&needbuffer, PNORELOCK | (PRIBIO+1), 493 "buf_malloc", 0, &bqueue_slock); 494 splx(s); 495 } 496 497 return addr; 498} 499 500static void 501buf_mrelease(caddr_t addr, size_t size) 502{ 503 504 pool_put(&bmempools[buf_mempoolidx(size)], addr); 505} 506 507 508static __inline struct buf * 509bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 510 int async) 511{ 512 struct buf *bp; 513 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 514 struct proc *p = l->l_proc; 515 516 bp = getblk(vp, blkno, size, 0, 0); 517 518#ifdef DIAGNOSTIC 519 if (bp == NULL) { 520 panic("bio_doread: no such buf"); 521 } 522#endif 523 524 /* 525 * If buffer does not have data valid, start a read. 526 * Note that if buffer is B_INVAL, getblk() won't return it. 527 * Therefore, it's valid if its I/O has completed or been delayed. 528 */ 529 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 530 /* Start I/O for the buffer. */ 531 SET(bp->b_flags, B_READ | async); 532 if (async) 533 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 534 else 535 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 536 VOP_STRATEGY(vp, bp); 537 538 /* Pay for the read. */ 539 p->p_stats->p_ru.ru_inblock++; 540 } else if (async) { 541 brelse(bp); 542 } 543 544 return (bp); 545} 546 547/* 548 * Read a disk block. 549 * This algorithm described in Bach (p.54). 550 */ 551int 552bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 553 struct buf **bpp) 554{ 555 struct buf *bp; 556 557 /* Get buffer for block. */ 558 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 559 560 /* Wait for the read to complete, and return result. */ 561 return (biowait(bp)); 562} 563 564/* 565 * Read-ahead multiple disk blocks. The first is sync, the rest async. 566 * Trivial modification to the breada algorithm presented in Bach (p.55). 567 */ 568int 569breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks, 570 int *rasizes, int nrablks, struct ucred *cred, struct buf **bpp) 571{ 572 struct buf *bp; 573 int i; 574 575 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 576 577 /* 578 * For each of the read-ahead blocks, start a read, if necessary. 579 */ 580 for (i = 0; i < nrablks; i++) { 581 /* If it's in the cache, just go on to next one. */ 582 if (incore(vp, rablks[i])) 583 continue; 584 585 /* Get a buffer for the read-ahead block */ 586 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC); 587 } 588 589 /* Otherwise, we had to start a read for it; wait until it's valid. */ 590 return (biowait(bp)); 591} 592 593/* 594 * Read with single-block read-ahead. Defined in Bach (p.55), but 595 * implemented as a call to breadn(). 596 * XXX for compatibility with old file systems. 597 */ 598int 599breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno, 600 int rabsize, struct ucred *cred, struct buf **bpp) 601{ 602 603 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp)); 604} 605 606/* 607 * Block write. Described in Bach (p.56) 608 */ 609int 610bwrite(struct buf *bp) 611{ 612 int rv, sync, wasdelayed, s; 613 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 614 struct proc *p = l->l_proc; 615 struct vnode *vp; 616 struct mount *mp; 617 618 KASSERT(ISSET(bp->b_flags, B_BUSY)); 619 620 vp = bp->b_vp; 621 if (vp != NULL) { 622 if (vp->v_type == VBLK) 623 mp = vp->v_specmountpoint; 624 else 625 mp = vp->v_mount; 626 } else { 627 mp = NULL; 628 } 629 630 /* 631 * Remember buffer type, to switch on it later. If the write was 632 * synchronous, but the file system was mounted with MNT_ASYNC, 633 * convert it to a delayed write. 634 * XXX note that this relies on delayed tape writes being converted 635 * to async, not sync writes (which is safe, but ugly). 636 */ 637 sync = !ISSET(bp->b_flags, B_ASYNC); 638 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) { 639 bdwrite(bp); 640 return (0); 641 } 642 643 /* 644 * Collect statistics on synchronous and asynchronous writes. 645 * Writes to block devices are charged to their associated 646 * filesystem (if any). 647 */ 648 if (mp != NULL) { 649 if (sync) 650 mp->mnt_stat.f_syncwrites++; 651 else 652 mp->mnt_stat.f_asyncwrites++; 653 } 654 655 s = splbio(); 656 simple_lock(&bp->b_interlock); 657 658 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 659 660 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 661 662 /* 663 * Pay for the I/O operation and make sure the buf is on the correct 664 * vnode queue. 665 */ 666 if (wasdelayed) 667 reassignbuf(bp, bp->b_vp); 668 else 669 p->p_stats->p_ru.ru_oublock++; 670 671 /* Initiate disk write. Make sure the appropriate party is charged. */ 672 V_INCR_NUMOUTPUT(bp->b_vp); 673 simple_unlock(&bp->b_interlock); 674 splx(s); 675 676 if (sync) 677 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 678 else 679 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 680 681 VOP_STRATEGY(vp, bp); 682 683 if (sync) { 684 /* If I/O was synchronous, wait for it to complete. */ 685 rv = biowait(bp); 686 687 /* Release the buffer. */ 688 brelse(bp); 689 690 return (rv); 691 } else { 692 return (0); 693 } 694} 695 696int 697vn_bwrite(void *v) 698{ 699 struct vop_bwrite_args *ap = v; 700 701 return (bwrite(ap->a_bp)); 702} 703 704/* 705 * Delayed write. 706 * 707 * The buffer is marked dirty, but is not queued for I/O. 708 * This routine should be used when the buffer is expected 709 * to be modified again soon, typically a small write that 710 * partially fills a buffer. 711 * 712 * NB: magnetic tapes cannot be delayed; they must be 713 * written in the order that the writes are requested. 714 * 715 * Described in Leffler, et al. (pp. 208-213). 716 */ 717void 718bdwrite(struct buf *bp) 719{ 720 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 721 struct proc *p = l->l_proc; 722 const struct bdevsw *bdev; 723 int s; 724 725 /* If this is a tape block, write the block now. */ 726 bdev = bdevsw_lookup(bp->b_dev); 727 if (bdev != NULL && bdev->d_type == D_TAPE) { 728 bawrite(bp); 729 return; 730 } 731 732 /* 733 * If the block hasn't been seen before: 734 * (1) Mark it as having been seen, 735 * (2) Charge for the write, 736 * (3) Make sure it's on its vnode's correct block list. 737 */ 738 s = splbio(); 739 simple_lock(&bp->b_interlock); 740 741 KASSERT(ISSET(bp->b_flags, B_BUSY)); 742 743 if (!ISSET(bp->b_flags, B_DELWRI)) { 744 SET(bp->b_flags, B_DELWRI); 745 p->p_stats->p_ru.ru_oublock++; 746 reassignbuf(bp, bp->b_vp); 747 } 748 749 /* Otherwise, the "write" is done, so mark and release the buffer. */ 750 CLR(bp->b_flags, B_DONE); 751 simple_unlock(&bp->b_interlock); 752 splx(s); 753 754 brelse(bp); 755} 756 757/* 758 * Asynchronous block write; just an asynchronous bwrite(). 759 */ 760void 761bawrite(struct buf *bp) 762{ 763 int s; 764 765 s = splbio(); 766 simple_lock(&bp->b_interlock); 767 768 KASSERT(ISSET(bp->b_flags, B_BUSY)); 769 770 SET(bp->b_flags, B_ASYNC); 771 simple_unlock(&bp->b_interlock); 772 splx(s); 773 VOP_BWRITE(bp); 774} 775 776/* 777 * Same as first half of bdwrite, mark buffer dirty, but do not release it. 778 * Call at splbio() and with the buffer interlock locked. 779 * Note: called only from biodone() through ffs softdep's bioops.io_complete() 780 */ 781void 782bdirty(struct buf *bp) 783{ 784 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 785 struct proc *p = l->l_proc; 786 787 LOCK_ASSERT(simple_lock_held(&bp->b_interlock)); 788 KASSERT(ISSET(bp->b_flags, B_BUSY)); 789 790 CLR(bp->b_flags, B_AGE); 791 792 if (!ISSET(bp->b_flags, B_DELWRI)) { 793 SET(bp->b_flags, B_DELWRI); 794 p->p_stats->p_ru.ru_oublock++; 795 reassignbuf(bp, bp->b_vp); 796 } 797} 798 799/* 800 * Release a buffer on to the free lists. 801 * Described in Bach (p. 46). 802 */ 803void 804brelse(struct buf *bp) 805{ 806 struct bqueues *bufq; 807 int s; 808 809 /* Block disk interrupts. */ 810 s = splbio(); 811 simple_lock(&bqueue_slock); 812 simple_lock(&bp->b_interlock); 813 814 KASSERT(ISSET(bp->b_flags, B_BUSY)); 815 KASSERT(!ISSET(bp->b_flags, B_CALL)); 816 817 /* Wake up any processes waiting for any buffer to become free. */ 818 if (needbuffer) { 819 needbuffer = 0; 820 wakeup(&needbuffer); 821 } 822 823 /* Wake up any proceeses waiting for _this_ buffer to become free. */ 824 if (ISSET(bp->b_flags, B_WANTED)) { 825 CLR(bp->b_flags, B_WANTED|B_AGE); 826 wakeup(bp); 827 } 828 829 /* 830 * Determine which queue the buffer should be on, then put it there. 831 */ 832 833 /* If it's locked, don't report an error; try again later. */ 834 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR)) 835 CLR(bp->b_flags, B_ERROR); 836 837 /* If it's not cacheable, or an error, mark it invalid. */ 838 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 839 SET(bp->b_flags, B_INVAL); 840 841 if (ISSET(bp->b_flags, B_VFLUSH)) { 842 /* 843 * This is a delayed write buffer that was just flushed to 844 * disk. It is still on the LRU queue. If it's become 845 * invalid, then we need to move it to a different queue; 846 * otherwise leave it in its current position. 847 */ 848 CLR(bp->b_flags, B_VFLUSH); 849 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) { 850 KDASSERT(!debug_verify_freelist || checkfreelist(bp, &bufqueues[BQ_LRU])); 851 goto already_queued; 852 } else { 853 bremfree(bp); 854 } 855 } 856 857 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_AGE])); 858 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LRU])); 859 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LOCKED])); 860 861 if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) { 862 /* 863 * If it's invalid or empty, dissociate it from its vnode 864 * and put on the head of the appropriate queue. 865 */ 866 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 867 (*bioops.io_deallocate)(bp); 868 CLR(bp->b_flags, B_DONE|B_DELWRI); 869 if (bp->b_vp) { 870 reassignbuf(bp, bp->b_vp); 871 brelvp(bp); 872 } 873 if (bp->b_bufsize <= 0) 874 /* no data */ 875 goto already_queued; 876 else 877 /* invalid data */ 878 bufq = &bufqueues[BQ_AGE]; 879 binsheadfree(bp, bufq); 880 } else { 881 /* 882 * It has valid data. Put it on the end of the appropriate 883 * queue, so that it'll stick around for as long as possible. 884 * If buf is AGE, but has dependencies, must put it on last 885 * bufqueue to be scanned, ie LRU. This protects against the 886 * livelock where BQ_AGE only has buffers with dependencies, 887 * and we thus never get to the dependent buffers in BQ_LRU. 888 */ 889 if (ISSET(bp->b_flags, B_LOCKED)) 890 /* locked in core */ 891 bufq = &bufqueues[BQ_LOCKED]; 892 else if (!ISSET(bp->b_flags, B_AGE)) 893 /* valid data */ 894 bufq = &bufqueues[BQ_LRU]; 895 else { 896 /* stale but valid data */ 897 int has_deps; 898 899 if (LIST_FIRST(&bp->b_dep) != NULL && 900 bioops.io_countdeps) 901 has_deps = (*bioops.io_countdeps)(bp, 0); 902 else 903 has_deps = 0; 904 bufq = has_deps ? &bufqueues[BQ_LRU] : 905 &bufqueues[BQ_AGE]; 906 } 907 binstailfree(bp, bufq); 908 } 909 910already_queued: 911 /* Unlock the buffer. */ 912 CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE); 913 SET(bp->b_flags, B_CACHE); 914 915 /* Allow disk interrupts. */ 916 simple_unlock(&bp->b_interlock); 917 simple_unlock(&bqueue_slock); 918 if (bp->b_bufsize <= 0) { 919#ifdef DEBUG 920 memset((char *)bp, 0, sizeof(*bp)); 921#endif 922 pool_put(&bufpool, bp); 923 } 924 splx(s); 925} 926 927/* 928 * Determine if a block is in the cache. 929 * Just look on what would be its hash chain. If it's there, return 930 * a pointer to it, unless it's marked invalid. If it's marked invalid, 931 * we normally don't return the buffer, unless the caller explicitly 932 * wants us to. 933 */ 934struct buf * 935incore(struct vnode *vp, daddr_t blkno) 936{ 937 struct buf *bp; 938 939 /* Search hash chain */ 940 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) { 941 if (bp->b_lblkno == blkno && bp->b_vp == vp && 942 !ISSET(bp->b_flags, B_INVAL)) 943 return (bp); 944 } 945 946 return (NULL); 947} 948 949/* 950 * Get a block of requested size that is associated with 951 * a given vnode and block offset. If it is found in the 952 * block cache, mark it as having been found, make it busy 953 * and return it. Otherwise, return an empty block of the 954 * correct size. It is up to the caller to insure that the 955 * cached blocks be of the correct size. 956 */ 957struct buf * 958getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo) 959{ 960 struct buf *bp; 961 int s, err; 962 int preserve; 963 964start: 965 s = splbio(); 966 simple_lock(&bqueue_slock); 967 bp = incore(vp, blkno); 968 if (bp != NULL) { 969 simple_lock(&bp->b_interlock); 970 if (ISSET(bp->b_flags, B_BUSY)) { 971 simple_unlock(&bqueue_slock); 972 if (curproc == uvm.pagedaemon_proc) { 973 simple_unlock(&bp->b_interlock); 974 splx(s); 975 return NULL; 976 } 977 SET(bp->b_flags, B_WANTED); 978 err = ltsleep(bp, slpflag | (PRIBIO + 1) | PNORELOCK, 979 "getblk", slptimeo, &bp->b_interlock); 980 splx(s); 981 if (err) 982 return (NULL); 983 goto start; 984 } 985#ifdef DIAGNOSTIC 986 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) && 987 bp->b_bcount < size && vp->v_type != VBLK) 988 panic("getblk: block size invariant failed"); 989#endif 990 SET(bp->b_flags, B_BUSY); 991 bremfree(bp); 992 preserve = 1; 993 } else { 994 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL) { 995 simple_unlock(&bqueue_slock); 996 splx(s); 997 goto start; 998 } 999 1000 binshash(bp, BUFHASH(vp, blkno)); 1001 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno; 1002 bgetvp(vp, bp); 1003 preserve = 0; 1004 } 1005 simple_unlock(&bp->b_interlock); 1006 simple_unlock(&bqueue_slock); 1007 splx(s); 1008 /* 1009 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes) 1010 * if we re-size buffers here. 1011 */ 1012 if (ISSET(bp->b_flags, B_LOCKED)) { 1013 KASSERT(bp->b_bufsize >= size); 1014 } else { 1015 allocbuf(bp, size, preserve); 1016 } 1017 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1018 return (bp); 1019} 1020 1021/* 1022 * Get an empty, disassociated buffer of given size. 1023 */ 1024struct buf * 1025geteblk(int size) 1026{ 1027 struct buf *bp; 1028 int s; 1029 1030 s = splbio(); 1031 simple_lock(&bqueue_slock); 1032 while ((bp = getnewbuf(0, 0, 0)) == 0) 1033 ; 1034 1035 SET(bp->b_flags, B_INVAL); 1036 binshash(bp, &invalhash); 1037 simple_unlock(&bqueue_slock); 1038 simple_unlock(&bp->b_interlock); 1039 splx(s); 1040 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1041 allocbuf(bp, size, 0); 1042 return (bp); 1043} 1044 1045/* 1046 * Expand or contract the actual memory allocated to a buffer. 1047 * 1048 * If the buffer shrinks, data is lost, so it's up to the 1049 * caller to have written it out *first*; this routine will not 1050 * start a write. If the buffer grows, it's the callers 1051 * responsibility to fill out the buffer's additional contents. 1052 */ 1053void 1054allocbuf(struct buf *bp, int size, int preserve) 1055{ 1056 vsize_t oldsize, desired_size; 1057 caddr_t addr; 1058 int s, delta; 1059 1060 desired_size = buf_roundsize(size); 1061 if (desired_size > MAXBSIZE) 1062 printf("allocbuf: buffer larger than MAXBSIZE requested"); 1063 1064 bp->b_bcount = size; 1065 1066 oldsize = bp->b_bufsize; 1067 if (oldsize == desired_size) 1068 return; 1069 1070 /* 1071 * If we want a buffer of a different size, re-allocate the 1072 * buffer's memory; copy old content only if needed. 1073 */ 1074 addr = buf_malloc(desired_size); 1075 if (preserve) 1076 memcpy(addr, bp->b_data, MIN(oldsize,desired_size)); 1077 if (bp->b_data != NULL) 1078 buf_mrelease(bp->b_data, oldsize); 1079 bp->b_data = addr; 1080 bp->b_bufsize = desired_size; 1081 1082 /* 1083 * Update overall buffer memory counter (protected by bqueue_slock) 1084 */ 1085 delta = (long)desired_size - (long)oldsize; 1086 1087 s = splbio(); 1088 simple_lock(&bqueue_slock); 1089 if ((bufmem += delta) > bufmem_hiwater) { 1090 /* 1091 * Need to trim overall memory usage. 1092 */ 1093 while (buf_canrelease()) { 1094 if (buf_trim() == 0) 1095 break; 1096 } 1097 } 1098 1099 simple_unlock(&bqueue_slock); 1100 splx(s); 1101} 1102 1103/* 1104 * Find a buffer which is available for use. 1105 * Select something from a free list. 1106 * Preference is to AGE list, then LRU list. 1107 * 1108 * Called at splbio and with buffer queues locked. 1109 * Return buffer locked. 1110 */ 1111struct buf * 1112getnewbuf(int slpflag, int slptimeo, int from_bufq) 1113{ 1114 struct buf *bp; 1115 1116start: 1117 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 1118 1119 /* 1120 * Get a new buffer from the pool; but use NOWAIT because 1121 * we have the buffer queues locked. 1122 */ 1123 if (buf_lotsfree() && !from_bufq && 1124 (bp = pool_get(&bufpool, PR_NOWAIT)) != NULL) { 1125 memset((char *)bp, 0, sizeof(*bp)); 1126 BUF_INIT(bp); 1127 bp->b_dev = NODEV; 1128 bp->b_vnbufs.le_next = NOLIST; 1129 bp->b_flags = B_BUSY; 1130 simple_lock(&bp->b_interlock); 1131 return (bp); 1132 } 1133 1134 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE])) != NULL || 1135 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU])) != NULL) { 1136 simple_lock(&bp->b_interlock); 1137 bremfree(bp); 1138 } else { 1139 /* wait for a free buffer of any kind */ 1140 needbuffer = 1; 1141 ltsleep(&needbuffer, slpflag|(PRIBIO+1), 1142 "getnewbuf", slptimeo, &bqueue_slock); 1143 return (NULL); 1144 } 1145 1146#ifdef DIAGNOSTIC 1147 if (bp->b_bufsize <= 0) 1148 panic("buffer %p: on queue but empty", bp); 1149#endif 1150 1151 if (ISSET(bp->b_flags, B_VFLUSH)) { 1152 /* 1153 * This is a delayed write buffer being flushed to disk. Make 1154 * sure it gets aged out of the queue when it's finished, and 1155 * leave it off the LRU queue. 1156 */ 1157 CLR(bp->b_flags, B_VFLUSH); 1158 SET(bp->b_flags, B_AGE); 1159 simple_unlock(&bp->b_interlock); 1160 goto start; 1161 } 1162 1163 /* Buffer is no longer on free lists. */ 1164 SET(bp->b_flags, B_BUSY); 1165 1166 /* 1167 * If buffer was a delayed write, start it and return NULL 1168 * (since we might sleep while starting the write). 1169 */ 1170 if (ISSET(bp->b_flags, B_DELWRI)) { 1171 /* 1172 * This buffer has gone through the LRU, so make sure it gets 1173 * reused ASAP. 1174 */ 1175 SET(bp->b_flags, B_AGE); 1176 simple_unlock(&bp->b_interlock); 1177 simple_unlock(&bqueue_slock); 1178 bawrite(bp); 1179 simple_lock(&bqueue_slock); 1180 return (NULL); 1181 } 1182 1183 /* disassociate us from our vnode, if we had one... */ 1184 if (bp->b_vp) 1185 brelvp(bp); 1186 1187 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 1188 (*bioops.io_deallocate)(bp); 1189 1190 /* clear out various other fields */ 1191 bp->b_flags = B_BUSY; 1192 bp->b_dev = NODEV; 1193 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0; 1194 bp->b_iodone = 0; 1195 bp->b_error = 0; 1196 bp->b_resid = 0; 1197 bp->b_bcount = 0; 1198 1199 bremhash(bp); 1200 return (bp); 1201} 1202 1203/* 1204 * Attempt to free an aged buffer off the queues. 1205 * Called at splbio and with queue lock held. 1206 * Returns the amount of buffer memory freed. 1207 */ 1208int 1209buf_trim(void) 1210{ 1211 struct buf *bp; 1212 long size = 0; 1213 int wanted; 1214 1215 /* Instruct getnewbuf() to get buffers off the queues */ 1216 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL) 1217 return 0; 1218 1219 wanted = ISSET(bp->b_flags, B_WANTED); 1220 simple_unlock(&bp->b_interlock); 1221 if (wanted) { 1222 printf("buftrim: got WANTED buffer\n"); 1223 SET(bp->b_flags, B_INVAL); 1224 binshash(bp, &invalhash); 1225 simple_unlock(&bqueue_slock); 1226 goto out; 1227 } 1228 size = bp->b_bufsize; 1229 bufmem -= size; 1230 simple_unlock(&bqueue_slock); 1231 if (size > 0) { 1232 buf_mrelease(bp->b_data, size); 1233 bp->b_bcount = bp->b_bufsize = 0; 1234 } 1235 1236out: 1237 /* brelse() will return the buffer to the global buffer pool */ 1238 brelse(bp); 1239 simple_lock(&bqueue_slock); 1240 return size; 1241} 1242 1243int 1244buf_drain(int n) 1245{ 1246 int s, size = 0; 1247 1248 s = splbio(); 1249 simple_lock(&bqueue_slock); 1250 1251 /* If not asked for a specific amount, make our own estimate */ 1252 if (n == 0) 1253 n = buf_canrelease(); 1254 1255 while (size < n && bufmem > bufmem_lowater) 1256 size += buf_trim(); 1257 1258 simple_unlock(&bqueue_slock); 1259 splx(s); 1260 return size; 1261} 1262 1263/* 1264 * Wait for operations on the buffer to complete. 1265 * When they do, extract and return the I/O's error value. 1266 */ 1267int 1268biowait(struct buf *bp) 1269{ 1270 int s, error; 1271 1272 s = splbio(); 1273 simple_lock(&bp->b_interlock); 1274 while (!ISSET(bp->b_flags, B_DONE | B_DELWRI)) 1275 ltsleep(bp, PRIBIO + 1, "biowait", 0, &bp->b_interlock); 1276 1277 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1278 if (ISSET(bp->b_flags, B_EINTR)) { 1279 CLR(bp->b_flags, B_EINTR); 1280 error = EINTR; 1281 } else if (ISSET(bp->b_flags, B_ERROR)) 1282 error = bp->b_error ? bp->b_error : EIO; 1283 else 1284 error = 0; 1285 1286 simple_unlock(&bp->b_interlock); 1287 splx(s); 1288 return (error); 1289} 1290 1291/* 1292 * Mark I/O complete on a buffer. 1293 * 1294 * If a callback has been requested, e.g. the pageout 1295 * daemon, do so. Otherwise, awaken waiting processes. 1296 * 1297 * [ Leffler, et al., says on p.247: 1298 * "This routine wakes up the blocked process, frees the buffer 1299 * for an asynchronous write, or, for a request by the pagedaemon 1300 * process, invokes a procedure specified in the buffer structure" ] 1301 * 1302 * In real life, the pagedaemon (or other system processes) wants 1303 * to do async stuff to, and doesn't want the buffer brelse()'d. 1304 * (for swap pager, that puts swap buffers on the free lists (!!!), 1305 * for the vn device, that puts malloc'd buffers on the free lists!) 1306 */ 1307void 1308biodone(struct buf *bp) 1309{ 1310 int s = splbio(); 1311 1312 simple_lock(&bp->b_interlock); 1313 if (ISSET(bp->b_flags, B_DONE)) 1314 panic("biodone already"); 1315 SET(bp->b_flags, B_DONE); /* note that it's done */ 1316 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1317 1318 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) 1319 (*bioops.io_complete)(bp); 1320 1321 if (!ISSET(bp->b_flags, B_READ)) /* wake up reader */ 1322 vwakeup(bp); 1323 1324 /* 1325 * If necessary, call out. Unlock the buffer before calling 1326 * iodone() as the buffer isn't valid any more when it return. 1327 */ 1328 if (ISSET(bp->b_flags, B_CALL)) { 1329 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1330 simple_unlock(&bp->b_interlock); 1331 (*bp->b_iodone)(bp); 1332 } else { 1333 if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release */ 1334 simple_unlock(&bp->b_interlock); 1335 brelse(bp); 1336 } else { /* or just wakeup the buffer */ 1337 CLR(bp->b_flags, B_WANTED); 1338 wakeup(bp); 1339 simple_unlock(&bp->b_interlock); 1340 } 1341 } 1342 1343 splx(s); 1344} 1345 1346/* 1347 * Return a count of buffers on the "locked" queue. 1348 */ 1349int 1350count_lock_queue(void) 1351{ 1352 struct buf *bp; 1353 int n = 0; 1354 1355 simple_lock(&bqueue_slock); 1356 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED], b_freelist) 1357 n++; 1358 simple_unlock(&bqueue_slock); 1359 return (n); 1360} 1361 1362/* 1363 * Wait for all buffers to complete I/O 1364 * Return the number of "stuck" buffers. 1365 */ 1366int 1367buf_syncwait(void) 1368{ 1369 struct buf *bp; 1370 int iter, nbusy, nbusy_prev = 0, dcount, s, ihash; 1371 1372 dcount = 10000; 1373 for (iter = 0; iter < 20;) { 1374 s = splbio(); 1375 simple_lock(&bqueue_slock); 1376 nbusy = 0; 1377 for (ihash = 0; ihash < bufhash+1; ihash++) { 1378 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1379 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1380 nbusy++; 1381 /* 1382 * With soft updates, some buffers that are 1383 * written will be remarked as dirty until other 1384 * buffers are written. 1385 */ 1386 if (bp->b_vp && bp->b_vp->v_mount 1387 && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP) 1388 && (bp->b_flags & B_DELWRI)) { 1389 simple_lock(&bp->b_interlock); 1390 bremfree(bp); 1391 bp->b_flags |= B_BUSY; 1392 nbusy++; 1393 simple_unlock(&bp->b_interlock); 1394 simple_unlock(&bqueue_slock); 1395 bawrite(bp); 1396 if (dcount-- <= 0) { 1397 printf("softdep "); 1398 goto fail; 1399 } 1400 simple_lock(&bqueue_slock); 1401 } 1402 } 1403 } 1404 1405 simple_unlock(&bqueue_slock); 1406 splx(s); 1407 1408 if (nbusy == 0) 1409 break; 1410 if (nbusy_prev == 0) 1411 nbusy_prev = nbusy; 1412 printf("%d ", nbusy); 1413 tsleep(&nbusy, PRIBIO, "bflush", 1414 (iter == 0) ? 1 : hz / 25 * iter); 1415 if (nbusy >= nbusy_prev) /* we didn't flush anything */ 1416 iter++; 1417 else 1418 nbusy_prev = nbusy; 1419 } 1420 1421 if (nbusy) { 1422fail:; 1423#if defined(DEBUG) || defined(DEBUG_HALT_BUSY) 1424 printf("giving up\nPrinting vnodes for busy buffers\n"); 1425 for (ihash = 0; ihash < bufhash+1; ihash++) { 1426 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1427 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1428 vprint(NULL, bp->b_vp); 1429 } 1430 } 1431#endif 1432 } 1433 1434 return nbusy; 1435} 1436 1437static void 1438sysctl_fillbuf(struct buf *i, struct buf_sysctl *o) 1439{ 1440 1441 o->b_flags = i->b_flags; 1442 o->b_error = i->b_error; 1443 o->b_prio = i->b_prio; 1444 o->b_dev = i->b_dev; 1445 o->b_bufsize = i->b_bufsize; 1446 o->b_bcount = i->b_bcount; 1447 o->b_resid = i->b_resid; 1448 o->b_addr = PTRTOUINT64(i->b_un.b_addr); 1449 o->b_blkno = i->b_blkno; 1450 o->b_rawblkno = i->b_rawblkno; 1451 o->b_iodone = PTRTOUINT64(i->b_iodone); 1452 o->b_proc = PTRTOUINT64(i->b_proc); 1453 o->b_vp = PTRTOUINT64(i->b_vp); 1454 o->b_saveaddr = PTRTOUINT64(i->b_saveaddr); 1455 o->b_lblkno = i->b_lblkno; 1456} 1457 1458#define KERN_BUFSLOP 20 1459static int 1460sysctl_dobuf(SYSCTLFN_ARGS) 1461{ 1462 struct buf *bp; 1463 struct buf_sysctl bs; 1464 char *dp; 1465 u_int i, op, arg; 1466 size_t len, needed, elem_size, out_size; 1467 int error, s, elem_count; 1468 1469 if (namelen == 1 && name[0] == CTL_QUERY) 1470 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1471 1472 if (namelen != 4) 1473 return (EINVAL); 1474 1475 dp = oldp; 1476 len = (oldp != NULL) ? *oldlenp : 0; 1477 op = name[0]; 1478 arg = name[1]; 1479 elem_size = name[2]; 1480 elem_count = name[3]; 1481 out_size = MIN(sizeof(bs), elem_size); 1482 1483 /* 1484 * at the moment, these are just "placeholders" to make the 1485 * API for retrieving kern.buf data more extensible in the 1486 * future. 1487 * 1488 * XXX kern.buf currently has "netbsd32" issues. hopefully 1489 * these will be resolved at a later point. 1490 */ 1491 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL || 1492 elem_size < 1 || elem_count < 0) 1493 return (EINVAL); 1494 1495 error = 0; 1496 needed = 0; 1497 s = splbio(); 1498 simple_lock(&bqueue_slock); 1499 for (i = 0; i < BQUEUES; i++) { 1500 TAILQ_FOREACH(bp, &bufqueues[i], b_freelist) { 1501 if (len >= elem_size && elem_count > 0) { 1502 sysctl_fillbuf(bp, &bs); 1503 error = copyout(&bs, dp, out_size); 1504 if (error) 1505 goto cleanup; 1506 dp += elem_size; 1507 len -= elem_size; 1508 } 1509 if (elem_count > 0) { 1510 needed += elem_size; 1511 if (elem_count != INT_MAX) 1512 elem_count--; 1513 } 1514 } 1515 } 1516cleanup: 1517 simple_unlock(&bqueue_slock); 1518 splx(s); 1519 1520 *oldlenp = needed; 1521 if (oldp == NULL) 1522 *oldlenp += KERN_BUFSLOP * sizeof(struct buf); 1523 1524 return (error); 1525} 1526 1527static int 1528sysctl_bufvm_update(SYSCTLFN_ARGS) 1529{ 1530 int t, error; 1531 struct sysctlnode node; 1532 1533 node = *rnode; 1534 node.sysctl_data = &t; 1535 t = *(int*)rnode->sysctl_data; 1536 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1537 if (error || newp == NULL) 1538 return (error); 1539 1540 if (rnode->sysctl_data == &bufcache) { 1541 if (t < 0 || t > 100) 1542 return (EINVAL); 1543 bufcache = t; 1544 bufmem_hiwater = buf_memcalc(); 1545 bufmem_lowater = (bufmem_hiwater >> 3); 1546 if (bufmem_lowater < 64 * 1024) 1547 /* Ensure a reasonable minimum value */ 1548 bufmem_lowater = 64 * 1024; 1549 1550 } else if (rnode->sysctl_data == &bufmem_lowater) { 1551 bufmem_lowater = t; 1552 } else if (rnode->sysctl_data == &bufmem_hiwater) { 1553 bufmem_hiwater = t; 1554 } else 1555 return (EINVAL); 1556 1557 /* Drain until below new high water mark */ 1558 while ((t = bufmem - bufmem_hiwater) >= 0) { 1559 if (buf_drain(t / (2*1024)) <= 0) 1560 break; 1561 } 1562 1563 return 0; 1564} 1565 1566SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup") 1567{ 1568 1569 sysctl_createv(clog, 0, NULL, NULL, 1570 CTLFLAG_PERMANENT, 1571 CTLTYPE_NODE, "kern", NULL, 1572 NULL, 0, NULL, 0, 1573 CTL_KERN, CTL_EOL); 1574 sysctl_createv(clog, 0, NULL, NULL, 1575 CTLFLAG_PERMANENT, 1576 CTLTYPE_NODE, "buf", NULL, 1577 sysctl_dobuf, 0, NULL, 0, 1578 CTL_KERN, KERN_BUF, CTL_EOL); 1579} 1580 1581SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup") 1582{ 1583 1584 sysctl_createv(clog, 0, NULL, NULL, 1585 CTLFLAG_PERMANENT, 1586 CTLTYPE_NODE, "vm", NULL, 1587 NULL, 0, NULL, 0, 1588 CTL_VM, CTL_EOL); 1589 1590 sysctl_createv(clog, 0, NULL, NULL, 1591 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1592 CTLTYPE_INT, "bufcache", NULL, 1593 sysctl_bufvm_update, 0, &bufcache, 0, 1594 CTL_VM, CTL_CREATE, CTL_EOL); 1595 sysctl_createv(clog, 0, NULL, NULL, 1596 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1597 CTLTYPE_INT, "bufmem_lowater", NULL, 1598 sysctl_bufvm_update, 0, &bufmem_lowater, 0, 1599 CTL_VM, CTL_CREATE, CTL_EOL); 1600 sysctl_createv(clog, 0, NULL, NULL, 1601 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1602 CTLTYPE_INT, "bufmem_hiwater", NULL, 1603 sysctl_bufvm_update, 0, &bufmem_hiwater, 0, 1604 CTL_VM, CTL_CREATE, CTL_EOL); 1605} 1606 1607#ifdef DEBUG 1608/* 1609 * Print out statistics on the current allocation of the buffer pool. 1610 * Can be enabled to print out on every ``sync'' by setting "syncprt" 1611 * in vfs_syscalls.c using sysctl. 1612 */ 1613void 1614vfs_bufstats(void) 1615{ 1616 int s, i, j, count; 1617 struct buf *bp; 1618 struct bqueues *dp; 1619 int counts[(MAXBSIZE / PAGE_SIZE) + 1]; 1620 static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" }; 1621 1622 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) { 1623 count = 0; 1624 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1625 counts[j] = 0; 1626 s = splbio(); 1627 TAILQ_FOREACH(bp, dp, b_freelist) { 1628 counts[bp->b_bufsize/PAGE_SIZE]++; 1629 count++; 1630 } 1631 splx(s); 1632 printf("%s: total-%d", bname[i], count); 1633 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1634 if (counts[j] != 0) 1635 printf(", %d-%d", j * PAGE_SIZE, counts[j]); 1636 printf("\n"); 1637 } 1638} 1639#endif /* DEBUG */ 1640