vfs_bio.c revision 70374
1/*
2 * Copyright (c) 1994,1997 John S. Dyson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice immediately at the beginning of the file, without modification,
10 *    this list of conditions, and the following disclaimer.
11 * 2. Absolutely no warranty of function or purpose is made by the author
12 *		John S. Dyson.
13 *
14 * $FreeBSD: head/sys/kern/vfs_bio.c 70374 2000-12-26 19:41:38Z dillon $
15 */
16
17/*
18 * this file contains a new buffer I/O scheme implementing a coherent
19 * VM object and buffer cache scheme.  Pains have been taken to make
20 * sure that the performance degradation associated with schemes such
21 * as this is not realized.
22 *
23 * Author:  John S. Dyson
24 * Significant help during the development and debugging phases
25 * had been provided by David Greenman, also of the FreeBSD core team.
26 *
27 * see man buf(9) for more info.
28 */
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/bio.h>
33#include <sys/buf.h>
34#include <sys/eventhandler.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mount.h>
38#include <sys/mutex.h>
39#include <sys/kernel.h>
40#include <sys/kthread.h>
41#include <sys/ktr.h>
42#include <sys/proc.h>
43#include <sys/reboot.h>
44#include <sys/resourcevar.h>
45#include <sys/sysctl.h>
46#include <sys/vmmeter.h>
47#include <sys/vnode.h>
48#include <vm/vm.h>
49#include <vm/vm_param.h>
50#include <vm/vm_kern.h>
51#include <vm/vm_pageout.h>
52#include <vm/vm_page.h>
53#include <vm/vm_object.h>
54#include <vm/vm_extern.h>
55#include <vm/vm_map.h>
56
57static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
58
59struct	bio_ops bioops;		/* I/O operation notification */
60
61struct buf *buf;		/* buffer header pool */
62struct swqueue bswlist;
63struct mtx buftimelock;		/* Interlock on setting prio and timo */
64
65static void vm_hold_free_pages(struct buf * bp, vm_offset_t from,
66		vm_offset_t to);
67static void vm_hold_load_pages(struct buf * bp, vm_offset_t from,
68		vm_offset_t to);
69static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off,
70			       int pageno, vm_page_t m);
71static void vfs_clean_pages(struct buf * bp);
72static void vfs_setdirty(struct buf *bp);
73static void vfs_vmio_release(struct buf *bp);
74static void vfs_backgroundwritedone(struct buf *bp);
75static int flushbufqueues(void);
76
77static int bd_request;
78
79static void buf_daemon __P((void));
80/*
81 * bogus page -- for I/O to/from partially complete buffers
82 * this is a temporary solution to the problem, but it is not
83 * really that bad.  it would be better to split the buffer
84 * for input in the case of buffers partially already in memory,
85 * but the code is intricate enough already.
86 */
87vm_page_t bogus_page;
88int vmiodirenable = FALSE;
89int runningbufspace;
90static vm_offset_t bogus_offset;
91
92static int bufspace, maxbufspace,
93	bufmallocspace, maxbufmallocspace, lobufspace, hibufspace;
94static int bufreusecnt, bufdefragcnt, buffreekvacnt;
95static int needsbuffer;
96static int lorunningspace, hirunningspace, runningbufreq;
97static int numdirtybuffers, lodirtybuffers, hidirtybuffers;
98static int numfreebuffers, lofreebuffers, hifreebuffers;
99static int getnewbufcalls;
100static int getnewbufrestarts;
101
102SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD,
103	&numdirtybuffers, 0, "");
104SYSCTL_INT(_vfs, OID_AUTO, lodirtybuffers, CTLFLAG_RW,
105	&lodirtybuffers, 0, "");
106SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW,
107	&hidirtybuffers, 0, "");
108SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD,
109	&numfreebuffers, 0, "");
110SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW,
111	&lofreebuffers, 0, "");
112SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW,
113	&hifreebuffers, 0, "");
114SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD,
115	&runningbufspace, 0, "");
116SYSCTL_INT(_vfs, OID_AUTO, lorunningspace, CTLFLAG_RW,
117	&lorunningspace, 0, "");
118SYSCTL_INT(_vfs, OID_AUTO, hirunningspace, CTLFLAG_RW,
119	&hirunningspace, 0, "");
120SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD,
121	&maxbufspace, 0, "");
122SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD,
123	&hibufspace, 0, "");
124SYSCTL_INT(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD,
125	&lobufspace, 0, "");
126SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD,
127	&bufspace, 0, "");
128SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW,
129	&maxbufmallocspace, 0, "");
130SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD,
131	&bufmallocspace, 0, "");
132SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW,
133	&getnewbufcalls, 0, "");
134SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW,
135	&getnewbufrestarts, 0, "");
136SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW,
137	&vmiodirenable, 0, "");
138SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW,
139	&bufdefragcnt, 0, "");
140SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW,
141	&buffreekvacnt, 0, "");
142SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW,
143	&bufreusecnt, 0, "");
144
145static int bufhashmask;
146static LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
147struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } };
148char *buf_wmesg = BUF_WMESG;
149
150extern int vm_swap_size;
151
152#define VFS_BIO_NEED_ANY	0x01	/* any freeable buffer */
153#define VFS_BIO_NEED_DIRTYFLUSH	0x02	/* waiting for dirty buffer flush */
154#define VFS_BIO_NEED_FREE	0x04	/* wait for free bufs, hi hysteresis */
155#define VFS_BIO_NEED_BUFSPACE	0x08	/* wait for buf space, lo hysteresis */
156
157/*
158 * Buffer hash table code.  Note that the logical block scans linearly, which
159 * gives us some L1 cache locality.
160 */
161
162static __inline
163struct bufhashhdr *
164bufhash(struct vnode *vnp, daddr_t bn)
165{
166	return(&bufhashtbl[(((uintptr_t)(vnp) >> 7) + (int)bn) & bufhashmask]);
167}
168
169/*
170 *	numdirtywakeup:
171 *
172 *	If someone is blocked due to there being too many dirty buffers,
173 *	and numdirtybuffers is now reasonable, wake them up.
174 */
175
176static __inline void
177numdirtywakeup(int level)
178{
179	if (numdirtybuffers <= level) {
180		if (needsbuffer & VFS_BIO_NEED_DIRTYFLUSH) {
181			needsbuffer &= ~VFS_BIO_NEED_DIRTYFLUSH;
182			wakeup(&needsbuffer);
183		}
184	}
185}
186
187/*
188 *	bufspacewakeup:
189 *
190 *	Called when buffer space is potentially available for recovery.
191 *	getnewbuf() will block on this flag when it is unable to free
192 *	sufficient buffer space.  Buffer space becomes recoverable when
193 *	bp's get placed back in the queues.
194 */
195
196static __inline void
197bufspacewakeup(void)
198{
199	/*
200	 * If someone is waiting for BUF space, wake them up.  Even
201	 * though we haven't freed the kva space yet, the waiting
202	 * process will be able to now.
203	 */
204	if (needsbuffer & VFS_BIO_NEED_BUFSPACE) {
205		needsbuffer &= ~VFS_BIO_NEED_BUFSPACE;
206		wakeup(&needsbuffer);
207	}
208}
209
210/*
211 * runningbufwakeup() - in-progress I/O accounting.
212 *
213 */
214static __inline void
215runningbufwakeup(struct buf *bp)
216{
217	if (bp->b_runningbufspace) {
218		runningbufspace -= bp->b_runningbufspace;
219		bp->b_runningbufspace = 0;
220		if (runningbufreq && runningbufspace <= lorunningspace) {
221			runningbufreq = 0;
222			wakeup(&runningbufreq);
223		}
224	}
225}
226
227/*
228 *	bufcountwakeup:
229 *
230 *	Called when a buffer has been added to one of the free queues to
231 *	account for the buffer and to wakeup anyone waiting for free buffers.
232 *	This typically occurs when large amounts of metadata are being handled
233 *	by the buffer cache ( else buffer space runs out first, usually ).
234 */
235
236static __inline void
237bufcountwakeup(void)
238{
239	++numfreebuffers;
240	if (needsbuffer) {
241		needsbuffer &= ~VFS_BIO_NEED_ANY;
242		if (numfreebuffers >= hifreebuffers)
243			needsbuffer &= ~VFS_BIO_NEED_FREE;
244		wakeup(&needsbuffer);
245	}
246}
247
248/*
249 *	waitrunningbufspace()
250 *
251 *	runningbufspace is a measure of the amount of I/O currently
252 *	running.  This routine is used in async-write situations to
253 *	prevent creating huge backups of pending writes to a device.
254 *	Only asynchronous writes are governed by this function.
255 *
256 *	Reads will adjust runningbufspace, but will not block based on it.
257 *	The read load has a side effect of reducing the allowed write load.
258 *
259 *	This does NOT turn an async write into a sync write.  It waits
260 *	for earlier writes to complete and generally returns before the
261 *	caller's write has reached the device.
262 */
263static __inline void
264waitrunningbufspace(void)
265{
266	while (runningbufspace > hirunningspace) {
267		++runningbufreq;
268		tsleep(&runningbufreq, PVM, "wdrain", 0);
269	}
270}
271
272
273/*
274 *	vfs_buf_test_cache:
275 *
276 *	Called when a buffer is extended.  This function clears the B_CACHE
277 *	bit if the newly extended portion of the buffer does not contain
278 *	valid data.
279 */
280static __inline__
281void
282vfs_buf_test_cache(struct buf *bp,
283		  vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
284		  vm_page_t m)
285{
286	if (bp->b_flags & B_CACHE) {
287		int base = (foff + off) & PAGE_MASK;
288		if (vm_page_is_valid(m, base, size) == 0)
289			bp->b_flags &= ~B_CACHE;
290	}
291}
292
293static __inline__
294void
295bd_wakeup(int dirtybuflevel)
296{
297	if (bd_request == 0 && numdirtybuffers >= dirtybuflevel) {
298		bd_request = 1;
299		wakeup(&bd_request);
300	}
301}
302
303/*
304 * bd_speedup - speedup the buffer cache flushing code
305 */
306
307static __inline__
308void
309bd_speedup(void)
310{
311	bd_wakeup(1);
312}
313
314/*
315 * Initialize buffer headers and related structures.
316 */
317
318caddr_t
319bufhashinit(caddr_t vaddr)
320{
321	/* first, make a null hash table */
322	for (bufhashmask = 8; bufhashmask < nbuf / 4; bufhashmask <<= 1)
323		;
324	bufhashtbl = (void *)vaddr;
325	vaddr = vaddr + sizeof(*bufhashtbl) * bufhashmask;
326	--bufhashmask;
327	return(vaddr);
328}
329
330void
331bufinit(void)
332{
333	struct buf *bp;
334	int i;
335
336	TAILQ_INIT(&bswlist);
337	LIST_INIT(&invalhash);
338	mtx_init(&buftimelock, "buftime lock", MTX_DEF);
339
340	for (i = 0; i <= bufhashmask; i++)
341		LIST_INIT(&bufhashtbl[i]);
342
343	/* next, make a null set of free lists */
344	for (i = 0; i < BUFFER_QUEUES; i++)
345		TAILQ_INIT(&bufqueues[i]);
346
347	/* finally, initialize each buffer header and stick on empty q */
348	for (i = 0; i < nbuf; i++) {
349		bp = &buf[i];
350		bzero(bp, sizeof *bp);
351		bp->b_flags = B_INVAL;	/* we're just an empty header */
352		bp->b_dev = NODEV;
353		bp->b_rcred = NOCRED;
354		bp->b_wcred = NOCRED;
355		bp->b_qindex = QUEUE_EMPTY;
356		bp->b_xflags = 0;
357		LIST_INIT(&bp->b_dep);
358		BUF_LOCKINIT(bp);
359		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
360		LIST_INSERT_HEAD(&invalhash, bp, b_hash);
361	}
362
363	/*
364	 * maxbufspace is the absolute maximum amount of buffer space we are
365	 * allowed to reserve in KVM and in real terms.  The absolute maximum
366	 * is nominally used by buf_daemon.  hibufspace is the nominal maximum
367	 * used by most other processes.  The differential is required to
368	 * ensure that buf_daemon is able to run when other processes might
369	 * be blocked waiting for buffer space.
370	 *
371	 * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
372	 * this may result in KVM fragmentation which is not handled optimally
373	 * by the system.
374	 */
375	maxbufspace = nbuf * BKVASIZE;
376	hibufspace = imax(3 * maxbufspace / 4, maxbufspace - MAXBSIZE * 10);
377	lobufspace = hibufspace - MAXBSIZE;
378
379	lorunningspace = 512 * 1024;
380	hirunningspace = 1024 * 1024;
381
382/*
383 * Limit the amount of malloc memory since it is wired permanently into
384 * the kernel space.  Even though this is accounted for in the buffer
385 * allocation, we don't want the malloced region to grow uncontrolled.
386 * The malloc scheme improves memory utilization significantly on average
387 * (small) directories.
388 */
389	maxbufmallocspace = hibufspace / 20;
390
391/*
392 * Reduce the chance of a deadlock occuring by limiting the number
393 * of delayed-write dirty buffers we allow to stack up.
394 */
395	hidirtybuffers = nbuf / 4 + 20;
396	numdirtybuffers = 0;
397/*
398 * To support extreme low-memory systems, make sure hidirtybuffers cannot
399 * eat up all available buffer space.  This occurs when our minimum cannot
400 * be met.  We try to size hidirtybuffers to 3/4 our buffer space assuming
401 * BKVASIZE'd (8K) buffers.
402 */
403	while (hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
404		hidirtybuffers >>= 1;
405	}
406	lodirtybuffers = hidirtybuffers / 2;
407
408/*
409 * Try to keep the number of free buffers in the specified range,
410 * and give special processes (e.g. like buf_daemon) access to an
411 * emergency reserve.
412 */
413	lofreebuffers = nbuf / 18 + 5;
414	hifreebuffers = 2 * lofreebuffers;
415	numfreebuffers = nbuf;
416
417/*
418 * Maximum number of async ops initiated per buf_daemon loop.  This is
419 * somewhat of a hack at the moment, we really need to limit ourselves
420 * based on the number of bytes of I/O in-transit that were initiated
421 * from buf_daemon.
422 */
423
424	bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
425	bogus_page = vm_page_alloc(kernel_object,
426			((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
427			VM_ALLOC_NORMAL);
428	cnt.v_wire_count++;
429
430}
431
432/*
433 * bfreekva() - free the kva allocation for a buffer.
434 *
435 *	Must be called at splbio() or higher as this is the only locking for
436 *	buffer_map.
437 *
438 *	Since this call frees up buffer space, we call bufspacewakeup().
439 */
440static void
441bfreekva(struct buf * bp)
442{
443	if (bp->b_kvasize) {
444		++buffreekvacnt;
445		bufspace -= bp->b_kvasize;
446		vm_map_delete(buffer_map,
447		    (vm_offset_t) bp->b_kvabase,
448		    (vm_offset_t) bp->b_kvabase + bp->b_kvasize
449		);
450		bp->b_kvasize = 0;
451		bufspacewakeup();
452	}
453}
454
455/*
456 *	bremfree:
457 *
458 *	Remove the buffer from the appropriate free list.
459 */
460void
461bremfree(struct buf * bp)
462{
463	int s = splbio();
464	int old_qindex = bp->b_qindex;
465
466	if (bp->b_qindex != QUEUE_NONE) {
467		KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
468		TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
469		bp->b_qindex = QUEUE_NONE;
470	} else {
471		if (BUF_REFCNT(bp) <= 1)
472			panic("bremfree: removing a buffer not on a queue");
473	}
474
475	/*
476	 * Fixup numfreebuffers count.  If the buffer is invalid or not
477	 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
478	 * the buffer was free and we must decrement numfreebuffers.
479	 */
480	if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
481		switch(old_qindex) {
482		case QUEUE_DIRTY:
483		case QUEUE_CLEAN:
484		case QUEUE_EMPTY:
485		case QUEUE_EMPTYKVA:
486			--numfreebuffers;
487			break;
488		default:
489			break;
490		}
491	}
492	splx(s);
493}
494
495
496/*
497 * Get a buffer with the specified data.  Look in the cache first.  We
498 * must clear BIO_ERROR and B_INVAL prior to initiating I/O.  If B_CACHE
499 * is set, the buffer is valid and we do not have to do anything ( see
500 * getblk() ).
501 */
502int
503bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred,
504    struct buf ** bpp)
505{
506	struct buf *bp;
507
508	bp = getblk(vp, blkno, size, 0, 0);
509	*bpp = bp;
510
511	/* if not found in cache, do some I/O */
512	if ((bp->b_flags & B_CACHE) == 0) {
513		if (curproc != idleproc)
514			curproc->p_stats->p_ru.ru_inblock++;
515		KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp));
516		bp->b_iocmd = BIO_READ;
517		bp->b_flags &= ~B_INVAL;
518		bp->b_ioflags &= ~BIO_ERROR;
519		if (bp->b_rcred == NOCRED) {
520			if (cred != NOCRED)
521				crhold(cred);
522			bp->b_rcred = cred;
523		}
524		vfs_busy_pages(bp, 0);
525		VOP_STRATEGY(vp, bp);
526		return (bufwait(bp));
527	}
528	return (0);
529}
530
531/*
532 * Operates like bread, but also starts asynchronous I/O on
533 * read-ahead blocks.  We must clear BIO_ERROR and B_INVAL prior
534 * to initiating I/O . If B_CACHE is set, the buffer is valid
535 * and we do not have to do anything.
536 */
537int
538breadn(struct vnode * vp, daddr_t blkno, int size,
539    daddr_t * rablkno, int *rabsize,
540    int cnt, struct ucred * cred, struct buf ** bpp)
541{
542	struct buf *bp, *rabp;
543	int i;
544	int rv = 0, readwait = 0;
545
546	*bpp = bp = getblk(vp, blkno, size, 0, 0);
547
548	/* if not found in cache, do some I/O */
549	if ((bp->b_flags & B_CACHE) == 0) {
550		if (curproc != idleproc)
551			curproc->p_stats->p_ru.ru_inblock++;
552		bp->b_iocmd = BIO_READ;
553		bp->b_flags &= ~B_INVAL;
554		bp->b_ioflags &= ~BIO_ERROR;
555		if (bp->b_rcred == NOCRED) {
556			if (cred != NOCRED)
557				crhold(cred);
558			bp->b_rcred = cred;
559		}
560		vfs_busy_pages(bp, 0);
561		VOP_STRATEGY(vp, bp);
562		++readwait;
563	}
564
565	for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
566		if (inmem(vp, *rablkno))
567			continue;
568		rabp = getblk(vp, *rablkno, *rabsize, 0, 0);
569
570		if ((rabp->b_flags & B_CACHE) == 0) {
571			if (curproc != idleproc)
572				curproc->p_stats->p_ru.ru_inblock++;
573			rabp->b_flags |= B_ASYNC;
574			rabp->b_flags &= ~B_INVAL;
575			rabp->b_ioflags &= ~BIO_ERROR;
576			rabp->b_iocmd = BIO_READ;
577			if (rabp->b_rcred == NOCRED) {
578				if (cred != NOCRED)
579					crhold(cred);
580				rabp->b_rcred = cred;
581			}
582			vfs_busy_pages(rabp, 0);
583			BUF_KERNPROC(rabp);
584			VOP_STRATEGY(vp, rabp);
585		} else {
586			brelse(rabp);
587		}
588	}
589
590	if (readwait) {
591		rv = bufwait(bp);
592	}
593	return (rv);
594}
595
596/*
597 * Write, release buffer on completion.  (Done by iodone
598 * if async).  Do not bother writing anything if the buffer
599 * is invalid.
600 *
601 * Note that we set B_CACHE here, indicating that buffer is
602 * fully valid and thus cacheable.  This is true even of NFS
603 * now so we set it generally.  This could be set either here
604 * or in biodone() since the I/O is synchronous.  We put it
605 * here.
606 */
607int
608bwrite(struct buf * bp)
609{
610	int oldflags, s;
611	struct buf *newbp;
612
613	if (bp->b_flags & B_INVAL) {
614		brelse(bp);
615		return (0);
616	}
617
618	oldflags = bp->b_flags;
619
620	if (BUF_REFCNT(bp) == 0)
621		panic("bwrite: buffer is not busy???");
622	s = splbio();
623	/*
624	 * If a background write is already in progress, delay
625	 * writing this block if it is asynchronous. Otherwise
626	 * wait for the background write to complete.
627	 */
628	if (bp->b_xflags & BX_BKGRDINPROG) {
629		if (bp->b_flags & B_ASYNC) {
630			splx(s);
631			bdwrite(bp);
632			return (0);
633		}
634		bp->b_xflags |= BX_BKGRDWAIT;
635		tsleep(&bp->b_xflags, PRIBIO, "biord", 0);
636		if (bp->b_xflags & BX_BKGRDINPROG)
637			panic("bwrite: still writing");
638	}
639
640	/* Mark the buffer clean */
641	bundirty(bp);
642
643	/*
644	 * If this buffer is marked for background writing and we
645	 * do not have to wait for it, make a copy and write the
646	 * copy so as to leave this buffer ready for further use.
647	 *
648	 * This optimization eats a lot of memory.  If we have a page
649	 * or buffer shortfall we can't do it.
650	 */
651	if ((bp->b_xflags & BX_BKGRDWRITE) &&
652	    (bp->b_flags & B_ASYNC) &&
653	    !vm_page_count_severe() &&
654	    !buf_dirty_count_severe()) {
655		if (bp->b_iodone != NULL) {
656			printf("bp->b_iodone = %p\n", bp->b_iodone);
657			panic("bwrite: need chained iodone");
658		}
659
660		/* get a new block */
661		newbp = geteblk(bp->b_bufsize);
662
663		/* set it to be identical to the old block */
664		memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
665		bgetvp(bp->b_vp, newbp);
666		newbp->b_lblkno = bp->b_lblkno;
667		newbp->b_blkno = bp->b_blkno;
668		newbp->b_offset = bp->b_offset;
669		newbp->b_iodone = vfs_backgroundwritedone;
670		newbp->b_flags |= B_ASYNC;
671		newbp->b_flags &= ~B_INVAL;
672
673		/* move over the dependencies */
674		if (LIST_FIRST(&bp->b_dep) != NULL)
675			buf_movedeps(bp, newbp);
676
677		/*
678		 * Initiate write on the copy, release the original to
679		 * the B_LOCKED queue so that it cannot go away until
680		 * the background write completes. If not locked it could go
681		 * away and then be reconstituted while it was being written.
682		 * If the reconstituted buffer were written, we could end up
683		 * with two background copies being written at the same time.
684		 */
685		bp->b_xflags |= BX_BKGRDINPROG;
686		bp->b_flags |= B_LOCKED;
687		bqrelse(bp);
688		bp = newbp;
689	}
690
691	bp->b_flags &= ~B_DONE;
692	bp->b_ioflags &= ~BIO_ERROR;
693	bp->b_flags |= B_WRITEINPROG | B_CACHE;
694	bp->b_iocmd = BIO_WRITE;
695
696	bp->b_vp->v_numoutput++;
697	vfs_busy_pages(bp, 1);
698	if (curproc != idleproc)
699		curproc->p_stats->p_ru.ru_oublock++;
700	splx(s);
701	if (oldflags & B_ASYNC)
702		BUF_KERNPROC(bp);
703	BUF_STRATEGY(bp);
704
705	if ((oldflags & B_ASYNC) == 0) {
706		int rtval = bufwait(bp);
707		brelse(bp);
708		return (rtval);
709	} else {
710		/*
711		 * don't allow the async write to saturate the I/O
712		 * system.  There is no chance of deadlock here because
713		 * we are blocking on I/O that is already in-progress.
714		 */
715		waitrunningbufspace();
716	}
717
718	return (0);
719}
720
721/*
722 * Complete a background write started from bwrite.
723 */
724static void
725vfs_backgroundwritedone(bp)
726	struct buf *bp;
727{
728	struct buf *origbp;
729
730	/*
731	 * Find the original buffer that we are writing.
732	 */
733	if ((origbp = gbincore(bp->b_vp, bp->b_lblkno)) == NULL)
734		panic("backgroundwritedone: lost buffer");
735	/*
736	 * Process dependencies then return any unfinished ones.
737	 */
738	if (LIST_FIRST(&bp->b_dep) != NULL)
739		buf_complete(bp);
740	if (LIST_FIRST(&bp->b_dep) != NULL)
741		buf_movedeps(bp, origbp);
742	/*
743	 * Clear the BX_BKGRDINPROG flag in the original buffer
744	 * and awaken it if it is waiting for the write to complete.
745	 * If BX_BKGRDINPROG is not set in the original buffer it must
746	 * have been released and re-instantiated - which is not legal.
747	 */
748	KASSERT((origbp->b_xflags & BX_BKGRDINPROG), ("backgroundwritedone: lost buffer2"));
749	origbp->b_xflags &= ~BX_BKGRDINPROG;
750	if (origbp->b_xflags & BX_BKGRDWAIT) {
751		origbp->b_xflags &= ~BX_BKGRDWAIT;
752		wakeup(&origbp->b_xflags);
753	}
754	/*
755	 * Clear the B_LOCKED flag and remove it from the locked
756	 * queue if it currently resides there.
757	 */
758	origbp->b_flags &= ~B_LOCKED;
759	if (BUF_LOCK(origbp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
760		bremfree(origbp);
761		bqrelse(origbp);
762	}
763	/*
764	 * This buffer is marked B_NOCACHE, so when it is released
765	 * by biodone, it will be tossed. We mark it with BIO_READ
766	 * to avoid biodone doing a second vwakeup.
767	 */
768	bp->b_flags |= B_NOCACHE;
769	bp->b_iocmd = BIO_READ;
770	bp->b_flags &= ~(B_CACHE | B_DONE);
771	bp->b_iodone = 0;
772	bufdone(bp);
773}
774
775/*
776 * Delayed write. (Buffer is marked dirty).  Do not bother writing
777 * anything if the buffer is marked invalid.
778 *
779 * Note that since the buffer must be completely valid, we can safely
780 * set B_CACHE.  In fact, we have to set B_CACHE here rather then in
781 * biodone() in order to prevent getblk from writing the buffer
782 * out synchronously.
783 */
784void
785bdwrite(struct buf * bp)
786{
787	if (BUF_REFCNT(bp) == 0)
788		panic("bdwrite: buffer is not busy");
789
790	if (bp->b_flags & B_INVAL) {
791		brelse(bp);
792		return;
793	}
794	bdirty(bp);
795
796	/*
797	 * Set B_CACHE, indicating that the buffer is fully valid.  This is
798	 * true even of NFS now.
799	 */
800	bp->b_flags |= B_CACHE;
801
802	/*
803	 * This bmap keeps the system from needing to do the bmap later,
804	 * perhaps when the system is attempting to do a sync.  Since it
805	 * is likely that the indirect block -- or whatever other datastructure
806	 * that the filesystem needs is still in memory now, it is a good
807	 * thing to do this.  Note also, that if the pageout daemon is
808	 * requesting a sync -- there might not be enough memory to do
809	 * the bmap then...  So, this is important to do.
810	 */
811	if (bp->b_lblkno == bp->b_blkno) {
812		VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
813	}
814
815	/*
816	 * Set the *dirty* buffer range based upon the VM system dirty pages.
817	 */
818	vfs_setdirty(bp);
819
820	/*
821	 * We need to do this here to satisfy the vnode_pager and the
822	 * pageout daemon, so that it thinks that the pages have been
823	 * "cleaned".  Note that since the pages are in a delayed write
824	 * buffer -- the VFS layer "will" see that the pages get written
825	 * out on the next sync, or perhaps the cluster will be completed.
826	 */
827	vfs_clean_pages(bp);
828	bqrelse(bp);
829
830	/*
831	 * Wakeup the buffer flushing daemon if we have a lot of dirty
832	 * buffers (midpoint between our recovery point and our stall
833	 * point).
834	 */
835	bd_wakeup((lodirtybuffers + hidirtybuffers) / 2);
836
837	/*
838	 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
839	 * due to the softdep code.
840	 */
841}
842
843/*
844 *	bdirty:
845 *
846 *	Turn buffer into delayed write request.  We must clear BIO_READ and
847 *	B_RELBUF, and we must set B_DELWRI.  We reassign the buffer to
848 *	itself to properly update it in the dirty/clean lists.  We mark it
849 *	B_DONE to ensure that any asynchronization of the buffer properly
850 *	clears B_DONE ( else a panic will occur later ).
851 *
852 *	bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
853 *	might have been set pre-getblk().  Unlike bwrite/bdwrite, bdirty()
854 *	should only be called if the buffer is known-good.
855 *
856 *	Since the buffer is not on a queue, we do not update the numfreebuffers
857 *	count.
858 *
859 *	Must be called at splbio().
860 *	The buffer must be on QUEUE_NONE.
861 */
862void
863bdirty(bp)
864	struct buf *bp;
865{
866	KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
867	bp->b_flags &= ~(B_RELBUF);
868	bp->b_iocmd = BIO_WRITE;
869
870	if ((bp->b_flags & B_DELWRI) == 0) {
871		bp->b_flags |= B_DONE | B_DELWRI;
872		reassignbuf(bp, bp->b_vp);
873		++numdirtybuffers;
874		bd_wakeup((lodirtybuffers + hidirtybuffers) / 2);
875	}
876}
877
878/*
879 *	bundirty:
880 *
881 *	Clear B_DELWRI for buffer.
882 *
883 *	Since the buffer is not on a queue, we do not update the numfreebuffers
884 *	count.
885 *
886 *	Must be called at splbio().
887 *	The buffer must be on QUEUE_NONE.
888 */
889
890void
891bundirty(bp)
892	struct buf *bp;
893{
894	KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
895
896	if (bp->b_flags & B_DELWRI) {
897		bp->b_flags &= ~B_DELWRI;
898		reassignbuf(bp, bp->b_vp);
899		--numdirtybuffers;
900		numdirtywakeup(lodirtybuffers);
901	}
902	/*
903	 * Since it is now being written, we can clear its deferred write flag.
904	 */
905	bp->b_flags &= ~B_DEFERRED;
906}
907
908/*
909 *	bawrite:
910 *
911 *	Asynchronous write.  Start output on a buffer, but do not wait for
912 *	it to complete.  The buffer is released when the output completes.
913 *
914 *	bwrite() ( or the VOP routine anyway ) is responsible for handling
915 *	B_INVAL buffers.  Not us.
916 */
917void
918bawrite(struct buf * bp)
919{
920	bp->b_flags |= B_ASYNC;
921	(void) BUF_WRITE(bp);
922}
923
924/*
925 *	bowrite:
926 *
927 *	Ordered write.  Start output on a buffer, and flag it so that the
928 *	device will write it in the order it was queued.  The buffer is
929 *	released when the output completes.  bwrite() ( or the VOP routine
930 *	anyway ) is responsible for handling B_INVAL buffers.
931 */
932int
933bowrite(struct buf * bp)
934{
935	bp->b_ioflags |= BIO_ORDERED;
936	bp->b_flags |= B_ASYNC;
937	return (BUF_WRITE(bp));
938}
939
940/*
941 *	bwillwrite:
942 *
943 *	Called prior to the locking of any vnodes when we are expecting to
944 *	write.  We do not want to starve the buffer cache with too many
945 *	dirty buffers so we block here.  By blocking prior to the locking
946 *	of any vnodes we attempt to avoid the situation where a locked vnode
947 *	prevents the various system daemons from flushing related buffers.
948 */
949
950void
951bwillwrite(void)
952{
953	if (numdirtybuffers >= hidirtybuffers) {
954		int s;
955
956		s = splbio();
957		while (numdirtybuffers >= hidirtybuffers) {
958			bd_wakeup(1);
959			needsbuffer |= VFS_BIO_NEED_DIRTYFLUSH;
960			tsleep(&needsbuffer, (PRIBIO + 4), "flswai", 0);
961		}
962		splx(s);
963	}
964}
965
966/*
967 * Return true if we have too many dirty buffers.
968 */
969int
970buf_dirty_count_severe(void)
971{
972	return(numdirtybuffers >= hidirtybuffers);
973}
974
975/*
976 *	brelse:
977 *
978 *	Release a busy buffer and, if requested, free its resources.  The
979 *	buffer will be stashed in the appropriate bufqueue[] allowing it
980 *	to be accessed later as a cache entity or reused for other purposes.
981 */
982void
983brelse(struct buf * bp)
984{
985	int s;
986
987	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
988
989	s = splbio();
990
991	if (bp->b_flags & B_LOCKED)
992		bp->b_ioflags &= ~BIO_ERROR;
993
994	if (bp->b_iocmd == BIO_WRITE &&
995	    (bp->b_ioflags & BIO_ERROR) &&
996	    !(bp->b_flags & B_INVAL)) {
997		/*
998		 * Failed write, redirty.  Must clear BIO_ERROR to prevent
999		 * pages from being scrapped.  If B_INVAL is set then
1000		 * this case is not run and the next case is run to
1001		 * destroy the buffer.  B_INVAL can occur if the buffer
1002		 * is outside the range supported by the underlying device.
1003		 */
1004		bp->b_ioflags &= ~BIO_ERROR;
1005		bdirty(bp);
1006	} else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
1007	    (bp->b_ioflags & BIO_ERROR) ||
1008	    bp->b_iocmd == BIO_DELETE || (bp->b_bufsize <= 0)) {
1009		/*
1010		 * Either a failed I/O or we were asked to free or not
1011		 * cache the buffer.
1012		 */
1013		bp->b_flags |= B_INVAL;
1014		if (LIST_FIRST(&bp->b_dep) != NULL)
1015			buf_deallocate(bp);
1016		if (bp->b_flags & B_DELWRI) {
1017			--numdirtybuffers;
1018			numdirtywakeup(lodirtybuffers);
1019		}
1020		bp->b_flags &= ~(B_DELWRI | B_CACHE);
1021		if ((bp->b_flags & B_VMIO) == 0) {
1022			if (bp->b_bufsize)
1023				allocbuf(bp, 0);
1024			if (bp->b_vp)
1025				brelvp(bp);
1026		}
1027	}
1028
1029	/*
1030	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_release()
1031	 * is called with B_DELWRI set, the underlying pages may wind up
1032	 * getting freed causing a previous write (bdwrite()) to get 'lost'
1033	 * because pages associated with a B_DELWRI bp are marked clean.
1034	 *
1035	 * We still allow the B_INVAL case to call vfs_vmio_release(), even
1036	 * if B_DELWRI is set.
1037	 *
1038	 * If B_DELWRI is not set we may have to set B_RELBUF if we are low
1039	 * on pages to return pages to the VM page queues.
1040	 */
1041	if (bp->b_flags & B_DELWRI)
1042		bp->b_flags &= ~B_RELBUF;
1043	else if (vm_page_count_severe() && !(bp->b_xflags & BX_BKGRDINPROG))
1044		bp->b_flags |= B_RELBUF;
1045
1046	/*
1047	 * VMIO buffer rundown.  It is not very necessary to keep a VMIO buffer
1048	 * constituted, not even NFS buffers now.  Two flags effect this.  If
1049	 * B_INVAL, the struct buf is invalidated but the VM object is kept
1050	 * around ( i.e. so it is trivial to reconstitute the buffer later ).
1051	 *
1052	 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
1053	 * invalidated.  BIO_ERROR cannot be set for a failed write unless the
1054	 * buffer is also B_INVAL because it hits the re-dirtying code above.
1055	 *
1056	 * Normally we can do this whether a buffer is B_DELWRI or not.  If
1057	 * the buffer is an NFS buffer, it is tracking piecemeal writes or
1058	 * the commit state and we cannot afford to lose the buffer. If the
1059	 * buffer has a background write in progress, we need to keep it
1060	 * around to prevent it from being reconstituted and starting a second
1061	 * background write.
1062	 */
1063	if ((bp->b_flags & B_VMIO)
1064	    && !(bp->b_vp->v_tag == VT_NFS &&
1065		 !vn_isdisk(bp->b_vp, NULL) &&
1066		 (bp->b_flags & B_DELWRI))
1067	    ) {
1068
1069		int i, j, resid;
1070		vm_page_t m;
1071		off_t foff;
1072		vm_pindex_t poff;
1073		vm_object_t obj;
1074		struct vnode *vp;
1075
1076		vp = bp->b_vp;
1077
1078		/*
1079		 * Get the base offset and length of the buffer.  Note that
1080		 * for block sizes that are less then PAGE_SIZE, the b_data
1081		 * base of the buffer does not represent exactly b_offset and
1082		 * neither b_offset nor b_size are necessarily page aligned.
1083		 * Instead, the starting position of b_offset is:
1084		 *
1085		 * 	b_data + (b_offset & PAGE_MASK)
1086		 *
1087		 * block sizes less then DEV_BSIZE (usually 512) are not
1088		 * supported due to the page granularity bits (m->valid,
1089		 * m->dirty, etc...).
1090		 *
1091		 * See man buf(9) for more information
1092		 */
1093		resid = bp->b_bufsize;
1094		foff = bp->b_offset;
1095
1096		for (i = 0; i < bp->b_npages; i++) {
1097			int had_bogus = 0;
1098
1099			m = bp->b_pages[i];
1100			vm_page_flag_clear(m, PG_ZERO);
1101
1102			/*
1103			 * If we hit a bogus page, fixup *all* the bogus pages
1104			 * now.
1105			 */
1106			if (m == bogus_page) {
1107				VOP_GETVOBJECT(vp, &obj);
1108				poff = OFF_TO_IDX(bp->b_offset);
1109				had_bogus = 1;
1110
1111				for (j = i; j < bp->b_npages; j++) {
1112					vm_page_t mtmp;
1113					mtmp = bp->b_pages[j];
1114					if (mtmp == bogus_page) {
1115						mtmp = vm_page_lookup(obj, poff + j);
1116						if (!mtmp) {
1117							panic("brelse: page missing\n");
1118						}
1119						bp->b_pages[j] = mtmp;
1120					}
1121				}
1122
1123				if ((bp->b_flags & B_INVAL) == 0) {
1124					pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
1125				}
1126				m = bp->b_pages[i];
1127			}
1128			if ((bp->b_flags & B_NOCACHE) || (bp->b_ioflags & BIO_ERROR)) {
1129				int poffset = foff & PAGE_MASK;
1130				int presid = resid > (PAGE_SIZE - poffset) ?
1131					(PAGE_SIZE - poffset) : resid;
1132
1133				KASSERT(presid >= 0, ("brelse: extra page"));
1134				vm_page_set_invalid(m, poffset, presid);
1135				if (had_bogus)
1136					printf("avoided corruption bug in bogus_page/brelse code\n");
1137			}
1138			resid -= PAGE_SIZE - (foff & PAGE_MASK);
1139			foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
1140		}
1141
1142		if (bp->b_flags & (B_INVAL | B_RELBUF))
1143			vfs_vmio_release(bp);
1144
1145	} else if (bp->b_flags & B_VMIO) {
1146
1147		if (bp->b_flags & (B_INVAL | B_RELBUF))
1148			vfs_vmio_release(bp);
1149
1150	}
1151
1152	if (bp->b_qindex != QUEUE_NONE)
1153		panic("brelse: free buffer onto another queue???");
1154	if (BUF_REFCNT(bp) > 1) {
1155		/* do not release to free list */
1156		BUF_UNLOCK(bp);
1157		splx(s);
1158		return;
1159	}
1160
1161	/* enqueue */
1162
1163	/* buffers with no memory */
1164	if (bp->b_bufsize == 0) {
1165		bp->b_flags |= B_INVAL;
1166		bp->b_xflags &= ~BX_BKGRDWRITE;
1167		if (bp->b_xflags & BX_BKGRDINPROG)
1168			panic("losing buffer 1");
1169		if (bp->b_kvasize) {
1170			bp->b_qindex = QUEUE_EMPTYKVA;
1171		} else {
1172			bp->b_qindex = QUEUE_EMPTY;
1173		}
1174		TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1175		LIST_REMOVE(bp, b_hash);
1176		LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1177		bp->b_dev = NODEV;
1178	/* buffers with junk contents */
1179	} else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) || (bp->b_ioflags & BIO_ERROR)) {
1180		bp->b_flags |= B_INVAL;
1181		bp->b_xflags &= ~BX_BKGRDWRITE;
1182		if (bp->b_xflags & BX_BKGRDINPROG)
1183			panic("losing buffer 2");
1184		bp->b_qindex = QUEUE_CLEAN;
1185		TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1186		LIST_REMOVE(bp, b_hash);
1187		LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1188		bp->b_dev = NODEV;
1189
1190	/* buffers that are locked */
1191	} else if (bp->b_flags & B_LOCKED) {
1192		bp->b_qindex = QUEUE_LOCKED;
1193		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1194
1195	/* remaining buffers */
1196	} else {
1197		switch(bp->b_flags & (B_DELWRI|B_AGE)) {
1198		case B_DELWRI | B_AGE:
1199		    bp->b_qindex = QUEUE_DIRTY;
1200		    TAILQ_INSERT_HEAD(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1201		    break;
1202		case B_DELWRI:
1203		    bp->b_qindex = QUEUE_DIRTY;
1204		    TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1205		    break;
1206		case B_AGE:
1207		    bp->b_qindex = QUEUE_CLEAN;
1208		    TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1209		    break;
1210		default:
1211		    bp->b_qindex = QUEUE_CLEAN;
1212		    TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1213		    break;
1214		}
1215	}
1216
1217	/*
1218	 * If B_INVAL, clear B_DELWRI.  We've already placed the buffer
1219	 * on the correct queue.
1220	 */
1221	if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) {
1222		bp->b_flags &= ~B_DELWRI;
1223		--numdirtybuffers;
1224		numdirtywakeup(lodirtybuffers);
1225	}
1226
1227	/*
1228	 * Fixup numfreebuffers count.  The bp is on an appropriate queue
1229	 * unless locked.  We then bump numfreebuffers if it is not B_DELWRI.
1230	 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1231	 * if B_INVAL is set ).
1232	 */
1233
1234	if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI))
1235		bufcountwakeup();
1236
1237	/*
1238	 * Something we can maybe free.
1239	 */
1240
1241	if (bp->b_bufsize || bp->b_kvasize)
1242		bufspacewakeup();
1243
1244	/* unlock */
1245	BUF_UNLOCK(bp);
1246	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1247	bp->b_ioflags &= ~BIO_ORDERED;
1248	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
1249		panic("brelse: not dirty");
1250	splx(s);
1251}
1252
1253/*
1254 * Release a buffer back to the appropriate queue but do not try to free
1255 * it.  The buffer is expected to be used again soon.
1256 *
1257 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1258 * biodone() to requeue an async I/O on completion.  It is also used when
1259 * known good buffers need to be requeued but we think we may need the data
1260 * again soon.
1261 */
1262void
1263bqrelse(struct buf * bp)
1264{
1265	int s;
1266
1267	s = splbio();
1268
1269	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1270
1271	if (bp->b_qindex != QUEUE_NONE)
1272		panic("bqrelse: free buffer onto another queue???");
1273	if (BUF_REFCNT(bp) > 1) {
1274		/* do not release to free list */
1275		BUF_UNLOCK(bp);
1276		splx(s);
1277		return;
1278	}
1279	if (bp->b_flags & B_LOCKED) {
1280		bp->b_ioflags &= ~BIO_ERROR;
1281		bp->b_qindex = QUEUE_LOCKED;
1282		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1283		/* buffers with stale but valid contents */
1284	} else if (bp->b_flags & B_DELWRI) {
1285		bp->b_qindex = QUEUE_DIRTY;
1286		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1287	} else if (vm_page_count_severe()) {
1288		/*
1289		 * We are too low on memory, we have to try to free the
1290		 * buffer (most importantly: the wired pages making up its
1291		 * backing store) *now*.
1292		 */
1293		splx(s);
1294		brelse(bp);
1295		return;
1296	} else {
1297		bp->b_qindex = QUEUE_CLEAN;
1298		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1299	}
1300
1301	if ((bp->b_flags & B_LOCKED) == 0 &&
1302	    ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))) {
1303		bufcountwakeup();
1304	}
1305
1306	/*
1307	 * Something we can maybe wakeup
1308	 */
1309	if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1310		bufspacewakeup();
1311
1312	/* unlock */
1313	BUF_UNLOCK(bp);
1314	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1315	bp->b_ioflags &= ~BIO_ORDERED;
1316	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
1317		panic("bqrelse: not dirty");
1318	splx(s);
1319}
1320
1321static void
1322vfs_vmio_release(bp)
1323	struct buf *bp;
1324{
1325	int i, s;
1326	vm_page_t m;
1327
1328	s = splvm();
1329	for (i = 0; i < bp->b_npages; i++) {
1330		m = bp->b_pages[i];
1331		bp->b_pages[i] = NULL;
1332		/*
1333		 * In order to keep page LRU ordering consistent, put
1334		 * everything on the inactive queue.
1335		 */
1336		vm_page_unwire(m, 0);
1337		/*
1338		 * We don't mess with busy pages, it is
1339		 * the responsibility of the process that
1340		 * busied the pages to deal with them.
1341		 */
1342		if ((m->flags & PG_BUSY) || (m->busy != 0))
1343			continue;
1344
1345		if (m->wire_count == 0) {
1346			vm_page_flag_clear(m, PG_ZERO);
1347			/*
1348			 * Might as well free the page if we can and it has
1349			 * no valid data.
1350			 */
1351			if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) {
1352				vm_page_busy(m);
1353				vm_page_protect(m, VM_PROT_NONE);
1354				vm_page_free(m);
1355			} else if (vm_page_count_severe()) {
1356				vm_page_try_to_cache(m);
1357			}
1358		}
1359	}
1360	splx(s);
1361	pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
1362	if (bp->b_bufsize) {
1363		bufspacewakeup();
1364		bp->b_bufsize = 0;
1365	}
1366	bp->b_npages = 0;
1367	bp->b_flags &= ~B_VMIO;
1368	if (bp->b_vp)
1369		brelvp(bp);
1370}
1371
1372/*
1373 * Check to see if a block is currently memory resident.
1374 */
1375struct buf *
1376gbincore(struct vnode * vp, daddr_t blkno)
1377{
1378	struct buf *bp;
1379	struct bufhashhdr *bh;
1380
1381	bh = bufhash(vp, blkno);
1382
1383	/* Search hash chain */
1384	LIST_FOREACH(bp, bh, b_hash) {
1385		/* hit */
1386		if (bp->b_vp == vp && bp->b_lblkno == blkno &&
1387		    (bp->b_flags & B_INVAL) == 0) {
1388			break;
1389		}
1390	}
1391	return (bp);
1392}
1393
1394/*
1395 *	vfs_bio_awrite:
1396 *
1397 *	Implement clustered async writes for clearing out B_DELWRI buffers.
1398 *	This is much better then the old way of writing only one buffer at
1399 *	a time.  Note that we may not be presented with the buffers in the
1400 *	correct order, so we search for the cluster in both directions.
1401 */
1402int
1403vfs_bio_awrite(struct buf * bp)
1404{
1405	int i;
1406	int j;
1407	daddr_t lblkno = bp->b_lblkno;
1408	struct vnode *vp = bp->b_vp;
1409	int s;
1410	int ncl;
1411	struct buf *bpa;
1412	int nwritten;
1413	int size;
1414	int maxcl;
1415
1416	s = splbio();
1417	/*
1418	 * right now we support clustered writing only to regular files.  If
1419	 * we find a clusterable block we could be in the middle of a cluster
1420	 * rather then at the beginning.
1421	 */
1422	if ((vp->v_type == VREG) &&
1423	    (vp->v_mount != 0) && /* Only on nodes that have the size info */
1424	    (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
1425
1426		size = vp->v_mount->mnt_stat.f_iosize;
1427		maxcl = MAXPHYS / size;
1428
1429		for (i = 1; i < maxcl; i++) {
1430			if ((bpa = gbincore(vp, lblkno + i)) &&
1431			    BUF_REFCNT(bpa) == 0 &&
1432			    ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1433			    (B_DELWRI | B_CLUSTEROK)) &&
1434			    (bpa->b_bufsize == size)) {
1435				if ((bpa->b_blkno == bpa->b_lblkno) ||
1436				    (bpa->b_blkno !=
1437				     bp->b_blkno + ((i * size) >> DEV_BSHIFT)))
1438					break;
1439			} else {
1440				break;
1441			}
1442		}
1443		for (j = 1; i + j <= maxcl && j <= lblkno; j++) {
1444			if ((bpa = gbincore(vp, lblkno - j)) &&
1445			    BUF_REFCNT(bpa) == 0 &&
1446			    ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1447			    (B_DELWRI | B_CLUSTEROK)) &&
1448			    (bpa->b_bufsize == size)) {
1449				if ((bpa->b_blkno == bpa->b_lblkno) ||
1450				    (bpa->b_blkno !=
1451				     bp->b_blkno - ((j * size) >> DEV_BSHIFT)))
1452					break;
1453			} else {
1454				break;
1455			}
1456		}
1457		--j;
1458		ncl = i + j;
1459		/*
1460		 * this is a possible cluster write
1461		 */
1462		if (ncl != 1) {
1463			nwritten = cluster_wbuild(vp, size, lblkno - j, ncl);
1464			splx(s);
1465			return nwritten;
1466		}
1467	}
1468
1469	BUF_LOCK(bp, LK_EXCLUSIVE);
1470	bremfree(bp);
1471	bp->b_flags |= B_ASYNC;
1472
1473	splx(s);
1474	/*
1475	 * default (old) behavior, writing out only one block
1476	 *
1477	 * XXX returns b_bufsize instead of b_bcount for nwritten?
1478	 */
1479	nwritten = bp->b_bufsize;
1480	(void) BUF_WRITE(bp);
1481
1482	return nwritten;
1483}
1484
1485/*
1486 *	getnewbuf:
1487 *
1488 *	Find and initialize a new buffer header, freeing up existing buffers
1489 *	in the bufqueues as necessary.  The new buffer is returned locked.
1490 *
1491 *	Important:  B_INVAL is not set.  If the caller wishes to throw the
1492 *	buffer away, the caller must set B_INVAL prior to calling brelse().
1493 *
1494 *	We block if:
1495 *		We have insufficient buffer headers
1496 *		We have insufficient buffer space
1497 *		buffer_map is too fragmented ( space reservation fails )
1498 *		If we have to flush dirty buffers ( but we try to avoid this )
1499 *
1500 *	To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1501 *	Instead we ask the buf daemon to do it for us.  We attempt to
1502 *	avoid piecemeal wakeups of the pageout daemon.
1503 */
1504
1505static struct buf *
1506getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
1507{
1508	struct buf *bp;
1509	struct buf *nbp;
1510	int defrag = 0;
1511	int nqindex;
1512	static int flushingbufs;
1513
1514	/*
1515	 * We can't afford to block since we might be holding a vnode lock,
1516	 * which may prevent system daemons from running.  We deal with
1517	 * low-memory situations by proactively returning memory and running
1518	 * async I/O rather then sync I/O.
1519	 */
1520
1521	++getnewbufcalls;
1522	--getnewbufrestarts;
1523restart:
1524	++getnewbufrestarts;
1525
1526	/*
1527	 * Setup for scan.  If we do not have enough free buffers,
1528	 * we setup a degenerate case that immediately fails.  Note
1529	 * that if we are specially marked process, we are allowed to
1530	 * dip into our reserves.
1531	 *
1532	 * The scanning sequence is nominally:  EMPTY->EMPTYKVA->CLEAN
1533	 *
1534	 * We start with EMPTYKVA.  If the list is empty we backup to EMPTY.
1535	 * However, there are a number of cases (defragging, reusing, ...)
1536	 * where we cannot backup.
1537	 */
1538	nqindex = QUEUE_EMPTYKVA;
1539	nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA]);
1540
1541	if (nbp == NULL) {
1542		/*
1543		 * If no EMPTYKVA buffers and we are either
1544		 * defragging or reusing, locate a CLEAN buffer
1545		 * to free or reuse.  If bufspace useage is low
1546		 * skip this step so we can allocate a new buffer.
1547		 */
1548		if (defrag || bufspace >= lobufspace) {
1549			nqindex = QUEUE_CLEAN;
1550			nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
1551		}
1552
1553		/*
1554		 * Nada.  If we are allowed to allocate an EMPTY
1555		 * buffer, go get one.
1556		 */
1557		if (nbp == NULL && defrag == 0 && bufspace < hibufspace) {
1558			nqindex = QUEUE_EMPTY;
1559			nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1560		}
1561	}
1562
1563	/*
1564	 * Run scan, possibly freeing data and/or kva mappings on the fly
1565	 * depending.
1566	 */
1567
1568	while ((bp = nbp) != NULL) {
1569		int qindex = nqindex;
1570
1571		/*
1572		 * Calculate next bp ( we can only use it if we do not block
1573		 * or do other fancy things ).
1574		 */
1575		if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
1576			switch(qindex) {
1577			case QUEUE_EMPTY:
1578				nqindex = QUEUE_EMPTYKVA;
1579				if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA])))
1580					break;
1581				/* fall through */
1582			case QUEUE_EMPTYKVA:
1583				nqindex = QUEUE_CLEAN;
1584				if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN])))
1585					break;
1586				/* fall through */
1587			case QUEUE_CLEAN:
1588				/*
1589				 * nbp is NULL.
1590				 */
1591				break;
1592			}
1593		}
1594
1595		/*
1596		 * Sanity Checks
1597		 */
1598		KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp));
1599
1600		/*
1601		 * Note: we no longer distinguish between VMIO and non-VMIO
1602		 * buffers.
1603		 */
1604
1605		KASSERT((bp->b_flags & B_DELWRI) == 0, ("delwri buffer %p found in queue %d", bp, qindex));
1606
1607		/*
1608		 * If we are defragging then we need a buffer with
1609		 * b_kvasize != 0.  XXX this situation should no longer
1610		 * occur, if defrag is non-zero the buffer's b_kvasize
1611		 * should also be non-zero at this point.  XXX
1612		 */
1613		if (defrag && bp->b_kvasize == 0) {
1614			printf("Warning: defrag empty buffer %p\n", bp);
1615			continue;
1616		}
1617
1618		/*
1619		 * Start freeing the bp.  This is somewhat involved.  nbp
1620		 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1621		 */
1622
1623		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1624			panic("getnewbuf: locked buf");
1625		bremfree(bp);
1626
1627		if (qindex == QUEUE_CLEAN) {
1628			if (bp->b_flags & B_VMIO) {
1629				bp->b_flags &= ~B_ASYNC;
1630				vfs_vmio_release(bp);
1631			}
1632			if (bp->b_vp)
1633				brelvp(bp);
1634		}
1635
1636		/*
1637		 * NOTE:  nbp is now entirely invalid.  We can only restart
1638		 * the scan from this point on.
1639		 *
1640		 * Get the rest of the buffer freed up.  b_kva* is still
1641		 * valid after this operation.
1642		 */
1643
1644		if (bp->b_rcred != NOCRED) {
1645			crfree(bp->b_rcred);
1646			bp->b_rcred = NOCRED;
1647		}
1648		if (bp->b_wcred != NOCRED) {
1649			crfree(bp->b_wcred);
1650			bp->b_wcred = NOCRED;
1651		}
1652		if (LIST_FIRST(&bp->b_dep) != NULL)
1653			buf_deallocate(bp);
1654		if (bp->b_xflags & BX_BKGRDINPROG)
1655			panic("losing buffer 3");
1656		LIST_REMOVE(bp, b_hash);
1657		LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1658
1659		if (bp->b_bufsize)
1660			allocbuf(bp, 0);
1661
1662		bp->b_flags = 0;
1663		bp->b_ioflags = 0;
1664		bp->b_xflags = 0;
1665		bp->b_dev = NODEV;
1666		bp->b_vp = NULL;
1667		bp->b_blkno = bp->b_lblkno = 0;
1668		bp->b_offset = NOOFFSET;
1669		bp->b_iodone = 0;
1670		bp->b_error = 0;
1671		bp->b_resid = 0;
1672		bp->b_bcount = 0;
1673		bp->b_npages = 0;
1674		bp->b_dirtyoff = bp->b_dirtyend = 0;
1675
1676		LIST_INIT(&bp->b_dep);
1677
1678		/*
1679		 * If we are defragging then free the buffer.
1680		 */
1681		if (defrag) {
1682			bp->b_flags |= B_INVAL;
1683			bfreekva(bp);
1684			brelse(bp);
1685			defrag = 0;
1686			goto restart;
1687		}
1688
1689		if (bufspace >= hibufspace)
1690			flushingbufs = 1;
1691		if (flushingbufs && bp->b_kvasize != 0) {
1692			bp->b_flags |= B_INVAL;
1693			bfreekva(bp);
1694			brelse(bp);
1695			goto restart;
1696		}
1697		if (bufspace < lobufspace)
1698			flushingbufs = 0;
1699		break;
1700	}
1701
1702	/*
1703	 * If we exhausted our list, sleep as appropriate.  We may have to
1704	 * wakeup various daemons and write out some dirty buffers.
1705	 *
1706	 * Generally we are sleeping due to insufficient buffer space.
1707	 */
1708
1709	if (bp == NULL) {
1710		int flags;
1711		char *waitmsg;
1712
1713		if (defrag) {
1714			flags = VFS_BIO_NEED_BUFSPACE;
1715			waitmsg = "nbufkv";
1716		} else if (bufspace >= hibufspace) {
1717			waitmsg = "nbufbs";
1718			flags = VFS_BIO_NEED_BUFSPACE;
1719		} else {
1720			waitmsg = "newbuf";
1721			flags = VFS_BIO_NEED_ANY;
1722		}
1723
1724		bd_speedup();	/* heeeelp */
1725
1726		needsbuffer |= flags;
1727		while (needsbuffer & flags) {
1728			if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag,
1729			    waitmsg, slptimeo))
1730				return (NULL);
1731		}
1732	} else {
1733		/*
1734		 * We finally have a valid bp.  We aren't quite out of the
1735		 * woods, we still have to reserve kva space.  In order
1736		 * to keep fragmentation sane we only allocate kva in
1737		 * BKVASIZE chunks.
1738		 */
1739		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
1740
1741		if (maxsize != bp->b_kvasize) {
1742			vm_offset_t addr = 0;
1743
1744			bfreekva(bp);
1745
1746			if (vm_map_findspace(buffer_map,
1747				vm_map_min(buffer_map), maxsize, &addr)) {
1748				/*
1749				 * Uh oh.  Buffer map is to fragmented.  We
1750				 * must defragment the map.
1751				 */
1752				++bufdefragcnt;
1753				defrag = 1;
1754				bp->b_flags |= B_INVAL;
1755				brelse(bp);
1756				goto restart;
1757			}
1758			if (addr) {
1759				vm_map_insert(buffer_map, NULL, 0,
1760					addr, addr + maxsize,
1761					VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
1762
1763				bp->b_kvabase = (caddr_t) addr;
1764				bp->b_kvasize = maxsize;
1765				bufspace += bp->b_kvasize;
1766				++bufreusecnt;
1767			}
1768		}
1769		bp->b_data = bp->b_kvabase;
1770	}
1771	return(bp);
1772}
1773
1774/*
1775 *	buf_daemon:
1776 *
1777 *	buffer flushing daemon.  Buffers are normally flushed by the
1778 *	update daemon but if it cannot keep up this process starts to
1779 *	take the load in an attempt to prevent getnewbuf() from blocking.
1780 */
1781
1782static struct proc *bufdaemonproc;
1783
1784static struct kproc_desc buf_kp = {
1785	"bufdaemon",
1786	buf_daemon,
1787	&bufdaemonproc
1788};
1789SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp)
1790
1791static void
1792buf_daemon()
1793{
1794	int s;
1795
1796	mtx_enter(&Giant, MTX_DEF);
1797
1798	/*
1799	 * This process needs to be suspended prior to shutdown sync.
1800	 */
1801	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, bufdaemonproc,
1802	    SHUTDOWN_PRI_LAST);
1803
1804	/*
1805	 * This process is allowed to take the buffer cache to the limit
1806	 */
1807	curproc->p_flag |= P_BUFEXHAUST;
1808	s = splbio();
1809
1810	for (;;) {
1811		kthread_suspend_check(bufdaemonproc);
1812
1813		bd_request = 0;
1814
1815		/*
1816		 * Do the flush.  Limit the amount of in-transit I/O we
1817		 * allow to build up, otherwise we would completely saturate
1818		 * the I/O system.  Wakeup any waiting processes before we
1819		 * normally would so they can run in parallel with our drain.
1820		 */
1821		while (numdirtybuffers > lodirtybuffers) {
1822			if (flushbufqueues() == 0)
1823				break;
1824			waitrunningbufspace();
1825			numdirtywakeup((lodirtybuffers + hidirtybuffers) / 2);
1826		}
1827
1828		/*
1829		 * Only clear bd_request if we have reached our low water
1830		 * mark.  The buf_daemon normally waits 5 seconds and
1831		 * then incrementally flushes any dirty buffers that have
1832		 * built up, within reason.
1833		 *
1834		 * If we were unable to hit our low water mark and couldn't
1835		 * find any flushable buffers, we sleep half a second.
1836		 * Otherwise we loop immediately.
1837		 */
1838		if (numdirtybuffers <= lodirtybuffers) {
1839			/*
1840			 * We reached our low water mark, reset the
1841			 * request and sleep until we are needed again.
1842			 * The sleep is just so the suspend code works.
1843			 */
1844			bd_request = 0;
1845			tsleep(&bd_request, PVM, "psleep", hz);
1846		} else {
1847			/*
1848			 * We couldn't find any flushable dirty buffers but
1849			 * still have too many dirty buffers, we
1850			 * have to sleep and try again.  (rare)
1851			 */
1852			tsleep(&bd_request, PVM, "qsleep", hz / 2);
1853		}
1854	}
1855}
1856
1857/*
1858 *	flushbufqueues:
1859 *
1860 *	Try to flush a buffer in the dirty queue.  We must be careful to
1861 *	free up B_INVAL buffers instead of write them, which NFS is
1862 *	particularly sensitive to.
1863 */
1864
1865static int
1866flushbufqueues(void)
1867{
1868	struct buf *bp;
1869	int r = 0;
1870
1871	bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1872
1873	while (bp) {
1874		KASSERT((bp->b_flags & B_DELWRI), ("unexpected clean buffer %p", bp));
1875		if ((bp->b_flags & B_DELWRI) != 0 &&
1876		    (bp->b_xflags & BX_BKGRDINPROG) == 0) {
1877			if (bp->b_flags & B_INVAL) {
1878				if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1879					panic("flushbufqueues: locked buf");
1880				bremfree(bp);
1881				brelse(bp);
1882				++r;
1883				break;
1884			}
1885			if (LIST_FIRST(&bp->b_dep) != NULL &&
1886			    (bp->b_flags & B_DEFERRED) == 0 &&
1887			    buf_countdeps(bp, 0)) {
1888				TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY],
1889				    bp, b_freelist);
1890				TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY],
1891				    bp, b_freelist);
1892				bp->b_flags |= B_DEFERRED;
1893				bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1894				continue;
1895			}
1896			vfs_bio_awrite(bp);
1897			++r;
1898			break;
1899		}
1900		bp = TAILQ_NEXT(bp, b_freelist);
1901	}
1902	return (r);
1903}
1904
1905/*
1906 * Check to see if a block is currently memory resident.
1907 */
1908struct buf *
1909incore(struct vnode * vp, daddr_t blkno)
1910{
1911	struct buf *bp;
1912
1913	int s = splbio();
1914	bp = gbincore(vp, blkno);
1915	splx(s);
1916	return (bp);
1917}
1918
1919/*
1920 * Returns true if no I/O is needed to access the
1921 * associated VM object.  This is like incore except
1922 * it also hunts around in the VM system for the data.
1923 */
1924
1925int
1926inmem(struct vnode * vp, daddr_t blkno)
1927{
1928	vm_object_t obj;
1929	vm_offset_t toff, tinc, size;
1930	vm_page_t m;
1931	vm_ooffset_t off;
1932
1933	if (incore(vp, blkno))
1934		return 1;
1935	if (vp->v_mount == NULL)
1936		return 0;
1937	if (VOP_GETVOBJECT(vp, &obj) != 0 || (vp->v_flag & VOBJBUF) == 0)
1938		return 0;
1939
1940	size = PAGE_SIZE;
1941	if (size > vp->v_mount->mnt_stat.f_iosize)
1942		size = vp->v_mount->mnt_stat.f_iosize;
1943	off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
1944
1945	for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
1946		m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
1947		if (!m)
1948			return 0;
1949		tinc = size;
1950		if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
1951			tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
1952		if (vm_page_is_valid(m,
1953		    (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
1954			return 0;
1955	}
1956	return 1;
1957}
1958
1959/*
1960 *	vfs_setdirty:
1961 *
1962 *	Sets the dirty range for a buffer based on the status of the dirty
1963 *	bits in the pages comprising the buffer.
1964 *
1965 *	The range is limited to the size of the buffer.
1966 *
1967 *	This routine is primarily used by NFS, but is generalized for the
1968 *	B_VMIO case.
1969 */
1970static void
1971vfs_setdirty(struct buf *bp)
1972{
1973	int i;
1974	vm_object_t object;
1975
1976	/*
1977	 * Degenerate case - empty buffer
1978	 */
1979
1980	if (bp->b_bufsize == 0)
1981		return;
1982
1983	/*
1984	 * We qualify the scan for modified pages on whether the
1985	 * object has been flushed yet.  The OBJ_WRITEABLE flag
1986	 * is not cleared simply by protecting pages off.
1987	 */
1988
1989	if ((bp->b_flags & B_VMIO) == 0)
1990		return;
1991
1992	object = bp->b_pages[0]->object;
1993
1994	if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY))
1995		printf("Warning: object %p writeable but not mightbedirty\n", object);
1996	if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY))
1997		printf("Warning: object %p mightbedirty but not writeable\n", object);
1998
1999	if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) {
2000		vm_offset_t boffset;
2001		vm_offset_t eoffset;
2002
2003		/*
2004		 * test the pages to see if they have been modified directly
2005		 * by users through the VM system.
2006		 */
2007		for (i = 0; i < bp->b_npages; i++) {
2008			vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
2009			vm_page_test_dirty(bp->b_pages[i]);
2010		}
2011
2012		/*
2013		 * Calculate the encompassing dirty range, boffset and eoffset,
2014		 * (eoffset - boffset) bytes.
2015		 */
2016
2017		for (i = 0; i < bp->b_npages; i++) {
2018			if (bp->b_pages[i]->dirty)
2019				break;
2020		}
2021		boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
2022
2023		for (i = bp->b_npages - 1; i >= 0; --i) {
2024			if (bp->b_pages[i]->dirty) {
2025				break;
2026			}
2027		}
2028		eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
2029
2030		/*
2031		 * Fit it to the buffer.
2032		 */
2033
2034		if (eoffset > bp->b_bcount)
2035			eoffset = bp->b_bcount;
2036
2037		/*
2038		 * If we have a good dirty range, merge with the existing
2039		 * dirty range.
2040		 */
2041
2042		if (boffset < eoffset) {
2043			if (bp->b_dirtyoff > boffset)
2044				bp->b_dirtyoff = boffset;
2045			if (bp->b_dirtyend < eoffset)
2046				bp->b_dirtyend = eoffset;
2047		}
2048	}
2049}
2050
2051/*
2052 *	getblk:
2053 *
2054 *	Get a block given a specified block and offset into a file/device.
2055 *	The buffers B_DONE bit will be cleared on return, making it almost
2056 * 	ready for an I/O initiation.  B_INVAL may or may not be set on
2057 *	return.  The caller should clear B_INVAL prior to initiating a
2058 *	READ.
2059 *
2060 *	For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2061 *	an existing buffer.
2062 *
2063 *	For a VMIO buffer, B_CACHE is modified according to the backing VM.
2064 *	If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2065 *	and then cleared based on the backing VM.  If the previous buffer is
2066 *	non-0-sized but invalid, B_CACHE will be cleared.
2067 *
2068 *	If getblk() must create a new buffer, the new buffer is returned with
2069 *	both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2070 *	case it is returned with B_INVAL clear and B_CACHE set based on the
2071 *	backing VM.
2072 *
2073 *	getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2074 *	B_CACHE bit is clear.
2075 *
2076 *	What this means, basically, is that the caller should use B_CACHE to
2077 *	determine whether the buffer is fully valid or not and should clear
2078 *	B_INVAL prior to issuing a read.  If the caller intends to validate
2079 *	the buffer by loading its data area with something, the caller needs
2080 *	to clear B_INVAL.  If the caller does this without issuing an I/O,
2081 *	the caller should set B_CACHE ( as an optimization ), else the caller
2082 *	should issue the I/O and biodone() will set B_CACHE if the I/O was
2083 *	a write attempt or if it was a successfull read.  If the caller
2084 *	intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
2085 *	prior to issuing the READ.  biodone() will *not* clear B_INVAL.
2086 */
2087struct buf *
2088getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo)
2089{
2090	struct buf *bp;
2091	int s;
2092	struct bufhashhdr *bh;
2093
2094	if (size > MAXBSIZE)
2095		panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE);
2096
2097	s = splbio();
2098loop:
2099	/*
2100	 * Block if we are low on buffers.   Certain processes are allowed
2101	 * to completely exhaust the buffer cache.
2102         *
2103         * If this check ever becomes a bottleneck it may be better to
2104         * move it into the else, when gbincore() fails.  At the moment
2105         * it isn't a problem.
2106	 *
2107	 * XXX remove if 0 sections (clean this up after its proven)
2108         */
2109	if (numfreebuffers == 0) {
2110		if (curproc == idleproc)
2111			return NULL;
2112		needsbuffer |= VFS_BIO_NEED_ANY;
2113	}
2114
2115	if ((bp = gbincore(vp, blkno))) {
2116		/*
2117		 * Buffer is in-core.  If the buffer is not busy, it must
2118		 * be on a queue.
2119		 */
2120
2121		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
2122			if (BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL,
2123			    "getblk", slpflag, slptimeo) == ENOLCK)
2124				goto loop;
2125			splx(s);
2126			return (struct buf *) NULL;
2127		}
2128
2129		/*
2130		 * The buffer is locked.  B_CACHE is cleared if the buffer is
2131		 * invalid.  Ohterwise, for a non-VMIO buffer, B_CACHE is set
2132		 * and for a VMIO buffer B_CACHE is adjusted according to the
2133		 * backing VM cache.
2134		 */
2135		if (bp->b_flags & B_INVAL)
2136			bp->b_flags &= ~B_CACHE;
2137		else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
2138			bp->b_flags |= B_CACHE;
2139		bremfree(bp);
2140
2141		/*
2142		 * check for size inconsistancies for non-VMIO case.
2143		 */
2144
2145		if (bp->b_bcount != size) {
2146			if ((bp->b_flags & B_VMIO) == 0 ||
2147			    (size > bp->b_kvasize)) {
2148				if (bp->b_flags & B_DELWRI) {
2149					bp->b_flags |= B_NOCACHE;
2150					BUF_WRITE(bp);
2151				} else {
2152					if ((bp->b_flags & B_VMIO) &&
2153					   (LIST_FIRST(&bp->b_dep) == NULL)) {
2154						bp->b_flags |= B_RELBUF;
2155						brelse(bp);
2156					} else {
2157						bp->b_flags |= B_NOCACHE;
2158						BUF_WRITE(bp);
2159					}
2160				}
2161				goto loop;
2162			}
2163		}
2164
2165		/*
2166		 * If the size is inconsistant in the VMIO case, we can resize
2167		 * the buffer.  This might lead to B_CACHE getting set or
2168		 * cleared.  If the size has not changed, B_CACHE remains
2169		 * unchanged from its previous state.
2170		 */
2171
2172		if (bp->b_bcount != size)
2173			allocbuf(bp, size);
2174
2175		KASSERT(bp->b_offset != NOOFFSET,
2176		    ("getblk: no buffer offset"));
2177
2178		/*
2179		 * A buffer with B_DELWRI set and B_CACHE clear must
2180		 * be committed before we can return the buffer in
2181		 * order to prevent the caller from issuing a read
2182		 * ( due to B_CACHE not being set ) and overwriting
2183		 * it.
2184		 *
2185		 * Most callers, including NFS and FFS, need this to
2186		 * operate properly either because they assume they
2187		 * can issue a read if B_CACHE is not set, or because
2188		 * ( for example ) an uncached B_DELWRI might loop due
2189		 * to softupdates re-dirtying the buffer.  In the latter
2190		 * case, B_CACHE is set after the first write completes,
2191		 * preventing further loops.
2192		 */
2193
2194		if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
2195			BUF_WRITE(bp);
2196			goto loop;
2197		}
2198
2199		splx(s);
2200		bp->b_flags &= ~B_DONE;
2201	} else {
2202		/*
2203		 * Buffer is not in-core, create new buffer.  The buffer
2204		 * returned by getnewbuf() is locked.  Note that the returned
2205		 * buffer is also considered valid (not marked B_INVAL).
2206		 */
2207		int bsize, maxsize, vmio;
2208		off_t offset;
2209
2210		if (vn_isdisk(vp, NULL))
2211			bsize = DEV_BSIZE;
2212		else if (vp->v_mountedhere)
2213			bsize = vp->v_mountedhere->mnt_stat.f_iosize;
2214		else if (vp->v_mount)
2215			bsize = vp->v_mount->mnt_stat.f_iosize;
2216		else
2217			bsize = size;
2218
2219		offset = (off_t)blkno * bsize;
2220		vmio = (VOP_GETVOBJECT(vp, NULL) == 0) && (vp->v_flag & VOBJBUF);
2221		maxsize = vmio ? size + (offset & PAGE_MASK) : size;
2222		maxsize = imax(maxsize, bsize);
2223
2224		if ((bp = getnewbuf(slpflag, slptimeo, size, maxsize)) == NULL) {
2225			if (slpflag || slptimeo) {
2226				splx(s);
2227				return NULL;
2228			}
2229			goto loop;
2230		}
2231
2232		/*
2233		 * This code is used to make sure that a buffer is not
2234		 * created while the getnewbuf routine is blocked.
2235		 * This can be a problem whether the vnode is locked or not.
2236		 * If the buffer is created out from under us, we have to
2237		 * throw away the one we just created.  There is now window
2238		 * race because we are safely running at splbio() from the
2239		 * point of the duplicate buffer creation through to here,
2240		 * and we've locked the buffer.
2241		 */
2242		if (gbincore(vp, blkno)) {
2243			bp->b_flags |= B_INVAL;
2244			brelse(bp);
2245			goto loop;
2246		}
2247
2248		/*
2249		 * Insert the buffer into the hash, so that it can
2250		 * be found by incore.
2251		 */
2252		bp->b_blkno = bp->b_lblkno = blkno;
2253		bp->b_offset = offset;
2254
2255		bgetvp(vp, bp);
2256		LIST_REMOVE(bp, b_hash);
2257		bh = bufhash(vp, blkno);
2258		LIST_INSERT_HEAD(bh, bp, b_hash);
2259
2260		/*
2261		 * set B_VMIO bit.  allocbuf() the buffer bigger.  Since the
2262		 * buffer size starts out as 0, B_CACHE will be set by
2263		 * allocbuf() for the VMIO case prior to it testing the
2264		 * backing store for validity.
2265		 */
2266
2267		if (vmio) {
2268			bp->b_flags |= B_VMIO;
2269#if defined(VFS_BIO_DEBUG)
2270			if (vp->v_type != VREG)
2271				printf("getblk: vmioing file type %d???\n", vp->v_type);
2272#endif
2273		} else {
2274			bp->b_flags &= ~B_VMIO;
2275		}
2276
2277		allocbuf(bp, size);
2278
2279		splx(s);
2280		bp->b_flags &= ~B_DONE;
2281	}
2282	return (bp);
2283}
2284
2285/*
2286 * Get an empty, disassociated buffer of given size.  The buffer is initially
2287 * set to B_INVAL.
2288 */
2289struct buf *
2290geteblk(int size)
2291{
2292	struct buf *bp;
2293	int s;
2294	int maxsize;
2295
2296	maxsize = (size + BKVAMASK) & ~BKVAMASK;
2297
2298	s = splbio();
2299	while ((bp = getnewbuf(0, 0, size, maxsize)) == 0);
2300	splx(s);
2301	allocbuf(bp, size);
2302	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
2303	return (bp);
2304}
2305
2306
2307/*
2308 * This code constitutes the buffer memory from either anonymous system
2309 * memory (in the case of non-VMIO operations) or from an associated
2310 * VM object (in the case of VMIO operations).  This code is able to
2311 * resize a buffer up or down.
2312 *
2313 * Note that this code is tricky, and has many complications to resolve
2314 * deadlock or inconsistant data situations.  Tread lightly!!!
2315 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2316 * the caller.  Calling this code willy nilly can result in the loss of data.
2317 *
2318 * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
2319 * B_CACHE for the non-VMIO case.
2320 */
2321
2322int
2323allocbuf(struct buf *bp, int size)
2324{
2325	int newbsize, mbsize;
2326	int i;
2327
2328	if (BUF_REFCNT(bp) == 0)
2329		panic("allocbuf: buffer not busy");
2330
2331	if (bp->b_kvasize < size)
2332		panic("allocbuf: buffer too small");
2333
2334	if ((bp->b_flags & B_VMIO) == 0) {
2335		caddr_t origbuf;
2336		int origbufsize;
2337		/*
2338		 * Just get anonymous memory from the kernel.  Don't
2339		 * mess with B_CACHE.
2340		 */
2341		mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2342#if !defined(NO_B_MALLOC)
2343		if (bp->b_flags & B_MALLOC)
2344			newbsize = mbsize;
2345		else
2346#endif
2347			newbsize = round_page(size);
2348
2349		if (newbsize < bp->b_bufsize) {
2350#if !defined(NO_B_MALLOC)
2351			/*
2352			 * malloced buffers are not shrunk
2353			 */
2354			if (bp->b_flags & B_MALLOC) {
2355				if (newbsize) {
2356					bp->b_bcount = size;
2357				} else {
2358					free(bp->b_data, M_BIOBUF);
2359					if (bp->b_bufsize) {
2360						bufmallocspace -= bp->b_bufsize;
2361						bufspacewakeup();
2362						bp->b_bufsize = 0;
2363					}
2364					bp->b_data = bp->b_kvabase;
2365					bp->b_bcount = 0;
2366					bp->b_flags &= ~B_MALLOC;
2367				}
2368				return 1;
2369			}
2370#endif
2371			vm_hold_free_pages(
2372			    bp,
2373			    (vm_offset_t) bp->b_data + newbsize,
2374			    (vm_offset_t) bp->b_data + bp->b_bufsize);
2375		} else if (newbsize > bp->b_bufsize) {
2376#if !defined(NO_B_MALLOC)
2377			/*
2378			 * We only use malloced memory on the first allocation.
2379			 * and revert to page-allocated memory when the buffer
2380			 * grows.
2381			 */
2382			if ( (bufmallocspace < maxbufmallocspace) &&
2383				(bp->b_bufsize == 0) &&
2384				(mbsize <= PAGE_SIZE/2)) {
2385
2386				bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
2387				bp->b_bufsize = mbsize;
2388				bp->b_bcount = size;
2389				bp->b_flags |= B_MALLOC;
2390				bufmallocspace += mbsize;
2391				return 1;
2392			}
2393#endif
2394			origbuf = NULL;
2395			origbufsize = 0;
2396#if !defined(NO_B_MALLOC)
2397			/*
2398			 * If the buffer is growing on its other-than-first allocation,
2399			 * then we revert to the page-allocation scheme.
2400			 */
2401			if (bp->b_flags & B_MALLOC) {
2402				origbuf = bp->b_data;
2403				origbufsize = bp->b_bufsize;
2404				bp->b_data = bp->b_kvabase;
2405				if (bp->b_bufsize) {
2406					bufmallocspace -= bp->b_bufsize;
2407					bufspacewakeup();
2408					bp->b_bufsize = 0;
2409				}
2410				bp->b_flags &= ~B_MALLOC;
2411				newbsize = round_page(newbsize);
2412			}
2413#endif
2414			vm_hold_load_pages(
2415			    bp,
2416			    (vm_offset_t) bp->b_data + bp->b_bufsize,
2417			    (vm_offset_t) bp->b_data + newbsize);
2418#if !defined(NO_B_MALLOC)
2419			if (origbuf) {
2420				bcopy(origbuf, bp->b_data, origbufsize);
2421				free(origbuf, M_BIOBUF);
2422			}
2423#endif
2424		}
2425	} else {
2426		vm_page_t m;
2427		int desiredpages;
2428
2429		newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2430		desiredpages = (size == 0) ? 0 :
2431			num_pages((bp->b_offset & PAGE_MASK) + newbsize);
2432
2433#if !defined(NO_B_MALLOC)
2434		if (bp->b_flags & B_MALLOC)
2435			panic("allocbuf: VMIO buffer can't be malloced");
2436#endif
2437		/*
2438		 * Set B_CACHE initially if buffer is 0 length or will become
2439		 * 0-length.
2440		 */
2441		if (size == 0 || bp->b_bufsize == 0)
2442			bp->b_flags |= B_CACHE;
2443
2444		if (newbsize < bp->b_bufsize) {
2445			/*
2446			 * DEV_BSIZE aligned new buffer size is less then the
2447			 * DEV_BSIZE aligned existing buffer size.  Figure out
2448			 * if we have to remove any pages.
2449			 */
2450			if (desiredpages < bp->b_npages) {
2451				for (i = desiredpages; i < bp->b_npages; i++) {
2452					/*
2453					 * the page is not freed here -- it
2454					 * is the responsibility of
2455					 * vnode_pager_setsize
2456					 */
2457					m = bp->b_pages[i];
2458					KASSERT(m != bogus_page,
2459					    ("allocbuf: bogus page found"));
2460					while (vm_page_sleep_busy(m, TRUE, "biodep"))
2461						;
2462
2463					bp->b_pages[i] = NULL;
2464					vm_page_unwire(m, 0);
2465				}
2466				pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) +
2467				    (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages));
2468				bp->b_npages = desiredpages;
2469			}
2470		} else if (size > bp->b_bcount) {
2471			/*
2472			 * We are growing the buffer, possibly in a
2473			 * byte-granular fashion.
2474			 */
2475			struct vnode *vp;
2476			vm_object_t obj;
2477			vm_offset_t toff;
2478			vm_offset_t tinc;
2479
2480			/*
2481			 * Step 1, bring in the VM pages from the object,
2482			 * allocating them if necessary.  We must clear
2483			 * B_CACHE if these pages are not valid for the
2484			 * range covered by the buffer.
2485			 */
2486
2487			vp = bp->b_vp;
2488			VOP_GETVOBJECT(vp, &obj);
2489
2490			while (bp->b_npages < desiredpages) {
2491				vm_page_t m;
2492				vm_pindex_t pi;
2493
2494				pi = OFF_TO_IDX(bp->b_offset) + bp->b_npages;
2495				if ((m = vm_page_lookup(obj, pi)) == NULL) {
2496					/*
2497					 * note: must allocate system pages
2498					 * since blocking here could intefere
2499					 * with paging I/O, no matter which
2500					 * process we are.
2501					 */
2502					m = vm_page_alloc(obj, pi, VM_ALLOC_SYSTEM);
2503					if (m == NULL) {
2504						VM_WAIT;
2505						vm_pageout_deficit += desiredpages - bp->b_npages;
2506					} else {
2507						vm_page_wire(m);
2508						vm_page_wakeup(m);
2509						bp->b_flags &= ~B_CACHE;
2510						bp->b_pages[bp->b_npages] = m;
2511						++bp->b_npages;
2512					}
2513					continue;
2514				}
2515
2516				/*
2517				 * We found a page.  If we have to sleep on it,
2518				 * retry because it might have gotten freed out
2519				 * from under us.
2520				 *
2521				 * We can only test PG_BUSY here.  Blocking on
2522				 * m->busy might lead to a deadlock:
2523				 *
2524				 *  vm_fault->getpages->cluster_read->allocbuf
2525				 *
2526				 */
2527
2528				if (vm_page_sleep_busy(m, FALSE, "pgtblk"))
2529					continue;
2530
2531				/*
2532				 * We have a good page.  Should we wakeup the
2533				 * page daemon?
2534				 */
2535				if ((curproc != pageproc) &&
2536				    ((m->queue - m->pc) == PQ_CACHE) &&
2537				    ((cnt.v_free_count + cnt.v_cache_count) <
2538					(cnt.v_free_min + cnt.v_cache_min))) {
2539					pagedaemon_wakeup();
2540				}
2541				vm_page_flag_clear(m, PG_ZERO);
2542				vm_page_wire(m);
2543				bp->b_pages[bp->b_npages] = m;
2544				++bp->b_npages;
2545			}
2546
2547			/*
2548			 * Step 2.  We've loaded the pages into the buffer,
2549			 * we have to figure out if we can still have B_CACHE
2550			 * set.  Note that B_CACHE is set according to the
2551			 * byte-granular range ( bcount and size ), new the
2552			 * aligned range ( newbsize ).
2553			 *
2554			 * The VM test is against m->valid, which is DEV_BSIZE
2555			 * aligned.  Needless to say, the validity of the data
2556			 * needs to also be DEV_BSIZE aligned.  Note that this
2557			 * fails with NFS if the server or some other client
2558			 * extends the file's EOF.  If our buffer is resized,
2559			 * B_CACHE may remain set! XXX
2560			 */
2561
2562			toff = bp->b_bcount;
2563			tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2564
2565			while ((bp->b_flags & B_CACHE) && toff < size) {
2566				vm_pindex_t pi;
2567
2568				if (tinc > (size - toff))
2569					tinc = size - toff;
2570
2571				pi = ((bp->b_offset & PAGE_MASK) + toff) >>
2572				    PAGE_SHIFT;
2573
2574				vfs_buf_test_cache(
2575				    bp,
2576				    bp->b_offset,
2577				    toff,
2578				    tinc,
2579				    bp->b_pages[pi]
2580				);
2581				toff += tinc;
2582				tinc = PAGE_SIZE;
2583			}
2584
2585			/*
2586			 * Step 3, fixup the KVM pmap.  Remember that
2587			 * bp->b_data is relative to bp->b_offset, but
2588			 * bp->b_offset may be offset into the first page.
2589			 */
2590
2591			bp->b_data = (caddr_t)
2592			    trunc_page((vm_offset_t)bp->b_data);
2593			pmap_qenter(
2594			    (vm_offset_t)bp->b_data,
2595			    bp->b_pages,
2596			    bp->b_npages
2597			);
2598			bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
2599			    (vm_offset_t)(bp->b_offset & PAGE_MASK));
2600		}
2601	}
2602	if (newbsize < bp->b_bufsize)
2603		bufspacewakeup();
2604	bp->b_bufsize = newbsize;	/* actual buffer allocation	*/
2605	bp->b_bcount = size;		/* requested buffer size	*/
2606	return 1;
2607}
2608
2609/*
2610 *	bufwait:
2611 *
2612 *	Wait for buffer I/O completion, returning error status.  The buffer
2613 *	is left locked and B_DONE on return.  B_EINTR is converted into a EINTR
2614 *	error and cleared.
2615 */
2616int
2617bufwait(register struct buf * bp)
2618{
2619	int s;
2620
2621	s = splbio();
2622	while ((bp->b_flags & B_DONE) == 0) {
2623		if (bp->b_iocmd == BIO_READ)
2624			tsleep(bp, PRIBIO, "biord", 0);
2625		else
2626			tsleep(bp, PRIBIO, "biowr", 0);
2627	}
2628	splx(s);
2629	if (bp->b_flags & B_EINTR) {
2630		bp->b_flags &= ~B_EINTR;
2631		return (EINTR);
2632	}
2633	if (bp->b_ioflags & BIO_ERROR) {
2634		return (bp->b_error ? bp->b_error : EIO);
2635	} else {
2636		return (0);
2637	}
2638}
2639
2640 /*
2641  * Call back function from struct bio back up to struct buf.
2642  * The corresponding initialization lives in sys/conf.h:DEV_STRATEGY().
2643  */
2644void
2645bufdonebio(struct bio *bp)
2646{
2647	bufdone(bp->bio_caller2);
2648}
2649
2650/*
2651 *	bufdone:
2652 *
2653 *	Finish I/O on a buffer, optionally calling a completion function.
2654 *	This is usually called from an interrupt so process blocking is
2655 *	not allowed.
2656 *
2657 *	biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2658 *	In a non-VMIO bp, B_CACHE will be set on the next getblk()
2659 *	assuming B_INVAL is clear.
2660 *
2661 *	For the VMIO case, we set B_CACHE if the op was a read and no
2662 *	read error occured, or if the op was a write.  B_CACHE is never
2663 *	set if the buffer is invalid or otherwise uncacheable.
2664 *
2665 *	biodone does not mess with B_INVAL, allowing the I/O routine or the
2666 *	initiator to leave B_INVAL set to brelse the buffer out of existance
2667 *	in the biodone routine.
2668 */
2669void
2670bufdone(struct buf *bp)
2671{
2672	int s, error;
2673	void    (*biodone) __P((struct buf *));
2674
2675	s = splbio();
2676
2677	KASSERT(BUF_REFCNT(bp) > 0, ("biodone: bp %p not busy %d", bp, BUF_REFCNT(bp)));
2678	KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
2679
2680	bp->b_flags |= B_DONE;
2681	runningbufwakeup(bp);
2682
2683	if (bp->b_iocmd == BIO_DELETE) {
2684		brelse(bp);
2685		splx(s);
2686		return;
2687	}
2688
2689	if (bp->b_iocmd == BIO_WRITE) {
2690		vwakeup(bp);
2691	}
2692
2693	/* call optional completion function if requested */
2694	if (bp->b_iodone != NULL) {
2695		biodone = bp->b_iodone;
2696		bp->b_iodone = NULL;
2697		(*biodone) (bp);
2698		splx(s);
2699		return;
2700	}
2701	if (LIST_FIRST(&bp->b_dep) != NULL)
2702		buf_complete(bp);
2703
2704	if (bp->b_flags & B_VMIO) {
2705		int i;
2706		vm_ooffset_t foff;
2707		vm_page_t m;
2708		vm_object_t obj;
2709		int iosize;
2710		struct vnode *vp = bp->b_vp;
2711
2712		error = VOP_GETVOBJECT(vp, &obj);
2713
2714#if defined(VFS_BIO_DEBUG)
2715		if (vp->v_usecount == 0) {
2716			panic("biodone: zero vnode ref count");
2717		}
2718
2719		if (error) {
2720			panic("biodone: missing VM object");
2721		}
2722
2723		if ((vp->v_flag & VOBJBUF) == 0) {
2724			panic("biodone: vnode is not setup for merged cache");
2725		}
2726#endif
2727
2728		foff = bp->b_offset;
2729		KASSERT(bp->b_offset != NOOFFSET,
2730		    ("biodone: no buffer offset"));
2731
2732		if (error) {
2733			panic("biodone: no object");
2734		}
2735#if defined(VFS_BIO_DEBUG)
2736		if (obj->paging_in_progress < bp->b_npages) {
2737			printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n",
2738			    obj->paging_in_progress, bp->b_npages);
2739		}
2740#endif
2741
2742		/*
2743		 * Set B_CACHE if the op was a normal read and no error
2744		 * occured.  B_CACHE is set for writes in the b*write()
2745		 * routines.
2746		 */
2747		iosize = bp->b_bcount - bp->b_resid;
2748		if (bp->b_iocmd == BIO_READ &&
2749		    !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
2750		    !(bp->b_ioflags & BIO_ERROR)) {
2751			bp->b_flags |= B_CACHE;
2752		}
2753
2754		for (i = 0; i < bp->b_npages; i++) {
2755			int bogusflag = 0;
2756			int resid;
2757
2758			resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
2759			if (resid > iosize)
2760				resid = iosize;
2761
2762			/*
2763			 * cleanup bogus pages, restoring the originals
2764			 */
2765			m = bp->b_pages[i];
2766			if (m == bogus_page) {
2767				bogusflag = 1;
2768				m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2769				if (m == NULL)
2770					panic("biodone: page disappeared!");
2771				bp->b_pages[i] = m;
2772				pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2773			}
2774#if defined(VFS_BIO_DEBUG)
2775			if (OFF_TO_IDX(foff) != m->pindex) {
2776				printf(
2777"biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2778				    (unsigned long)foff, m->pindex);
2779			}
2780#endif
2781
2782			/*
2783			 * In the write case, the valid and clean bits are
2784			 * already changed correctly ( see bdwrite() ), so we
2785			 * only need to do this here in the read case.
2786			 */
2787			if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
2788				vfs_page_set_valid(bp, foff, i, m);
2789			}
2790			vm_page_flag_clear(m, PG_ZERO);
2791
2792			/*
2793			 * when debugging new filesystems or buffer I/O methods, this
2794			 * is the most common error that pops up.  if you see this, you
2795			 * have not set the page busy flag correctly!!!
2796			 */
2797			if (m->busy == 0) {
2798				printf("biodone: page busy < 0, "
2799				    "pindex: %d, foff: 0x(%x,%x), "
2800				    "resid: %d, index: %d\n",
2801				    (int) m->pindex, (int)(foff >> 32),
2802						(int) foff & 0xffffffff, resid, i);
2803				if (!vn_isdisk(vp, NULL))
2804					printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2805					    bp->b_vp->v_mount->mnt_stat.f_iosize,
2806					    (int) bp->b_lblkno,
2807					    bp->b_flags, bp->b_npages);
2808				else
2809					printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2810					    (int) bp->b_lblkno,
2811					    bp->b_flags, bp->b_npages);
2812				printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2813				    m->valid, m->dirty, m->wire_count);
2814				panic("biodone: page busy < 0\n");
2815			}
2816			vm_page_io_finish(m);
2817			vm_object_pip_subtract(obj, 1);
2818			foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
2819			iosize -= resid;
2820		}
2821		if (obj)
2822			vm_object_pip_wakeupn(obj, 0);
2823	}
2824
2825	/*
2826	 * For asynchronous completions, release the buffer now. The brelse
2827	 * will do a wakeup there if necessary - so no need to do a wakeup
2828	 * here in the async case. The sync case always needs to do a wakeup.
2829	 */
2830
2831	if (bp->b_flags & B_ASYNC) {
2832		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
2833			brelse(bp);
2834		else
2835			bqrelse(bp);
2836	} else {
2837		wakeup(bp);
2838	}
2839	splx(s);
2840}
2841
2842/*
2843 * This routine is called in lieu of iodone in the case of
2844 * incomplete I/O.  This keeps the busy status for pages
2845 * consistant.
2846 */
2847void
2848vfs_unbusy_pages(struct buf * bp)
2849{
2850	int i;
2851
2852	runningbufwakeup(bp);
2853	if (bp->b_flags & B_VMIO) {
2854		struct vnode *vp = bp->b_vp;
2855		vm_object_t obj;
2856
2857		VOP_GETVOBJECT(vp, &obj);
2858
2859		for (i = 0; i < bp->b_npages; i++) {
2860			vm_page_t m = bp->b_pages[i];
2861
2862			if (m == bogus_page) {
2863				m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
2864				if (!m) {
2865					panic("vfs_unbusy_pages: page missing\n");
2866				}
2867				bp->b_pages[i] = m;
2868				pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2869			}
2870			vm_object_pip_subtract(obj, 1);
2871			vm_page_flag_clear(m, PG_ZERO);
2872			vm_page_io_finish(m);
2873		}
2874		vm_object_pip_wakeupn(obj, 0);
2875	}
2876}
2877
2878/*
2879 * vfs_page_set_valid:
2880 *
2881 *	Set the valid bits in a page based on the supplied offset.   The
2882 *	range is restricted to the buffer's size.
2883 *
2884 *	This routine is typically called after a read completes.
2885 */
2886static void
2887vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m)
2888{
2889	vm_ooffset_t soff, eoff;
2890
2891	/*
2892	 * Start and end offsets in buffer.  eoff - soff may not cross a
2893	 * page boundry or cross the end of the buffer.  The end of the
2894	 * buffer, in this case, is our file EOF, not the allocation size
2895	 * of the buffer.
2896	 */
2897	soff = off;
2898	eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
2899	if (eoff > bp->b_offset + bp->b_bcount)
2900		eoff = bp->b_offset + bp->b_bcount;
2901
2902	/*
2903	 * Set valid range.  This is typically the entire buffer and thus the
2904	 * entire page.
2905	 */
2906	if (eoff > soff) {
2907		vm_page_set_validclean(
2908		    m,
2909		   (vm_offset_t) (soff & PAGE_MASK),
2910		   (vm_offset_t) (eoff - soff)
2911		);
2912	}
2913}
2914
2915/*
2916 * This routine is called before a device strategy routine.
2917 * It is used to tell the VM system that paging I/O is in
2918 * progress, and treat the pages associated with the buffer
2919 * almost as being PG_BUSY.  Also the object paging_in_progress
2920 * flag is handled to make sure that the object doesn't become
2921 * inconsistant.
2922 *
2923 * Since I/O has not been initiated yet, certain buffer flags
2924 * such as BIO_ERROR or B_INVAL may be in an inconsistant state
2925 * and should be ignored.
2926 */
2927void
2928vfs_busy_pages(struct buf * bp, int clear_modify)
2929{
2930	int i, bogus;
2931
2932	bp->b_runningbufspace = bp->b_bufsize;
2933	runningbufspace += bp->b_runningbufspace;
2934
2935	if (bp->b_flags & B_VMIO) {
2936		struct vnode *vp = bp->b_vp;
2937		vm_object_t obj;
2938		vm_ooffset_t foff;
2939
2940		VOP_GETVOBJECT(vp, &obj);
2941		foff = bp->b_offset;
2942		KASSERT(bp->b_offset != NOOFFSET,
2943		    ("vfs_busy_pages: no buffer offset"));
2944		vfs_setdirty(bp);
2945
2946retry:
2947		for (i = 0; i < bp->b_npages; i++) {
2948			vm_page_t m = bp->b_pages[i];
2949			if (vm_page_sleep_busy(m, FALSE, "vbpage"))
2950				goto retry;
2951		}
2952
2953		bogus = 0;
2954		for (i = 0; i < bp->b_npages; i++) {
2955			vm_page_t m = bp->b_pages[i];
2956
2957			vm_page_flag_clear(m, PG_ZERO);
2958			if ((bp->b_flags & B_CLUSTER) == 0) {
2959				vm_object_pip_add(obj, 1);
2960				vm_page_io_start(m);
2961			}
2962
2963			/*
2964			 * When readying a buffer for a read ( i.e
2965			 * clear_modify == 0 ), it is important to do
2966			 * bogus_page replacement for valid pages in
2967			 * partially instantiated buffers.  Partially
2968			 * instantiated buffers can, in turn, occur when
2969			 * reconstituting a buffer from its VM backing store
2970			 * base.  We only have to do this if B_CACHE is
2971			 * clear ( which causes the I/O to occur in the
2972			 * first place ).  The replacement prevents the read
2973			 * I/O from overwriting potentially dirty VM-backed
2974			 * pages.  XXX bogus page replacement is, uh, bogus.
2975			 * It may not work properly with small-block devices.
2976			 * We need to find a better way.
2977			 */
2978
2979			vm_page_protect(m, VM_PROT_NONE);
2980			if (clear_modify)
2981				vfs_page_set_valid(bp, foff, i, m);
2982			else if (m->valid == VM_PAGE_BITS_ALL &&
2983				(bp->b_flags & B_CACHE) == 0) {
2984				bp->b_pages[i] = bogus_page;
2985				bogus++;
2986			}
2987			foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
2988		}
2989		if (bogus)
2990			pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2991	}
2992}
2993
2994/*
2995 * Tell the VM system that the pages associated with this buffer
2996 * are clean.  This is used for delayed writes where the data is
2997 * going to go to disk eventually without additional VM intevention.
2998 *
2999 * Note that while we only really need to clean through to b_bcount, we
3000 * just go ahead and clean through to b_bufsize.
3001 */
3002static void
3003vfs_clean_pages(struct buf * bp)
3004{
3005	int i;
3006
3007	if (bp->b_flags & B_VMIO) {
3008		vm_ooffset_t foff;
3009
3010		foff = bp->b_offset;
3011		KASSERT(bp->b_offset != NOOFFSET,
3012		    ("vfs_clean_pages: no buffer offset"));
3013		for (i = 0; i < bp->b_npages; i++) {
3014			vm_page_t m = bp->b_pages[i];
3015			vm_ooffset_t noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3016			vm_ooffset_t eoff = noff;
3017
3018			if (eoff > bp->b_offset + bp->b_bufsize)
3019				eoff = bp->b_offset + bp->b_bufsize;
3020			vfs_page_set_valid(bp, foff, i, m);
3021			/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
3022			foff = noff;
3023		}
3024	}
3025}
3026
3027/*
3028 *	vfs_bio_set_validclean:
3029 *
3030 *	Set the range within the buffer to valid and clean.  The range is
3031 *	relative to the beginning of the buffer, b_offset.  Note that b_offset
3032 *	itself may be offset from the beginning of the first page.
3033 */
3034
3035void
3036vfs_bio_set_validclean(struct buf *bp, int base, int size)
3037{
3038	if (bp->b_flags & B_VMIO) {
3039		int i;
3040		int n;
3041
3042		/*
3043		 * Fixup base to be relative to beginning of first page.
3044		 * Set initial n to be the maximum number of bytes in the
3045		 * first page that can be validated.
3046		 */
3047
3048		base += (bp->b_offset & PAGE_MASK);
3049		n = PAGE_SIZE - (base & PAGE_MASK);
3050
3051		for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
3052			vm_page_t m = bp->b_pages[i];
3053
3054			if (n > size)
3055				n = size;
3056
3057			vm_page_set_validclean(m, base & PAGE_MASK, n);
3058			base += n;
3059			size -= n;
3060			n = PAGE_SIZE;
3061		}
3062	}
3063}
3064
3065/*
3066 *	vfs_bio_clrbuf:
3067 *
3068 *	clear a buffer.  This routine essentially fakes an I/O, so we need
3069 *	to clear BIO_ERROR and B_INVAL.
3070 *
3071 *	Note that while we only theoretically need to clear through b_bcount,
3072 *	we go ahead and clear through b_bufsize.
3073 */
3074
3075void
3076vfs_bio_clrbuf(struct buf *bp) {
3077	int i, mask = 0;
3078	caddr_t sa, ea;
3079	if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) {
3080		bp->b_flags &= ~B_INVAL;
3081		bp->b_ioflags &= ~BIO_ERROR;
3082		if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
3083		    (bp->b_offset & PAGE_MASK) == 0) {
3084			mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
3085			if (((bp->b_pages[0]->flags & PG_ZERO) == 0) &&
3086			    ((bp->b_pages[0]->valid & mask) != mask)) {
3087				bzero(bp->b_data, bp->b_bufsize);
3088			}
3089			bp->b_pages[0]->valid |= mask;
3090			bp->b_resid = 0;
3091			return;
3092		}
3093		ea = sa = bp->b_data;
3094		for(i=0;i<bp->b_npages;i++,sa=ea) {
3095			int j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
3096			ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE);
3097			ea = (caddr_t)(vm_offset_t)ulmin(
3098			    (u_long)(vm_offset_t)ea,
3099			    (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize);
3100			mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
3101			if ((bp->b_pages[i]->valid & mask) == mask)
3102				continue;
3103			if ((bp->b_pages[i]->valid & mask) == 0) {
3104				if ((bp->b_pages[i]->flags & PG_ZERO) == 0) {
3105					bzero(sa, ea - sa);
3106				}
3107			} else {
3108				for (; sa < ea; sa += DEV_BSIZE, j++) {
3109					if (((bp->b_pages[i]->flags & PG_ZERO) == 0) &&
3110						(bp->b_pages[i]->valid & (1<<j)) == 0)
3111						bzero(sa, DEV_BSIZE);
3112				}
3113			}
3114			bp->b_pages[i]->valid |= mask;
3115			vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
3116		}
3117		bp->b_resid = 0;
3118	} else {
3119		clrbuf(bp);
3120	}
3121}
3122
3123/*
3124 * vm_hold_load_pages and vm_hold_unload pages get pages into
3125 * a buffers address space.  The pages are anonymous and are
3126 * not associated with a file object.
3127 */
3128void
3129vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3130{
3131	vm_offset_t pg;
3132	vm_page_t p;
3133	int index;
3134
3135	to = round_page(to);
3136	from = round_page(from);
3137	index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3138
3139	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3140
3141tryagain:
3142
3143		/*
3144		 * note: must allocate system pages since blocking here
3145		 * could intefere with paging I/O, no matter which
3146		 * process we are.
3147		 */
3148		p = vm_page_alloc(kernel_object,
3149			((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
3150		    VM_ALLOC_SYSTEM);
3151		if (!p) {
3152			vm_pageout_deficit += (to - from) >> PAGE_SHIFT;
3153			VM_WAIT;
3154			goto tryagain;
3155		}
3156		vm_page_wire(p);
3157		p->valid = VM_PAGE_BITS_ALL;
3158		vm_page_flag_clear(p, PG_ZERO);
3159		pmap_kenter(pg, VM_PAGE_TO_PHYS(p));
3160		bp->b_pages[index] = p;
3161		vm_page_wakeup(p);
3162	}
3163	bp->b_npages = index;
3164}
3165
3166void
3167vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3168{
3169	vm_offset_t pg;
3170	vm_page_t p;
3171	int index, newnpages;
3172
3173	from = round_page(from);
3174	to = round_page(to);
3175	newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3176
3177	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3178		p = bp->b_pages[index];
3179		if (p && (index < bp->b_npages)) {
3180			if (p->busy) {
3181				printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3182					bp->b_blkno, bp->b_lblkno);
3183			}
3184			bp->b_pages[index] = NULL;
3185			pmap_kremove(pg);
3186			vm_page_busy(p);
3187			vm_page_unwire(p, 0);
3188			vm_page_free(p);
3189		}
3190	}
3191	bp->b_npages = newnpages;
3192}
3193
3194
3195#include "opt_ddb.h"
3196#ifdef DDB
3197#include <ddb/ddb.h>
3198
3199DB_SHOW_COMMAND(buffer, db_show_buffer)
3200{
3201	/* get args */
3202	struct buf *bp = (struct buf *)addr;
3203
3204	if (!have_addr) {
3205		db_printf("usage: show buffer <addr>\n");
3206		return;
3207	}
3208
3209	db_printf("b_flags = 0x%b\n", (u_int)bp->b_flags, PRINT_BUF_FLAGS);
3210	db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3211		  "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3212		  "b_blkno = %d, b_pblkno = %d\n",
3213		  bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
3214		  major(bp->b_dev), minor(bp->b_dev),
3215		  bp->b_data, bp->b_blkno, bp->b_pblkno);
3216	if (bp->b_npages) {
3217		int i;
3218		db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
3219		for (i = 0; i < bp->b_npages; i++) {
3220			vm_page_t m;
3221			m = bp->b_pages[i];
3222			db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
3223			    (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
3224			if ((i + 1) < bp->b_npages)
3225				db_printf(",");
3226		}
3227		db_printf("\n");
3228	}
3229}
3230#endif /* DDB */
3231