32#include <sys/buf.h>
33#include <sys/eventhandler.h>
34#include <sys/lock.h>
35#include <sys/malloc.h>
36#include <sys/mount.h>
37#include <sys/kernel.h>
38#include <sys/kthread.h>
39#include <sys/proc.h>
40#include <sys/reboot.h>
41#include <sys/resourcevar.h>
42#include <sys/sysctl.h>
43#include <sys/vmmeter.h>
44#include <sys/vnode.h>
45#include <vm/vm.h>
46#include <vm/vm_param.h>
47#include <vm/vm_kern.h>
48#include <vm/vm_pageout.h>
49#include <vm/vm_page.h>
50#include <vm/vm_object.h>
51#include <vm/vm_extern.h>
52#include <vm/vm_map.h>
53
54static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
55
56struct bio_ops bioops; /* I/O operation notification */
57
58struct buf *buf; /* buffer header pool */
59struct swqueue bswlist;
60struct simplelock buftimelock; /* Interlock on setting prio and timo */
61
62static void vm_hold_free_pages(struct buf * bp, vm_offset_t from,
63 vm_offset_t to);
64static void vm_hold_load_pages(struct buf * bp, vm_offset_t from,
65 vm_offset_t to);
66static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off,
67 int pageno, vm_page_t m);
68static void vfs_clean_pages(struct buf * bp);
69static void vfs_setdirty(struct buf *bp);
70static void vfs_vmio_release(struct buf *bp);
71static void vfs_backgroundwritedone(struct buf *bp);
72static int flushbufqueues(void);
73
74static int bd_request;
75
76static void buf_daemon __P((void));
77/*
78 * bogus page -- for I/O to/from partially complete buffers
79 * this is a temporary solution to the problem, but it is not
80 * really that bad. it would be better to split the buffer
81 * for input in the case of buffers partially already in memory,
82 * but the code is intricate enough already.
83 */
84vm_page_t bogus_page;
85int runningbufspace;
86int vmiodirenable = FALSE;
87static vm_offset_t bogus_offset;
88
89static int bufspace, maxbufspace,
90 bufmallocspace, maxbufmallocspace, lobufspace, hibufspace;
91static int bufreusecnt, bufdefragcnt, buffreekvacnt;
92static int maxbdrun;
93static int needsbuffer;
94static int numdirtybuffers, hidirtybuffers;
95static int numfreebuffers, lofreebuffers, hifreebuffers;
96static int getnewbufcalls;
97static int getnewbufrestarts;
98
99SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD,
100 &numdirtybuffers, 0, "");
101SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW,
102 &hidirtybuffers, 0, "");
103SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD,
104 &numfreebuffers, 0, "");
105SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW,
106 &lofreebuffers, 0, "");
107SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW,
108 &hifreebuffers, 0, "");
109SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD,
110 &runningbufspace, 0, "");
111SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD,
112 &maxbufspace, 0, "");
113SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD,
114 &hibufspace, 0, "");
115SYSCTL_INT(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD,
116 &lobufspace, 0, "");
117SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD,
118 &bufspace, 0, "");
119SYSCTL_INT(_vfs, OID_AUTO, maxbdrun, CTLFLAG_RW,
120 &maxbdrun, 0, "");
121SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW,
122 &maxbufmallocspace, 0, "");
123SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD,
124 &bufmallocspace, 0, "");
125SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW,
126 &getnewbufcalls, 0, "");
127SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW,
128 &getnewbufrestarts, 0, "");
129SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW,
130 &vmiodirenable, 0, "");
131SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW,
132 &bufdefragcnt, 0, "");
133SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW,
134 &buffreekvacnt, 0, "");
135SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW,
136 &bufreusecnt, 0, "");
137
138static int bufhashmask;
139static LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
140struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } };
141char *buf_wmesg = BUF_WMESG;
142
143extern int vm_swap_size;
144
145#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
146#define VFS_BIO_NEED_DIRTYFLUSH 0x02 /* waiting for dirty buffer flush */
147#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
148#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
149
150/*
151 * Buffer hash table code. Note that the logical block scans linearly, which
152 * gives us some L1 cache locality.
153 */
154
155static __inline
156struct bufhashhdr *
157bufhash(struct vnode *vnp, daddr_t bn)
158{
159 return(&bufhashtbl[(((uintptr_t)(vnp) >> 7) + (int)bn) & bufhashmask]);
160}
161
162/*
163 * numdirtywakeup:
164 *
165 * If someone is blocked due to there being too many dirty buffers,
166 * and numdirtybuffers is now reasonable, wake them up.
167 */
168
169static __inline void
170numdirtywakeup(void)
171{
172 if (numdirtybuffers < hidirtybuffers) {
173 if (needsbuffer & VFS_BIO_NEED_DIRTYFLUSH) {
174 needsbuffer &= ~VFS_BIO_NEED_DIRTYFLUSH;
175 wakeup(&needsbuffer);
176 }
177 }
178}
179
180/*
181 * bufspacewakeup:
182 *
183 * Called when buffer space is potentially available for recovery.
184 * getnewbuf() will block on this flag when it is unable to free
185 * sufficient buffer space. Buffer space becomes recoverable when
186 * bp's get placed back in the queues.
187 */
188
189static __inline void
190bufspacewakeup(void)
191{
192 /*
193 * If someone is waiting for BUF space, wake them up. Even
194 * though we haven't freed the kva space yet, the waiting
195 * process will be able to now.
196 */
197 if (needsbuffer & VFS_BIO_NEED_BUFSPACE) {
198 needsbuffer &= ~VFS_BIO_NEED_BUFSPACE;
199 wakeup(&needsbuffer);
200 }
201}
202
203/*
204 * bufcountwakeup:
205 *
206 * Called when a buffer has been added to one of the free queues to
207 * account for the buffer and to wakeup anyone waiting for free buffers.
208 * This typically occurs when large amounts of metadata are being handled
209 * by the buffer cache ( else buffer space runs out first, usually ).
210 */
211
212static __inline void
213bufcountwakeup(void)
214{
215 ++numfreebuffers;
216 if (needsbuffer) {
217 needsbuffer &= ~VFS_BIO_NEED_ANY;
218 if (numfreebuffers >= hifreebuffers)
219 needsbuffer &= ~VFS_BIO_NEED_FREE;
220 wakeup(&needsbuffer);
221 }
222}
223
224/*
225 * vfs_buf_test_cache:
226 *
227 * Called when a buffer is extended. This function clears the B_CACHE
228 * bit if the newly extended portion of the buffer does not contain
229 * valid data.
230 */
231static __inline__
232void
233vfs_buf_test_cache(struct buf *bp,
234 vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
235 vm_page_t m)
236{
237 if (bp->b_flags & B_CACHE) {
238 int base = (foff + off) & PAGE_MASK;
239 if (vm_page_is_valid(m, base, size) == 0)
240 bp->b_flags &= ~B_CACHE;
241 }
242}
243
244static __inline__
245void
246bd_wakeup(int dirtybuflevel)
247{
248 if (numdirtybuffers >= dirtybuflevel && bd_request == 0) {
249 bd_request = 1;
250 wakeup(&bd_request);
251 }
252}
253
254/*
255 * bd_speedup - speedup the buffer cache flushing code
256 */
257
258static __inline__
259void
260bd_speedup(void)
261{
262 bd_wakeup(1);
263}
264
265/*
266 * Initialize buffer headers and related structures.
267 */
268
269caddr_t
270bufhashinit(caddr_t vaddr)
271{
272 /* first, make a null hash table */
273 for (bufhashmask = 8; bufhashmask < nbuf / 4; bufhashmask <<= 1)
274 ;
275 bufhashtbl = (void *)vaddr;
276 vaddr = vaddr + sizeof(*bufhashtbl) * bufhashmask;
277 --bufhashmask;
278 return(vaddr);
279}
280
281void
282bufinit(void)
283{
284 struct buf *bp;
285 int i;
286
287 TAILQ_INIT(&bswlist);
288 LIST_INIT(&invalhash);
289 simple_lock_init(&buftimelock);
290
291 for (i = 0; i <= bufhashmask; i++)
292 LIST_INIT(&bufhashtbl[i]);
293
294 /* next, make a null set of free lists */
295 for (i = 0; i < BUFFER_QUEUES; i++)
296 TAILQ_INIT(&bufqueues[i]);
297
298 /* finally, initialize each buffer header and stick on empty q */
299 for (i = 0; i < nbuf; i++) {
300 bp = &buf[i];
301 bzero(bp, sizeof *bp);
302 bp->b_flags = B_INVAL; /* we're just an empty header */
303 bp->b_dev = NODEV;
304 bp->b_rcred = NOCRED;
305 bp->b_wcred = NOCRED;
306 bp->b_qindex = QUEUE_EMPTY;
307 bp->b_xflags = 0;
308 LIST_INIT(&bp->b_dep);
309 BUF_LOCKINIT(bp);
310 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
311 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
312 }
313
314 /*
315 * maxbufspace is the absolute maximum amount of buffer space we are
316 * allowed to reserve in KVM and in real terms. The absolute maximum
317 * is nominally used by buf_daemon. hibufspace is the nominal maximum
318 * used by most other processes. The differential is required to
319 * ensure that buf_daemon is able to run when other processes might
320 * be blocked waiting for buffer space.
321 *
322 * maxbufspace is based on BKVASIZE. Allocating buffers larger then
323 * this may result in KVM fragmentation which is not handled optimally
324 * by the system.
325 */
326 maxbufspace = nbuf * BKVASIZE;
327 hibufspace = imax(3 * maxbufspace / 4, maxbufspace - MAXBSIZE * 10);
328 lobufspace = hibufspace - MAXBSIZE;
329
330/*
331 * Limit the amount of malloc memory since it is wired permanently into
332 * the kernel space. Even though this is accounted for in the buffer
333 * allocation, we don't want the malloced region to grow uncontrolled.
334 * The malloc scheme improves memory utilization significantly on average
335 * (small) directories.
336 */
337 maxbufmallocspace = hibufspace / 20;
338
339/*
340 * Reduce the chance of a deadlock occuring by limiting the number
341 * of delayed-write dirty buffers we allow to stack up.
342 */
343 hidirtybuffers = nbuf / 4 + 20;
344 numdirtybuffers = 0;
345/*
346 * To support extreme low-memory systems, make sure hidirtybuffers cannot
347 * eat up all available buffer space. This occurs when our minimum cannot
348 * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
349 * BKVASIZE'd (8K) buffers.
350 */
351 while (hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
352 hidirtybuffers >>= 1;
353 }
354
355/*
356 * Try to keep the number of free buffers in the specified range,
357 * and give special processes (e.g. like buf_daemon) access to an
358 * emergency reserve.
359 */
360 lofreebuffers = nbuf / 18 + 5;
361 hifreebuffers = 2 * lofreebuffers;
362 numfreebuffers = nbuf;
363
364/*
365 * Maximum number of async ops initiated per buf_daemon loop. This is
366 * somewhat of a hack at the moment, we really need to limit ourselves
367 * based on the number of bytes of I/O in-transit that were initiated
368 * from buf_daemon.
369 */
370 if ((maxbdrun = nswbuf / 4) < 4)
371 maxbdrun = 4;
372
373 bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
374 bogus_page = vm_page_alloc(kernel_object,
375 ((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
376 VM_ALLOC_NORMAL);
377 cnt.v_wire_count++;
378
379}
380
381/*
382 * bfreekva() - free the kva allocation for a buffer.
383 *
384 * Must be called at splbio() or higher as this is the only locking for
385 * buffer_map.
386 *
387 * Since this call frees up buffer space, we call bufspacewakeup().
388 */
389static void
390bfreekva(struct buf * bp)
391{
392 if (bp->b_kvasize) {
393 ++buffreekvacnt;
394 bufspace -= bp->b_kvasize;
395 vm_map_delete(buffer_map,
396 (vm_offset_t) bp->b_kvabase,
397 (vm_offset_t) bp->b_kvabase + bp->b_kvasize
398 );
399 bp->b_kvasize = 0;
400 bufspacewakeup();
401 }
402}
403
404/*
405 * bremfree:
406 *
407 * Remove the buffer from the appropriate free list.
408 */
409void
410bremfree(struct buf * bp)
411{
412 int s = splbio();
413 int old_qindex = bp->b_qindex;
414
415 if (bp->b_qindex != QUEUE_NONE) {
416 KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
417 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
418 bp->b_qindex = QUEUE_NONE;
419 runningbufspace += bp->b_bufsize;
420 } else {
421 if (BUF_REFCNT(bp) <= 1)
422 panic("bremfree: removing a buffer not on a queue");
423 }
424
425 /*
426 * Fixup numfreebuffers count. If the buffer is invalid or not
427 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
428 * the buffer was free and we must decrement numfreebuffers.
429 */
430 if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
431 switch(old_qindex) {
432 case QUEUE_DIRTY:
433 case QUEUE_CLEAN:
434 case QUEUE_EMPTY:
435 case QUEUE_EMPTYKVA:
436 --numfreebuffers;
437 break;
438 default:
439 break;
440 }
441 }
442 splx(s);
443}
444
445
446/*
447 * Get a buffer with the specified data. Look in the cache first. We
448 * must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
449 * is set, the buffer is valid and we do not have to do anything ( see
450 * getblk() ).
451 */
452int
453bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred,
454 struct buf ** bpp)
455{
456 struct buf *bp;
457
458 bp = getblk(vp, blkno, size, 0, 0);
459 *bpp = bp;
460
461 /* if not found in cache, do some I/O */
462 if ((bp->b_flags & B_CACHE) == 0) {
463 if (curproc != NULL)
464 curproc->p_stats->p_ru.ru_inblock++;
465 KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp));
466 bp->b_iocmd = BIO_READ;
467 bp->b_flags &= ~B_INVAL;
468 bp->b_ioflags &= ~BIO_ERROR;
469 if (bp->b_rcred == NOCRED) {
470 if (cred != NOCRED)
471 crhold(cred);
472 bp->b_rcred = cred;
473 }
474 vfs_busy_pages(bp, 0);
475 VOP_STRATEGY(vp, bp);
476 return (bufwait(bp));
477 }
478 return (0);
479}
480
481/*
482 * Operates like bread, but also starts asynchronous I/O on
483 * read-ahead blocks. We must clear BIO_ERROR and B_INVAL prior
484 * to initiating I/O . If B_CACHE is set, the buffer is valid
485 * and we do not have to do anything.
486 */
487int
488breadn(struct vnode * vp, daddr_t blkno, int size,
489 daddr_t * rablkno, int *rabsize,
490 int cnt, struct ucred * cred, struct buf ** bpp)
491{
492 struct buf *bp, *rabp;
493 int i;
494 int rv = 0, readwait = 0;
495
496 *bpp = bp = getblk(vp, blkno, size, 0, 0);
497
498 /* if not found in cache, do some I/O */
499 if ((bp->b_flags & B_CACHE) == 0) {
500 if (curproc != NULL)
501 curproc->p_stats->p_ru.ru_inblock++;
502 bp->b_iocmd = BIO_READ;
503 bp->b_flags &= ~B_INVAL;
504 bp->b_ioflags &= ~BIO_ERROR;
505 if (bp->b_rcred == NOCRED) {
506 if (cred != NOCRED)
507 crhold(cred);
508 bp->b_rcred = cred;
509 }
510 vfs_busy_pages(bp, 0);
511 VOP_STRATEGY(vp, bp);
512 ++readwait;
513 }
514
515 for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
516 if (inmem(vp, *rablkno))
517 continue;
518 rabp = getblk(vp, *rablkno, *rabsize, 0, 0);
519
520 if ((rabp->b_flags & B_CACHE) == 0) {
521 if (curproc != NULL)
522 curproc->p_stats->p_ru.ru_inblock++;
523 rabp->b_flags |= B_ASYNC;
524 rabp->b_flags &= ~B_INVAL;
525 rabp->b_ioflags &= ~BIO_ERROR;
526 rabp->b_iocmd = BIO_READ;
527 if (rabp->b_rcred == NOCRED) {
528 if (cred != NOCRED)
529 crhold(cred);
530 rabp->b_rcred = cred;
531 }
532 vfs_busy_pages(rabp, 0);
533 BUF_KERNPROC(rabp);
534 VOP_STRATEGY(vp, rabp);
535 } else {
536 brelse(rabp);
537 }
538 }
539
540 if (readwait) {
541 rv = bufwait(bp);
542 }
543 return (rv);
544}
545
546/*
547 * Write, release buffer on completion. (Done by iodone
548 * if async). Do not bother writing anything if the buffer
549 * is invalid.
550 *
551 * Note that we set B_CACHE here, indicating that buffer is
552 * fully valid and thus cacheable. This is true even of NFS
553 * now so we set it generally. This could be set either here
554 * or in biodone() since the I/O is synchronous. We put it
555 * here.
556 */
557int
558bwrite(struct buf * bp)
559{
560 int oldflags, s;
561 struct buf *newbp;
562
563 if (bp->b_flags & B_INVAL) {
564 brelse(bp);
565 return (0);
566 }
567
568 oldflags = bp->b_flags;
569
570 if (BUF_REFCNT(bp) == 0)
571 panic("bwrite: buffer is not busy???");
572 s = splbio();
573 /*
574 * If a background write is already in progress, delay
575 * writing this block if it is asynchronous. Otherwise
576 * wait for the background write to complete.
577 */
578 if (bp->b_xflags & BX_BKGRDINPROG) {
579 if (bp->b_flags & B_ASYNC) {
580 splx(s);
581 bdwrite(bp);
582 return (0);
583 }
584 bp->b_xflags |= BX_BKGRDWAIT;
585 tsleep(&bp->b_xflags, PRIBIO, "biord", 0);
586 if (bp->b_xflags & BX_BKGRDINPROG)
587 panic("bwrite: still writing");
588 }
589
590 /* Mark the buffer clean */
591 bundirty(bp);
592
593 /*
594 * If this buffer is marked for background writing and we
595 * do not have to wait for it, make a copy and write the
596 * copy so as to leave this buffer ready for further use.
597 */
598 if ((bp->b_xflags & BX_BKGRDWRITE) && (bp->b_flags & B_ASYNC)) {
599 if (bp->b_iodone != NULL) {
600 printf("bp->b_iodone = %p\n", bp->b_iodone);
601 panic("bwrite: need chained iodone");
602 }
603
604 /* get a new block */
605 newbp = geteblk(bp->b_bufsize);
606
607 /* set it to be identical to the old block */
608 memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
609 bgetvp(bp->b_vp, newbp);
610 newbp->b_lblkno = bp->b_lblkno;
611 newbp->b_blkno = bp->b_blkno;
612 newbp->b_offset = bp->b_offset;
613 newbp->b_iodone = vfs_backgroundwritedone;
614 newbp->b_flags |= B_ASYNC;
615 newbp->b_flags &= ~B_INVAL;
616
617 /* move over the dependencies */
618 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
619 (*bioops.io_movedeps)(bp, newbp);
620
621 /*
622 * Initiate write on the copy, release the original to
623 * the B_LOCKED queue so that it cannot go away until
624 * the background write completes. If not locked it could go
625 * away and then be reconstituted while it was being written.
626 * If the reconstituted buffer were written, we could end up
627 * with two background copies being written at the same time.
628 */
629 bp->b_xflags |= BX_BKGRDINPROG;
630 bp->b_flags |= B_LOCKED;
631 bqrelse(bp);
632 bp = newbp;
633 }
634
635 bp->b_flags &= ~B_DONE;
636 bp->b_ioflags &= ~BIO_ERROR;
637 bp->b_flags |= B_WRITEINPROG | B_CACHE;
638 bp->b_iocmd = BIO_WRITE;
639
640 bp->b_vp->v_numoutput++;
641 vfs_busy_pages(bp, 1);
642 if (curproc != NULL)
643 curproc->p_stats->p_ru.ru_oublock++;
644 splx(s);
645 if (oldflags & B_ASYNC)
646 BUF_KERNPROC(bp);
647 BUF_STRATEGY(bp);
648
649 if ((oldflags & B_ASYNC) == 0) {
650 int rtval = bufwait(bp);
651 brelse(bp);
652 return (rtval);
653 }
654
655 return (0);
656}
657
658/*
659 * Complete a background write started from bwrite.
660 */
661static void
662vfs_backgroundwritedone(bp)
663 struct buf *bp;
664{
665 struct buf *origbp;
666
667 /*
668 * Find the original buffer that we are writing.
669 */
670 if ((origbp = gbincore(bp->b_vp, bp->b_lblkno)) == NULL)
671 panic("backgroundwritedone: lost buffer");
672 /*
673 * Process dependencies then return any unfinished ones.
674 */
675 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
676 (*bioops.io_complete)(bp);
677 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
678 (*bioops.io_movedeps)(bp, origbp);
679 /*
680 * Clear the BX_BKGRDINPROG flag in the original buffer
681 * and awaken it if it is waiting for the write to complete.
682 */
683 origbp->b_xflags &= ~BX_BKGRDINPROG;
684 if (origbp->b_xflags & BX_BKGRDWAIT) {
685 origbp->b_xflags &= ~BX_BKGRDWAIT;
686 wakeup(&origbp->b_xflags);
687 }
688 /*
689 * Clear the B_LOCKED flag and remove it from the locked
690 * queue if it currently resides there.
691 */
692 origbp->b_flags &= ~B_LOCKED;
693 if (BUF_LOCK(origbp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
694 bremfree(origbp);
695 bqrelse(origbp);
696 }
697 /*
698 * This buffer is marked B_NOCACHE, so when it is released
699 * by biodone, it will be tossed. We mark it with BIO_READ
700 * to avoid biodone doing a second vwakeup.
701 */
702 bp->b_flags |= B_NOCACHE;
703 bp->b_iocmd = BIO_READ;
704 bp->b_flags &= ~(B_CACHE | B_DONE);
705 bp->b_iodone = 0;
706 bufdone(bp);
707}
708
709/*
710 * Delayed write. (Buffer is marked dirty). Do not bother writing
711 * anything if the buffer is marked invalid.
712 *
713 * Note that since the buffer must be completely valid, we can safely
714 * set B_CACHE. In fact, we have to set B_CACHE here rather then in
715 * biodone() in order to prevent getblk from writing the buffer
716 * out synchronously.
717 */
718void
719bdwrite(struct buf * bp)
720{
721 if (BUF_REFCNT(bp) == 0)
722 panic("bdwrite: buffer is not busy");
723
724 if (bp->b_flags & B_INVAL) {
725 brelse(bp);
726 return;
727 }
728 bdirty(bp);
729
730 /*
731 * Set B_CACHE, indicating that the buffer is fully valid. This is
732 * true even of NFS now.
733 */
734 bp->b_flags |= B_CACHE;
735
736 /*
737 * This bmap keeps the system from needing to do the bmap later,
738 * perhaps when the system is attempting to do a sync. Since it
739 * is likely that the indirect block -- or whatever other datastructure
740 * that the filesystem needs is still in memory now, it is a good
741 * thing to do this. Note also, that if the pageout daemon is
742 * requesting a sync -- there might not be enough memory to do
743 * the bmap then... So, this is important to do.
744 */
745 if (bp->b_lblkno == bp->b_blkno) {
746 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
747 }
748
749 /*
750 * Set the *dirty* buffer range based upon the VM system dirty pages.
751 */
752 vfs_setdirty(bp);
753
754 /*
755 * We need to do this here to satisfy the vnode_pager and the
756 * pageout daemon, so that it thinks that the pages have been
757 * "cleaned". Note that since the pages are in a delayed write
758 * buffer -- the VFS layer "will" see that the pages get written
759 * out on the next sync, or perhaps the cluster will be completed.
760 */
761 vfs_clean_pages(bp);
762 bqrelse(bp);
763
764 /*
765 * Wakeup the buffer flushing daemon if we have saturated the
766 * buffer cache.
767 */
768
769 bd_wakeup(hidirtybuffers);
770
771 /*
772 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
773 * due to the softdep code.
774 */
775}
776
777/*
778 * bdirty:
779 *
780 * Turn buffer into delayed write request. We must clear BIO_READ and
781 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
782 * itself to properly update it in the dirty/clean lists. We mark it
783 * B_DONE to ensure that any asynchronization of the buffer properly
784 * clears B_DONE ( else a panic will occur later ).
785 *
786 * bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
787 * might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
788 * should only be called if the buffer is known-good.
789 *
790 * Since the buffer is not on a queue, we do not update the numfreebuffers
791 * count.
792 *
793 * Must be called at splbio().
794 * The buffer must be on QUEUE_NONE.
795 */
796void
797bdirty(bp)
798 struct buf *bp;
799{
800 KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
801 bp->b_flags &= ~(B_RELBUF);
802 bp->b_iocmd = BIO_WRITE;
803
804 if ((bp->b_flags & B_DELWRI) == 0) {
805 bp->b_flags |= B_DONE | B_DELWRI;
806 reassignbuf(bp, bp->b_vp);
807 ++numdirtybuffers;
808 bd_wakeup(hidirtybuffers);
809 }
810}
811
812/*
813 * bundirty:
814 *
815 * Clear B_DELWRI for buffer.
816 *
817 * Since the buffer is not on a queue, we do not update the numfreebuffers
818 * count.
819 *
820 * Must be called at splbio().
821 * The buffer must be on QUEUE_NONE.
822 */
823
824void
825bundirty(bp)
826 struct buf *bp;
827{
828 KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
829
830 if (bp->b_flags & B_DELWRI) {
831 bp->b_flags &= ~B_DELWRI;
832 reassignbuf(bp, bp->b_vp);
833 --numdirtybuffers;
834 numdirtywakeup();
835 }
836 /*
837 * Since it is now being written, we can clear its deferred write flag.
838 */
839 bp->b_flags &= ~B_DEFERRED;
840}
841
842/*
843 * bawrite:
844 *
845 * Asynchronous write. Start output on a buffer, but do not wait for
846 * it to complete. The buffer is released when the output completes.
847 *
848 * bwrite() ( or the VOP routine anyway ) is responsible for handling
849 * B_INVAL buffers. Not us.
850 */
851void
852bawrite(struct buf * bp)
853{
854 bp->b_flags |= B_ASYNC;
855 (void) BUF_WRITE(bp);
856}
857
858/*
859 * bowrite:
860 *
861 * Ordered write. Start output on a buffer, and flag it so that the
862 * device will write it in the order it was queued. The buffer is
863 * released when the output completes. bwrite() ( or the VOP routine
864 * anyway ) is responsible for handling B_INVAL buffers.
865 */
866int
867bowrite(struct buf * bp)
868{
869 bp->b_ioflags |= BIO_ORDERED;
870 bp->b_flags |= B_ASYNC;
871 return (BUF_WRITE(bp));
872}
873
874/*
875 * bwillwrite:
876 *
877 * Called prior to the locking of any vnodes when we are expecting to
878 * write. We do not want to starve the buffer cache with too many
879 * dirty buffers so we block here. By blocking prior to the locking
880 * of any vnodes we attempt to avoid the situation where a locked vnode
881 * prevents the various system daemons from flushing related buffers.
882 */
883
884void
885bwillwrite(void)
886{
887 int slop = hidirtybuffers / 10;
888
889 if (numdirtybuffers > hidirtybuffers + slop) {
890 int s;
891
892 s = splbio();
893 while (numdirtybuffers > hidirtybuffers) {
894 bd_wakeup(hidirtybuffers);
895 needsbuffer |= VFS_BIO_NEED_DIRTYFLUSH;
896 tsleep(&needsbuffer, (PRIBIO + 4), "flswai", 0);
897 }
898 splx(s);
899 }
900}
901
902/*
903 * brelse:
904 *
905 * Release a busy buffer and, if requested, free its resources. The
906 * buffer will be stashed in the appropriate bufqueue[] allowing it
907 * to be accessed later as a cache entity or reused for other purposes.
908 */
909void
910brelse(struct buf * bp)
911{
912 int s;
913
914 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
915
916 s = splbio();
917
918 if (bp->b_flags & B_LOCKED)
919 bp->b_ioflags &= ~BIO_ERROR;
920
921 if (bp->b_iocmd == BIO_WRITE &&
922 (bp->b_ioflags & BIO_ERROR) &&
923 !(bp->b_flags & B_INVAL)) {
924 /*
925 * Failed write, redirty. Must clear BIO_ERROR to prevent
926 * pages from being scrapped. If B_INVAL is set then
927 * this case is not run and the next case is run to
928 * destroy the buffer. B_INVAL can occur if the buffer
929 * is outside the range supported by the underlying device.
930 */
931 bp->b_ioflags &= ~BIO_ERROR;
932 bdirty(bp);
933 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
934 (bp->b_ioflags & BIO_ERROR) ||
935 bp->b_iocmd == BIO_DELETE || (bp->b_bufsize <= 0)) {
936 /*
937 * Either a failed I/O or we were asked to free or not
938 * cache the buffer.
939 */
940 bp->b_flags |= B_INVAL;
941 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
942 (*bioops.io_deallocate)(bp);
943 if (bp->b_flags & B_DELWRI) {
944 --numdirtybuffers;
945 numdirtywakeup();
946 }
947 bp->b_flags &= ~(B_DELWRI | B_CACHE);
948 if ((bp->b_flags & B_VMIO) == 0) {
949 if (bp->b_bufsize)
950 allocbuf(bp, 0);
951 if (bp->b_vp)
952 brelvp(bp);
953 }
954 }
955
956 /*
957 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release()
958 * is called with B_DELWRI set, the underlying pages may wind up
959 * getting freed causing a previous write (bdwrite()) to get 'lost'
960 * because pages associated with a B_DELWRI bp are marked clean.
961 *
962 * We still allow the B_INVAL case to call vfs_vmio_release(), even
963 * if B_DELWRI is set.
964 */
965
966 if (bp->b_flags & B_DELWRI)
967 bp->b_flags &= ~B_RELBUF;
968
969 /*
970 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
971 * constituted, not even NFS buffers now. Two flags effect this. If
972 * B_INVAL, the struct buf is invalidated but the VM object is kept
973 * around ( i.e. so it is trivial to reconstitute the buffer later ).
974 *
975 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
976 * invalidated. BIO_ERROR cannot be set for a failed write unless the
977 * buffer is also B_INVAL because it hits the re-dirtying code above.
978 *
979 * Normally we can do this whether a buffer is B_DELWRI or not. If
980 * the buffer is an NFS buffer, it is tracking piecemeal writes or
981 * the commit state and we cannot afford to lose the buffer. If the
982 * buffer has a background write in progress, we need to keep it
983 * around to prevent it from being reconstituted and starting a second
984 * background write.
985 */
986 if ((bp->b_flags & B_VMIO)
987 && !(bp->b_vp->v_tag == VT_NFS &&
988 !vn_isdisk(bp->b_vp, NULL) &&
989 (bp->b_flags & B_DELWRI) &&
990 (bp->b_xflags & BX_BKGRDINPROG))
991 ) {
992
993 int i, j, resid;
994 vm_page_t m;
995 off_t foff;
996 vm_pindex_t poff;
997 vm_object_t obj;
998 struct vnode *vp;
999
1000 vp = bp->b_vp;
1001
1002 /*
1003 * Get the base offset and length of the buffer. Note that
1004 * for block sizes that are less then PAGE_SIZE, the b_data
1005 * base of the buffer does not represent exactly b_offset and
1006 * neither b_offset nor b_size are necessarily page aligned.
1007 * Instead, the starting position of b_offset is:
1008 *
1009 * b_data + (b_offset & PAGE_MASK)
1010 *
1011 * block sizes less then DEV_BSIZE (usually 512) are not
1012 * supported due to the page granularity bits (m->valid,
1013 * m->dirty, etc...).
1014 *
1015 * See man buf(9) for more information
1016 */
1017
1018 resid = bp->b_bufsize;
1019 foff = bp->b_offset;
1020
1021 for (i = 0; i < bp->b_npages; i++) {
1022 m = bp->b_pages[i];
1023 vm_page_flag_clear(m, PG_ZERO);
1024 if (m == bogus_page) {
1025
1026 obj = (vm_object_t) vp->v_object;
1027 poff = OFF_TO_IDX(bp->b_offset);
1028
1029 for (j = i; j < bp->b_npages; j++) {
1030 m = bp->b_pages[j];
1031 if (m == bogus_page) {
1032 m = vm_page_lookup(obj, poff + j);
1033 if (!m) {
1034 panic("brelse: page missing\n");
1035 }
1036 bp->b_pages[j] = m;
1037 }
1038 }
1039
1040 if ((bp->b_flags & B_INVAL) == 0) {
1041 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
1042 }
1043 }
1044 if ((bp->b_flags & B_NOCACHE) || (bp->b_ioflags & BIO_ERROR)) {
1045 int poffset = foff & PAGE_MASK;
1046 int presid = resid > (PAGE_SIZE - poffset) ?
1047 (PAGE_SIZE - poffset) : resid;
1048
1049 KASSERT(presid >= 0, ("brelse: extra page"));
1050 vm_page_set_invalid(m, poffset, presid);
1051 }
1052 resid -= PAGE_SIZE - (foff & PAGE_MASK);
1053 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
1054 }
1055
1056 if (bp->b_flags & (B_INVAL | B_RELBUF))
1057 vfs_vmio_release(bp);
1058
1059 } else if (bp->b_flags & B_VMIO) {
1060
1061 if (bp->b_flags & (B_INVAL | B_RELBUF))
1062 vfs_vmio_release(bp);
1063
1064 }
1065
1066 if (bp->b_qindex != QUEUE_NONE)
1067 panic("brelse: free buffer onto another queue???");
1068 if (BUF_REFCNT(bp) > 1) {
1069 /* Temporary panic to verify exclusive locking */
1070 /* This panic goes away when we allow shared refs */
1071 panic("brelse: multiple refs");
1072 /* do not release to free list */
1073 BUF_UNLOCK(bp);
1074 splx(s);
1075 return;
1076 }
1077
1078 /* enqueue */
1079
1080 /* buffers with no memory */
1081 if (bp->b_bufsize == 0) {
1082 bp->b_flags |= B_INVAL;
1083 bp->b_xflags &= ~BX_BKGRDWRITE;
1084 if (bp->b_xflags & BX_BKGRDINPROG)
1085 panic("losing buffer 1");
1086 if (bp->b_kvasize) {
1087 bp->b_qindex = QUEUE_EMPTYKVA;
1088 } else {
1089 bp->b_qindex = QUEUE_EMPTY;
1090 }
1091 TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1092 LIST_REMOVE(bp, b_hash);
1093 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1094 bp->b_dev = NODEV;
1095 /* buffers with junk contents */
1096 } else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) || (bp->b_ioflags & BIO_ERROR)) {
1097 bp->b_flags |= B_INVAL;
1098 bp->b_xflags &= ~BX_BKGRDWRITE;
1099 if (bp->b_xflags & BX_BKGRDINPROG)
1100 panic("losing buffer 2");
1101 bp->b_qindex = QUEUE_CLEAN;
1102 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1103 LIST_REMOVE(bp, b_hash);
1104 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1105 bp->b_dev = NODEV;
1106
1107 /* buffers that are locked */
1108 } else if (bp->b_flags & B_LOCKED) {
1109 bp->b_qindex = QUEUE_LOCKED;
1110 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1111
1112 /* remaining buffers */
1113 } else {
1114 switch(bp->b_flags & (B_DELWRI|B_AGE)) {
1115 case B_DELWRI | B_AGE:
1116 bp->b_qindex = QUEUE_DIRTY;
1117 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1118 break;
1119 case B_DELWRI:
1120 bp->b_qindex = QUEUE_DIRTY;
1121 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1122 break;
1123 case B_AGE:
1124 bp->b_qindex = QUEUE_CLEAN;
1125 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1126 break;
1127 default:
1128 bp->b_qindex = QUEUE_CLEAN;
1129 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1130 break;
1131 }
1132 }
1133
1134 /*
1135 * If B_INVAL, clear B_DELWRI. We've already placed the buffer
1136 * on the correct queue.
1137 */
1138 if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) {
1139 bp->b_flags &= ~B_DELWRI;
1140 --numdirtybuffers;
1141 numdirtywakeup();
1142 }
1143
1144 runningbufspace -= bp->b_bufsize;
1145
1146 /*
1147 * Fixup numfreebuffers count. The bp is on an appropriate queue
1148 * unless locked. We then bump numfreebuffers if it is not B_DELWRI.
1149 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1150 * if B_INVAL is set ).
1151 */
1152
1153 if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI))
1154 bufcountwakeup();
1155
1156 /*
1157 * Something we can maybe free.
1158 */
1159
1160 if (bp->b_bufsize || bp->b_kvasize)
1161 bufspacewakeup();
1162
1163 /* unlock */
1164 BUF_UNLOCK(bp);
1165 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1166 bp->b_ioflags &= ~BIO_ORDERED;
1167 splx(s);
1168}
1169
1170/*
1171 * Release a buffer back to the appropriate queue but do not try to free
1172 * it.
1173 *
1174 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1175 * biodone() to requeue an async I/O on completion. It is also used when
1176 * known good buffers need to be requeued but we think we may need the data
1177 * again soon.
1178 */
1179void
1180bqrelse(struct buf * bp)
1181{
1182 int s;
1183
1184 s = splbio();
1185
1186 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1187
1188 if (bp->b_qindex != QUEUE_NONE)
1189 panic("bqrelse: free buffer onto another queue???");
1190 if (BUF_REFCNT(bp) > 1) {
1191 /* do not release to free list */
1192 panic("bqrelse: multiple refs");
1193 BUF_UNLOCK(bp);
1194 splx(s);
1195 return;
1196 }
1197 if (bp->b_flags & B_LOCKED) {
1198 bp->b_ioflags &= ~BIO_ERROR;
1199 bp->b_qindex = QUEUE_LOCKED;
1200 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1201 /* buffers with stale but valid contents */
1202 } else if (bp->b_flags & B_DELWRI) {
1203 bp->b_qindex = QUEUE_DIRTY;
1204 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1205 } else {
1206 bp->b_qindex = QUEUE_CLEAN;
1207 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1208 }
1209
1210 runningbufspace -= bp->b_bufsize;
1211
1212 if ((bp->b_flags & B_LOCKED) == 0 &&
1213 ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))) {
1214 bufcountwakeup();
1215 }
1216
1217 /*
1218 * Something we can maybe wakeup
1219 */
1220 if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1221 bufspacewakeup();
1222
1223 /* unlock */
1224 BUF_UNLOCK(bp);
1225 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1226 bp->b_ioflags &= ~BIO_ORDERED;
1227 splx(s);
1228}
1229
1230static void
1231vfs_vmio_release(bp)
1232 struct buf *bp;
1233{
1234 int i, s;
1235 vm_page_t m;
1236
1237 s = splvm();
1238 for (i = 0; i < bp->b_npages; i++) {
1239 m = bp->b_pages[i];
1240 bp->b_pages[i] = NULL;
1241 /*
1242 * In order to keep page LRU ordering consistent, put
1243 * everything on the inactive queue.
1244 */
1245 vm_page_unwire(m, 0);
1246 /*
1247 * We don't mess with busy pages, it is
1248 * the responsibility of the process that
1249 * busied the pages to deal with them.
1250 */
1251 if ((m->flags & PG_BUSY) || (m->busy != 0))
1252 continue;
1253
1254 if (m->wire_count == 0) {
1255 vm_page_flag_clear(m, PG_ZERO);
1256 /*
1257 * Might as well free the page if we can and it has
1258 * no valid data.
1259 */
1260 if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) {
1261 vm_page_busy(m);
1262 vm_page_protect(m, VM_PROT_NONE);
1263 vm_page_free(m);
1264 }
1265 }
1266 }
1267 runningbufspace -= bp->b_bufsize;
1268 splx(s);
1269 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
1270 if (bp->b_bufsize)
1271 bufspacewakeup();
1272 bp->b_npages = 0;
1273 bp->b_bufsize = 0;
1274 bp->b_flags &= ~B_VMIO;
1275 if (bp->b_vp)
1276 brelvp(bp);
1277}
1278
1279/*
1280 * Check to see if a block is currently memory resident.
1281 */
1282struct buf *
1283gbincore(struct vnode * vp, daddr_t blkno)
1284{
1285 struct buf *bp;
1286 struct bufhashhdr *bh;
1287
1288 bh = bufhash(vp, blkno);
1289
1290 /* Search hash chain */
1291 LIST_FOREACH(bp, bh, b_hash) {
1292 /* hit */
1293 if (bp->b_vp == vp && bp->b_lblkno == blkno &&
1294 (bp->b_flags & B_INVAL) == 0) {
1295 break;
1296 }
1297 }
1298 return (bp);
1299}
1300
1301/*
1302 * vfs_bio_awrite:
1303 *
1304 * Implement clustered async writes for clearing out B_DELWRI buffers.
1305 * This is much better then the old way of writing only one buffer at
1306 * a time. Note that we may not be presented with the buffers in the
1307 * correct order, so we search for the cluster in both directions.
1308 */
1309int
1310vfs_bio_awrite(struct buf * bp)
1311{
1312 int i;
1313 int j;
1314 daddr_t lblkno = bp->b_lblkno;
1315 struct vnode *vp = bp->b_vp;
1316 int s;
1317 int ncl;
1318 struct buf *bpa;
1319 int nwritten;
1320 int size;
1321 int maxcl;
1322
1323 s = splbio();
1324 /*
1325 * right now we support clustered writing only to regular files. If
1326 * we find a clusterable block we could be in the middle of a cluster
1327 * rather then at the beginning.
1328 */
1329 if ((vp->v_type == VREG) &&
1330 (vp->v_mount != 0) && /* Only on nodes that have the size info */
1331 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
1332
1333 size = vp->v_mount->mnt_stat.f_iosize;
1334 maxcl = MAXPHYS / size;
1335
1336 for (i = 1; i < maxcl; i++) {
1337 if ((bpa = gbincore(vp, lblkno + i)) &&
1338 BUF_REFCNT(bpa) == 0 &&
1339 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1340 (B_DELWRI | B_CLUSTEROK)) &&
1341 (bpa->b_bufsize == size)) {
1342 if ((bpa->b_blkno == bpa->b_lblkno) ||
1343 (bpa->b_blkno !=
1344 bp->b_blkno + ((i * size) >> DEV_BSHIFT)))
1345 break;
1346 } else {
1347 break;
1348 }
1349 }
1350 for (j = 1; i + j <= maxcl && j <= lblkno; j++) {
1351 if ((bpa = gbincore(vp, lblkno - j)) &&
1352 BUF_REFCNT(bpa) == 0 &&
1353 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1354 (B_DELWRI | B_CLUSTEROK)) &&
1355 (bpa->b_bufsize == size)) {
1356 if ((bpa->b_blkno == bpa->b_lblkno) ||
1357 (bpa->b_blkno !=
1358 bp->b_blkno - ((j * size) >> DEV_BSHIFT)))
1359 break;
1360 } else {
1361 break;
1362 }
1363 }
1364 --j;
1365 ncl = i + j;
1366 /*
1367 * this is a possible cluster write
1368 */
1369 if (ncl != 1) {
1370 nwritten = cluster_wbuild(vp, size, lblkno - j, ncl);
1371 splx(s);
1372 return nwritten;
1373 }
1374 }
1375
1376 BUF_LOCK(bp, LK_EXCLUSIVE);
1377 bremfree(bp);
1378 bp->b_flags |= B_ASYNC;
1379
1380 splx(s);
1381 /*
1382 * default (old) behavior, writing out only one block
1383 *
1384 * XXX returns b_bufsize instead of b_bcount for nwritten?
1385 */
1386 nwritten = bp->b_bufsize;
1387 (void) BUF_WRITE(bp);
1388
1389 return nwritten;
1390}
1391
1392/*
1393 * getnewbuf:
1394 *
1395 * Find and initialize a new buffer header, freeing up existing buffers
1396 * in the bufqueues as necessary. The new buffer is returned locked.
1397 *
1398 * Important: B_INVAL is not set. If the caller wishes to throw the
1399 * buffer away, the caller must set B_INVAL prior to calling brelse().
1400 *
1401 * We block if:
1402 * We have insufficient buffer headers
1403 * We have insufficient buffer space
1404 * buffer_map is too fragmented ( space reservation fails )
1405 * If we have to flush dirty buffers ( but we try to avoid this )
1406 *
1407 * To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1408 * Instead we ask the buf daemon to do it for us. We attempt to
1409 * avoid piecemeal wakeups of the pageout daemon.
1410 */
1411
1412static struct buf *
1413getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
1414{
1415 struct buf *bp;
1416 struct buf *nbp;
1417 int defrag = 0;
1418 int nqindex;
1419 int isspecial;
1420 static int flushingbufs;
1421
1422 if (curproc && (curproc->p_flag & P_BUFEXHAUST) == 0)
1423 isspecial = 0;
1424 else
1425 isspecial = 1;
1426
1427 ++getnewbufcalls;
1428 --getnewbufrestarts;
1429restart:
1430 ++getnewbufrestarts;
1431
1432 /*
1433 * Setup for scan. If we do not have enough free buffers,
1434 * we setup a degenerate case that immediately fails. Note
1435 * that if we are specially marked process, we are allowed to
1436 * dip into our reserves.
1437 *
1438 * The scanning sequence is nominally: EMPTY->EMPTYKVA->CLEAN
1439 *
1440 * We start with EMPTYKVA. If the list is empty we backup to EMPTY.
1441 * However, there are a number of cases (defragging, reusing, ...)
1442 * where we cannot backup.
1443 */
1444
1445 if (isspecial == 0 && numfreebuffers < lofreebuffers) {
1446 /*
1447 * This will cause an immediate failure
1448 */
1449 nqindex = QUEUE_CLEAN;
1450 nbp = NULL;
1451 } else {
1452 /*
1453 * Locate a buffer which already has KVA assigned. First
1454 * try EMPTYKVA buffers.
1455 */
1456 nqindex = QUEUE_EMPTYKVA;
1457 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA]);
1458
1459 if (nbp == NULL) {
1460 /*
1461 * If no EMPTYKVA buffers and we are either
1462 * defragging or reusing, locate a CLEAN buffer
1463 * to free or reuse. If bufspace useage is low
1464 * skip this step so we can allocate a new buffer.
1465 */
1466 if (defrag || bufspace >= lobufspace) {
1467 nqindex = QUEUE_CLEAN;
1468 nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
1469 }
1470
1471 /*
1472 * Nada. If we are allowed to allocate an EMPTY
1473 * buffer, go get one.
1474 */
1475 if (nbp == NULL && defrag == 0 &&
1476 (isspecial || bufspace < hibufspace)) {
1477 nqindex = QUEUE_EMPTY;
1478 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1479 }
1480 }
1481 }
1482
1483 /*
1484 * Run scan, possibly freeing data and/or kva mappings on the fly
1485 * depending.
1486 */
1487
1488 while ((bp = nbp) != NULL) {
1489 int qindex = nqindex;
1490
1491 /*
1492 * Calculate next bp ( we can only use it if we do not block
1493 * or do other fancy things ).
1494 */
1495 if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
1496 switch(qindex) {
1497 case QUEUE_EMPTY:
1498 nqindex = QUEUE_EMPTYKVA;
1499 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA])))
1500 break;
1501 /* fall through */
1502 case QUEUE_EMPTYKVA:
1503 nqindex = QUEUE_CLEAN;
1504 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN])))
1505 break;
1506 /* fall through */
1507 case QUEUE_CLEAN:
1508 /*
1509 * nbp is NULL.
1510 */
1511 break;
1512 }
1513 }
1514
1515 /*
1516 * Sanity Checks
1517 */
1518 KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp));
1519
1520 /*
1521 * Note: we no longer distinguish between VMIO and non-VMIO
1522 * buffers.
1523 */
1524
1525 KASSERT((bp->b_flags & B_DELWRI) == 0, ("delwri buffer %p found in queue %d", bp, qindex));
1526
1527 /*
1528 * If we are defragging then we need a buffer with
1529 * b_kvasize != 0. XXX this situation should no longer
1530 * occur, if defrag is non-zero the buffer's b_kvasize
1531 * should also be non-zero at this point. XXX
1532 */
1533 if (defrag && bp->b_kvasize == 0) {
1534 printf("Warning: defrag empty buffer %p\n", bp);
1535 continue;
1536 }
1537
1538 /*
1539 * Start freeing the bp. This is somewhat involved. nbp
1540 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1541 */
1542
1543 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1544 panic("getnewbuf: locked buf");
1545 bremfree(bp);
1546
1547 if (qindex == QUEUE_CLEAN) {
1548 if (bp->b_flags & B_VMIO) {
1549 bp->b_flags &= ~B_ASYNC;
1550 vfs_vmio_release(bp);
1551 }
1552 if (bp->b_vp)
1553 brelvp(bp);
1554 }
1555
1556 /*
1557 * NOTE: nbp is now entirely invalid. We can only restart
1558 * the scan from this point on.
1559 *
1560 * Get the rest of the buffer freed up. b_kva* is still
1561 * valid after this operation.
1562 */
1563
1564 if (bp->b_rcred != NOCRED) {
1565 crfree(bp->b_rcred);
1566 bp->b_rcred = NOCRED;
1567 }
1568 if (bp->b_wcred != NOCRED) {
1569 crfree(bp->b_wcred);
1570 bp->b_wcred = NOCRED;
1571 }
1572 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
1573 (*bioops.io_deallocate)(bp);
1574 if (bp->b_xflags & BX_BKGRDINPROG)
1575 panic("losing buffer 3");
1576 LIST_REMOVE(bp, b_hash);
1577 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1578
1579 if (bp->b_bufsize)
1580 allocbuf(bp, 0);
1581
1582 bp->b_flags = 0;
1583 bp->b_ioflags = 0;
1584 bp->b_xflags = 0;
1585 bp->b_dev = NODEV;
1586 bp->b_vp = NULL;
1587 bp->b_blkno = bp->b_lblkno = 0;
1588 bp->b_offset = NOOFFSET;
1589 bp->b_iodone = 0;
1590 bp->b_error = 0;
1591 bp->b_resid = 0;
1592 bp->b_bcount = 0;
1593 bp->b_npages = 0;
1594 bp->b_dirtyoff = bp->b_dirtyend = 0;
1595
1596 LIST_INIT(&bp->b_dep);
1597
1598 /*
1599 * If we are defragging then free the buffer.
1600 */
1601 if (defrag) {
1602 bp->b_flags |= B_INVAL;
1603 bfreekva(bp);
1604 brelse(bp);
1605 defrag = 0;
1606 goto restart;
1607 }
1608
1609 /*
1610 * If we are a normal process then deal with bufspace
1611 * hysteresis. A normal process tries to keep bufspace
1612 * between lobufspace and hibufspace. Note: if we encounter
1613 * a buffer with b_kvasize == 0 then it means we started
1614 * our scan on the EMPTY list and should allocate a new
1615 * buffer.
1616 */
1617 if (isspecial == 0) {
1618 if (bufspace > hibufspace)
1619 flushingbufs = 1;
1620 if (flushingbufs && bp->b_kvasize != 0) {
1621 bp->b_flags |= B_INVAL;
1622 bfreekva(bp);
1623 brelse(bp);
1624 goto restart;
1625 }
1626 if (bufspace < lobufspace)
1627 flushingbufs = 0;
1628 }
1629 break;
1630 }
1631
1632 /*
1633 * If we exhausted our list, sleep as appropriate. We may have to
1634 * wakeup various daemons and write out some dirty buffers.
1635 *
1636 * Generally we are sleeping due to insufficient buffer space.
1637 */
1638
1639 if (bp == NULL) {
1640 int flags;
1641 char *waitmsg;
1642
1643 if (defrag) {
1644 flags = VFS_BIO_NEED_BUFSPACE;
1645 waitmsg = "nbufkv";
1646 } else if (bufspace >= hibufspace) {
1647 waitmsg = "nbufbs";
1648 flags = VFS_BIO_NEED_BUFSPACE;
1649 } else {
1650 waitmsg = "newbuf";
1651 flags = VFS_BIO_NEED_ANY;
1652 }
1653
1654 bd_speedup(); /* heeeelp */
1655
1656 needsbuffer |= flags;
1657 while (needsbuffer & flags) {
1658 if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag,
1659 waitmsg, slptimeo))
1660 return (NULL);
1661 }
1662 } else {
1663 /*
1664 * We finally have a valid bp. We aren't quite out of the
1665 * woods, we still have to reserve kva space. In order
1666 * to keep fragmentation sane we only allocate kva in
1667 * BKVASIZE chunks.
1668 */
1669 maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
1670
1671 if (maxsize != bp->b_kvasize) {
1672 vm_offset_t addr = 0;
1673
1674 bfreekva(bp);
1675
1676 if (vm_map_findspace(buffer_map,
1677 vm_map_min(buffer_map), maxsize, &addr)) {
1678 /*
1679 * Uh oh. Buffer map is to fragmented. We
1680 * must defragment the map.
1681 */
1682 ++bufdefragcnt;
1683 defrag = 1;
1684 bp->b_flags |= B_INVAL;
1685 brelse(bp);
1686 goto restart;
1687 }
1688 if (addr) {
1689 vm_map_insert(buffer_map, NULL, 0,
1690 addr, addr + maxsize,
1691 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
1692
1693 bp->b_kvabase = (caddr_t) addr;
1694 bp->b_kvasize = maxsize;
1695 bufspace += bp->b_kvasize;
1696 ++bufreusecnt;
1697 }
1698 }
1699 bp->b_data = bp->b_kvabase;
1700 }
1701 return(bp);
1702}
1703
1704/*
1705 * waitfreebuffers:
1706 *
1707 * Wait for sufficient free buffers. Only called from normal processes.
1708 */
1709
1710static void
1711waitfreebuffers(int slpflag, int slptimeo)
1712{
1713 while (numfreebuffers < hifreebuffers) {
1714 if (numfreebuffers >= hifreebuffers)
1715 break;
1716 needsbuffer |= VFS_BIO_NEED_FREE;
1717 if (tsleep(&needsbuffer, (PRIBIO + 4)|slpflag, "biofre", slptimeo))
1718 break;
1719 }
1720}
1721
1722/*
1723 * buf_daemon:
1724 *
1725 * buffer flushing daemon. Buffers are normally flushed by the
1726 * update daemon but if it cannot keep up this process starts to
1727 * take the load in an attempt to prevent getnewbuf() from blocking.
1728 */
1729
1730static struct proc *bufdaemonproc;
1731static int bd_interval;
1732static int bd_flushto;
1733static int bd_flushinc;
1734
1735static struct kproc_desc buf_kp = {
1736 "bufdaemon",
1737 buf_daemon,
1738 &bufdaemonproc
1739};
1740SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp)
1741
1742static void
1743buf_daemon()
1744{
1745 int s;
1746
1747 /*
1748 * This process needs to be suspended prior to shutdown sync.
1749 */
1750 EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, bufdaemonproc,
1751 SHUTDOWN_PRI_LAST);
1752
1753 /*
1754 * This process is allowed to take the buffer cache to the limit
1755 */
1756 curproc->p_flag |= P_BUFEXHAUST;
1757 s = splbio();
1758
1759 bd_interval = 5 * hz; /* dynamically adjusted */
1760 bd_flushto = hidirtybuffers; /* dynamically adjusted */
1761 bd_flushinc = 1;
1762
1763 for (;;) {
1764 kproc_suspend_loop(bufdaemonproc);
1765
1766 bd_request = 0;
1767
1768 /*
1769 * Do the flush. Limit the number of buffers we flush in one
1770 * go. The failure condition occurs when processes are writing
1771 * buffers faster then we can dispose of them. In this case
1772 * we may be flushing so often that the previous set of flushes
1773 * have not had time to complete, causing us to run out of
1774 * physical buffers and block.
1775 */
1776 {
1777 int runcount = maxbdrun;
1778
1779 while (numdirtybuffers > bd_flushto && runcount) {
1780 --runcount;
1781 if (flushbufqueues() == 0)
1782 break;
1783 }
1784 }
1785
1786 if (bd_request ||
1787 tsleep(&bd_request, PVM, "psleep", bd_interval) == 0) {
1788 /*
1789 * Another request is pending or we were woken up
1790 * without timing out. Flush more.
1791 */
1792 --bd_flushto;
1793 if (bd_flushto >= numdirtybuffers - 5) {
1794 bd_flushto = numdirtybuffers - 10;
1795 bd_flushinc = 1;
1796 }
1797 if (bd_flushto < 2)
1798 bd_flushto = 2;
1799 } else {
1800 /*
1801 * We slept and timed out, we can slow down.
1802 */
1803 bd_flushto += bd_flushinc;
1804 if (bd_flushto > hidirtybuffers)
1805 bd_flushto = hidirtybuffers;
1806 ++bd_flushinc;
1807 if (bd_flushinc > hidirtybuffers / 20 + 1)
1808 bd_flushinc = hidirtybuffers / 20 + 1;
1809 }
1810
1811 /*
1812 * Set the interval on a linear scale based on hidirtybuffers
1813 * with a maximum frequency of 1/10 second.
1814 */
1815 bd_interval = bd_flushto * 5 * hz / hidirtybuffers;
1816 if (bd_interval < hz / 10)
1817 bd_interval = hz / 10;
1818 }
1819}
1820
1821/*
1822 * flushbufqueues:
1823 *
1824 * Try to flush a buffer in the dirty queue. We must be careful to
1825 * free up B_INVAL buffers instead of write them, which NFS is
1826 * particularly sensitive to.
1827 */
1828
1829static int
1830flushbufqueues(void)
1831{
1832 struct buf *bp;
1833 int r = 0;
1834
1835 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1836
1837 while (bp) {
1838 KASSERT((bp->b_flags & B_DELWRI), ("unexpected clean buffer %p", bp));
1839 if ((bp->b_flags & B_DELWRI) != 0 &&
1840 (bp->b_xflags & BX_BKGRDINPROG) == 0) {
1841 if (bp->b_flags & B_INVAL) {
1842 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1843 panic("flushbufqueues: locked buf");
1844 bremfree(bp);
1845 brelse(bp);
1846 ++r;
1847 break;
1848 }
1849 if (LIST_FIRST(&bp->b_dep) != NULL &&
1850 bioops.io_countdeps &&
1851 (bp->b_flags & B_DEFERRED) == 0 &&
1852 (*bioops.io_countdeps)(bp, 0)) {
1853 TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY],
1854 bp, b_freelist);
1855 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY],
1856 bp, b_freelist);
1857 bp->b_flags |= B_DEFERRED;
1858 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1859 continue;
1860 }
1861 vfs_bio_awrite(bp);
1862 ++r;
1863 break;
1864 }
1865 bp = TAILQ_NEXT(bp, b_freelist);
1866 }
1867 return (r);
1868}
1869
1870/*
1871 * Check to see if a block is currently memory resident.
1872 */
1873struct buf *
1874incore(struct vnode * vp, daddr_t blkno)
1875{
1876 struct buf *bp;
1877
1878 int s = splbio();
1879 bp = gbincore(vp, blkno);
1880 splx(s);
1881 return (bp);
1882}
1883
1884/*
1885 * Returns true if no I/O is needed to access the
1886 * associated VM object. This is like incore except
1887 * it also hunts around in the VM system for the data.
1888 */
1889
1890int
1891inmem(struct vnode * vp, daddr_t blkno)
1892{
1893 vm_object_t obj;
1894 vm_offset_t toff, tinc, size;
1895 vm_page_t m;
1896 vm_ooffset_t off;
1897
1898 if (incore(vp, blkno))
1899 return 1;
1900 if (vp->v_mount == NULL)
1901 return 0;
1902 if ((vp->v_object == NULL) || (vp->v_flag & VOBJBUF) == 0)
1903 return 0;
1904
1905 obj = vp->v_object;
1906 size = PAGE_SIZE;
1907 if (size > vp->v_mount->mnt_stat.f_iosize)
1908 size = vp->v_mount->mnt_stat.f_iosize;
1909 off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
1910
1911 for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
1912 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
1913 if (!m)
1914 return 0;
1915 tinc = size;
1916 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
1917 tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
1918 if (vm_page_is_valid(m,
1919 (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
1920 return 0;
1921 }
1922 return 1;
1923}
1924
1925/*
1926 * vfs_setdirty:
1927 *
1928 * Sets the dirty range for a buffer based on the status of the dirty
1929 * bits in the pages comprising the buffer.
1930 *
1931 * The range is limited to the size of the buffer.
1932 *
1933 * This routine is primarily used by NFS, but is generalized for the
1934 * B_VMIO case.
1935 */
1936static void
1937vfs_setdirty(struct buf *bp)
1938{
1939 int i;
1940 vm_object_t object;
1941
1942 /*
1943 * Degenerate case - empty buffer
1944 */
1945
1946 if (bp->b_bufsize == 0)
1947 return;
1948
1949 /*
1950 * We qualify the scan for modified pages on whether the
1951 * object has been flushed yet. The OBJ_WRITEABLE flag
1952 * is not cleared simply by protecting pages off.
1953 */
1954
1955 if ((bp->b_flags & B_VMIO) == 0)
1956 return;
1957
1958 object = bp->b_pages[0]->object;
1959
1960 if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY))
1961 printf("Warning: object %p writeable but not mightbedirty\n", object);
1962 if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY))
1963 printf("Warning: object %p mightbedirty but not writeable\n", object);
1964
1965 if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) {
1966 vm_offset_t boffset;
1967 vm_offset_t eoffset;
1968
1969 /*
1970 * test the pages to see if they have been modified directly
1971 * by users through the VM system.
1972 */
1973 for (i = 0; i < bp->b_npages; i++) {
1974 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
1975 vm_page_test_dirty(bp->b_pages[i]);
1976 }
1977
1978 /*
1979 * Calculate the encompassing dirty range, boffset and eoffset,
1980 * (eoffset - boffset) bytes.
1981 */
1982
1983 for (i = 0; i < bp->b_npages; i++) {
1984 if (bp->b_pages[i]->dirty)
1985 break;
1986 }
1987 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1988
1989 for (i = bp->b_npages - 1; i >= 0; --i) {
1990 if (bp->b_pages[i]->dirty) {
1991 break;
1992 }
1993 }
1994 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1995
1996 /*
1997 * Fit it to the buffer.
1998 */
1999
2000 if (eoffset > bp->b_bcount)
2001 eoffset = bp->b_bcount;
2002
2003 /*
2004 * If we have a good dirty range, merge with the existing
2005 * dirty range.
2006 */
2007
2008 if (boffset < eoffset) {
2009 if (bp->b_dirtyoff > boffset)
2010 bp->b_dirtyoff = boffset;
2011 if (bp->b_dirtyend < eoffset)
2012 bp->b_dirtyend = eoffset;
2013 }
2014 }
2015}
2016
2017/*
2018 * getblk:
2019 *
2020 * Get a block given a specified block and offset into a file/device.
2021 * The buffers B_DONE bit will be cleared on return, making it almost
2022 * ready for an I/O initiation. B_INVAL may or may not be set on
2023 * return. The caller should clear B_INVAL prior to initiating a
2024 * READ.
2025 *
2026 * For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2027 * an existing buffer.
2028 *
2029 * For a VMIO buffer, B_CACHE is modified according to the backing VM.
2030 * If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2031 * and then cleared based on the backing VM. If the previous buffer is
2032 * non-0-sized but invalid, B_CACHE will be cleared.
2033 *
2034 * If getblk() must create a new buffer, the new buffer is returned with
2035 * both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2036 * case it is returned with B_INVAL clear and B_CACHE set based on the
2037 * backing VM.
2038 *
2039 * getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2040 * B_CACHE bit is clear.
2041 *
2042 * What this means, basically, is that the caller should use B_CACHE to
2043 * determine whether the buffer is fully valid or not and should clear
2044 * B_INVAL prior to issuing a read. If the caller intends to validate
2045 * the buffer by loading its data area with something, the caller needs
2046 * to clear B_INVAL. If the caller does this without issuing an I/O,
2047 * the caller should set B_CACHE ( as an optimization ), else the caller
2048 * should issue the I/O and biodone() will set B_CACHE if the I/O was
2049 * a write attempt or if it was a successfull read. If the caller
2050 * intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
2051 * prior to issuing the READ. biodone() will *not* clear B_INVAL.
2052 */
2053struct buf *
2054getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo)
2055{
2056 struct buf *bp;
2057 int s;
2058 struct bufhashhdr *bh;
2059
2060 if (size > MAXBSIZE)
2061 panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE);
2062
2063 s = splbio();
2064loop:
2065 /*
2066 * Block if we are low on buffers. Certain processes are allowed
2067 * to completely exhaust the buffer cache.
2068 *
2069 * If this check ever becomes a bottleneck it may be better to
2070 * move it into the else, when gbincore() fails. At the moment
2071 * it isn't a problem.
2072 */
2073 if (!curproc || (curproc->p_flag & P_BUFEXHAUST)) {
2074 if (numfreebuffers == 0) {
2075 if (!curproc)
2076 return NULL;
2077 needsbuffer |= VFS_BIO_NEED_ANY;
2078 tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, "newbuf",
2079 slptimeo);
2080 }
2081 } else if (numfreebuffers < lofreebuffers) {
2082 waitfreebuffers(slpflag, slptimeo);
2083 }
2084
2085 if ((bp = gbincore(vp, blkno))) {
2086 /*
2087 * Buffer is in-core. If the buffer is not busy, it must
2088 * be on a queue.
2089 */
2090
2091 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
2092 if (BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL,
2093 "getblk", slpflag, slptimeo) == ENOLCK)
2094 goto loop;
2095 splx(s);
2096 return (struct buf *) NULL;
2097 }
2098
2099 /*
2100 * The buffer is locked. B_CACHE is cleared if the buffer is
2101 * invalid. Ohterwise, for a non-VMIO buffer, B_CACHE is set
2102 * and for a VMIO buffer B_CACHE is adjusted according to the
2103 * backing VM cache.
2104 */
2105 if (bp->b_flags & B_INVAL)
2106 bp->b_flags &= ~B_CACHE;
2107 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
2108 bp->b_flags |= B_CACHE;
2109 bremfree(bp);
2110
2111 /*
2112 * check for size inconsistancies for non-VMIO case.
2113 */
2114
2115 if (bp->b_bcount != size) {
2116 if ((bp->b_flags & B_VMIO) == 0 ||
2117 (size > bp->b_kvasize)) {
2118 if (bp->b_flags & B_DELWRI) {
2119 bp->b_flags |= B_NOCACHE;
2120 BUF_WRITE(bp);
2121 } else {
2122 if ((bp->b_flags & B_VMIO) &&
2123 (LIST_FIRST(&bp->b_dep) == NULL)) {
2124 bp->b_flags |= B_RELBUF;
2125 brelse(bp);
2126 } else {
2127 bp->b_flags |= B_NOCACHE;
2128 BUF_WRITE(bp);
2129 }
2130 }
2131 goto loop;
2132 }
2133 }
2134
2135 /*
2136 * If the size is inconsistant in the VMIO case, we can resize
2137 * the buffer. This might lead to B_CACHE getting set or
2138 * cleared. If the size has not changed, B_CACHE remains
2139 * unchanged from its previous state.
2140 */
2141
2142 if (bp->b_bcount != size)
2143 allocbuf(bp, size);
2144
2145 KASSERT(bp->b_offset != NOOFFSET,
2146 ("getblk: no buffer offset"));
2147
2148 /*
2149 * A buffer with B_DELWRI set and B_CACHE clear must
2150 * be committed before we can return the buffer in
2151 * order to prevent the caller from issuing a read
2152 * ( due to B_CACHE not being set ) and overwriting
2153 * it.
2154 *
2155 * Most callers, including NFS and FFS, need this to
2156 * operate properly either because they assume they
2157 * can issue a read if B_CACHE is not set, or because
2158 * ( for example ) an uncached B_DELWRI might loop due
2159 * to softupdates re-dirtying the buffer. In the latter
2160 * case, B_CACHE is set after the first write completes,
2161 * preventing further loops.
2162 */
2163
2164 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
2165 BUF_WRITE(bp);
2166 goto loop;
2167 }
2168
2169 splx(s);
2170 bp->b_flags &= ~B_DONE;
2171 } else {
2172 /*
2173 * Buffer is not in-core, create new buffer. The buffer
2174 * returned by getnewbuf() is locked. Note that the returned
2175 * buffer is also considered valid (not marked B_INVAL).
2176 */
2177 int bsize, maxsize, vmio;
2178 off_t offset;
2179
2180 if (vn_isdisk(vp, NULL))
2181 bsize = DEV_BSIZE;
2182 else if (vp->v_mountedhere)
2183 bsize = vp->v_mountedhere->mnt_stat.f_iosize;
2184 else if (vp->v_mount)
2185 bsize = vp->v_mount->mnt_stat.f_iosize;
2186 else
2187 bsize = size;
2188
2189 offset = (off_t)blkno * bsize;
2190 vmio = (vp->v_object != 0) && (vp->v_flag & VOBJBUF);
2191 maxsize = vmio ? size + (offset & PAGE_MASK) : size;
2192 maxsize = imax(maxsize, bsize);
2193
2194 if ((bp = getnewbuf(slpflag, slptimeo, size, maxsize)) == NULL) {
2195 if (slpflag || slptimeo) {
2196 splx(s);
2197 return NULL;
2198 }
2199 goto loop;
2200 }
2201
2202 /*
2203 * This code is used to make sure that a buffer is not
2204 * created while the getnewbuf routine is blocked.
2205 * This can be a problem whether the vnode is locked or not.
2206 * If the buffer is created out from under us, we have to
2207 * throw away the one we just created. There is now window
2208 * race because we are safely running at splbio() from the
2209 * point of the duplicate buffer creation through to here,
2210 * and we've locked the buffer.
2211 */
2212 if (gbincore(vp, blkno)) {
2213 bp->b_flags |= B_INVAL;
2214 brelse(bp);
2215 goto loop;
2216 }
2217
2218 /*
2219 * Insert the buffer into the hash, so that it can
2220 * be found by incore.
2221 */
2222 bp->b_blkno = bp->b_lblkno = blkno;
2223 bp->b_offset = offset;
2224
2225 bgetvp(vp, bp);
2226 LIST_REMOVE(bp, b_hash);
2227 bh = bufhash(vp, blkno);
2228 LIST_INSERT_HEAD(bh, bp, b_hash);
2229
2230 /*
2231 * set B_VMIO bit. allocbuf() the buffer bigger. Since the
2232 * buffer size starts out as 0, B_CACHE will be set by
2233 * allocbuf() for the VMIO case prior to it testing the
2234 * backing store for validity.
2235 */
2236
2237 if (vmio) {
2238 bp->b_flags |= B_VMIO;
2239#if defined(VFS_BIO_DEBUG)
2240 if (vp->v_type != VREG && vp->v_type != VBLK)
2241 printf("getblk: vmioing file type %d???\n", vp->v_type);
2242#endif
2243 } else {
2244 bp->b_flags &= ~B_VMIO;
2245 }
2246
2247 allocbuf(bp, size);
2248
2249 splx(s);
2250 bp->b_flags &= ~B_DONE;
2251 }
2252 return (bp);
2253}
2254
2255/*
2256 * Get an empty, disassociated buffer of given size. The buffer is initially
2257 * set to B_INVAL.
2258 */
2259struct buf *
2260geteblk(int size)
2261{
2262 struct buf *bp;
2263 int s;
2264 int maxsize;
2265
2266 maxsize = (size + BKVAMASK) & ~BKVAMASK;
2267
2268 s = splbio();
2269 while ((bp = getnewbuf(0, 0, size, maxsize)) == 0);
2270 splx(s);
2271 allocbuf(bp, size);
2272 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
2273 return (bp);
2274}
2275
2276
2277/*
2278 * This code constitutes the buffer memory from either anonymous system
2279 * memory (in the case of non-VMIO operations) or from an associated
2280 * VM object (in the case of VMIO operations). This code is able to
2281 * resize a buffer up or down.
2282 *
2283 * Note that this code is tricky, and has many complications to resolve
2284 * deadlock or inconsistant data situations. Tread lightly!!!
2285 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2286 * the caller. Calling this code willy nilly can result in the loss of data.
2287 *
2288 * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
2289 * B_CACHE for the non-VMIO case.
2290 */
2291
2292int
2293allocbuf(struct buf *bp, int size)
2294{
2295 int newbsize, mbsize;
2296 int i;
2297
2298 if (BUF_REFCNT(bp) == 0)
2299 panic("allocbuf: buffer not busy");
2300
2301 if (bp->b_kvasize < size)
2302 panic("allocbuf: buffer too small");
2303
2304 if ((bp->b_flags & B_VMIO) == 0) {
2305 caddr_t origbuf;
2306 int origbufsize;
2307 /*
2308 * Just get anonymous memory from the kernel. Don't
2309 * mess with B_CACHE.
2310 */
2311 mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2312#if !defined(NO_B_MALLOC)
2313 if (bp->b_flags & B_MALLOC)
2314 newbsize = mbsize;
2315 else
2316#endif
2317 newbsize = round_page(size);
2318
2319 if (newbsize < bp->b_bufsize) {
2320#if !defined(NO_B_MALLOC)
2321 /*
2322 * malloced buffers are not shrunk
2323 */
2324 if (bp->b_flags & B_MALLOC) {
2325 if (newbsize) {
2326 bp->b_bcount = size;
2327 } else {
2328 free(bp->b_data, M_BIOBUF);
2329 bufmallocspace -= bp->b_bufsize;
2330 runningbufspace -= bp->b_bufsize;
2331 if (bp->b_bufsize)
2332 bufspacewakeup();
2333 bp->b_data = bp->b_kvabase;
2334 bp->b_bufsize = 0;
2335 bp->b_bcount = 0;
2336 bp->b_flags &= ~B_MALLOC;
2337 }
2338 return 1;
2339 }
2340#endif
2341 vm_hold_free_pages(
2342 bp,
2343 (vm_offset_t) bp->b_data + newbsize,
2344 (vm_offset_t) bp->b_data + bp->b_bufsize);
2345 } else if (newbsize > bp->b_bufsize) {
2346#if !defined(NO_B_MALLOC)
2347 /*
2348 * We only use malloced memory on the first allocation.
2349 * and revert to page-allocated memory when the buffer
2350 * grows.
2351 */
2352 if ( (bufmallocspace < maxbufmallocspace) &&
2353 (bp->b_bufsize == 0) &&
2354 (mbsize <= PAGE_SIZE/2)) {
2355
2356 bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
2357 bp->b_bufsize = mbsize;
2358 bp->b_bcount = size;
2359 bp->b_flags |= B_MALLOC;
2360 bufmallocspace += mbsize;
2361 runningbufspace += bp->b_bufsize;
2362 return 1;
2363 }
2364#endif
2365 origbuf = NULL;
2366 origbufsize = 0;
2367#if !defined(NO_B_MALLOC)
2368 /*
2369 * If the buffer is growing on its other-than-first allocation,
2370 * then we revert to the page-allocation scheme.
2371 */
2372 if (bp->b_flags & B_MALLOC) {
2373 origbuf = bp->b_data;
2374 origbufsize = bp->b_bufsize;
2375 bp->b_data = bp->b_kvabase;
2376 bufmallocspace -= bp->b_bufsize;
2377 runningbufspace -= bp->b_bufsize;
2378 if (bp->b_bufsize)
2379 bufspacewakeup();
2380 bp->b_bufsize = 0;
2381 bp->b_flags &= ~B_MALLOC;
2382 newbsize = round_page(newbsize);
2383 }
2384#endif
2385 vm_hold_load_pages(
2386 bp,
2387 (vm_offset_t) bp->b_data + bp->b_bufsize,
2388 (vm_offset_t) bp->b_data + newbsize);
2389#if !defined(NO_B_MALLOC)
2390 if (origbuf) {
2391 bcopy(origbuf, bp->b_data, origbufsize);
2392 free(origbuf, M_BIOBUF);
2393 }
2394#endif
2395 }
2396 } else {
2397 vm_page_t m;
2398 int desiredpages;
2399
2400 newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2401 desiredpages = (size == 0) ? 0 :
2402 num_pages((bp->b_offset & PAGE_MASK) + newbsize);
2403
2404#if !defined(NO_B_MALLOC)
2405 if (bp->b_flags & B_MALLOC)
2406 panic("allocbuf: VMIO buffer can't be malloced");
2407#endif
2408 /*
2409 * Set B_CACHE initially if buffer is 0 length or will become
2410 * 0-length.
2411 */
2412 if (size == 0 || bp->b_bufsize == 0)
2413 bp->b_flags |= B_CACHE;
2414
2415 if (newbsize < bp->b_bufsize) {
2416 /*
2417 * DEV_BSIZE aligned new buffer size is less then the
2418 * DEV_BSIZE aligned existing buffer size. Figure out
2419 * if we have to remove any pages.
2420 */
2421 if (desiredpages < bp->b_npages) {
2422 for (i = desiredpages; i < bp->b_npages; i++) {
2423 /*
2424 * the page is not freed here -- it
2425 * is the responsibility of
2426 * vnode_pager_setsize
2427 */
2428 m = bp->b_pages[i];
2429 KASSERT(m != bogus_page,
2430 ("allocbuf: bogus page found"));
2431 while (vm_page_sleep_busy(m, TRUE, "biodep"))
2432 ;
2433
2434 bp->b_pages[i] = NULL;
2435 vm_page_unwire(m, 0);
2436 }
2437 pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) +
2438 (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages));
2439 bp->b_npages = desiredpages;
2440 }
2441 } else if (size > bp->b_bcount) {
2442 /*
2443 * We are growing the buffer, possibly in a
2444 * byte-granular fashion.
2445 */
2446 struct vnode *vp;
2447 vm_object_t obj;
2448 vm_offset_t toff;
2449 vm_offset_t tinc;
2450
2451 /*
2452 * Step 1, bring in the VM pages from the object,
2453 * allocating them if necessary. We must clear
2454 * B_CACHE if these pages are not valid for the
2455 * range covered by the buffer.
2456 */
2457
2458 vp = bp->b_vp;
2459 obj = vp->v_object;
2460
2461 while (bp->b_npages < desiredpages) {
2462 vm_page_t m;
2463 vm_pindex_t pi;
2464
2465 pi = OFF_TO_IDX(bp->b_offset) + bp->b_npages;
2466 if ((m = vm_page_lookup(obj, pi)) == NULL) {
2467 m = vm_page_alloc(obj, pi, VM_ALLOC_NORMAL);
2468 if (m == NULL) {
2469 VM_WAIT;
2470 vm_pageout_deficit += desiredpages - bp->b_npages;
2471 } else {
2472 vm_page_wire(m);
2473 vm_page_wakeup(m);
2474 bp->b_flags &= ~B_CACHE;
2475 bp->b_pages[bp->b_npages] = m;
2476 ++bp->b_npages;
2477 }
2478 continue;
2479 }
2480
2481 /*
2482 * We found a page. If we have to sleep on it,
2483 * retry because it might have gotten freed out
2484 * from under us.
2485 *
2486 * We can only test PG_BUSY here. Blocking on
2487 * m->busy might lead to a deadlock:
2488 *
2489 * vm_fault->getpages->cluster_read->allocbuf
2490 *
2491 */
2492
2493 if (vm_page_sleep_busy(m, FALSE, "pgtblk"))
2494 continue;
2495
2496 /*
2497 * We have a good page. Should we wakeup the
2498 * page daemon?
2499 */
2500 if ((curproc != pageproc) &&
2501 ((m->queue - m->pc) == PQ_CACHE) &&
2502 ((cnt.v_free_count + cnt.v_cache_count) <
2503 (cnt.v_free_min + cnt.v_cache_min))) {
2504 pagedaemon_wakeup();
2505 }
2506 vm_page_flag_clear(m, PG_ZERO);
2507 vm_page_wire(m);
2508 bp->b_pages[bp->b_npages] = m;
2509 ++bp->b_npages;
2510 }
2511
2512 /*
2513 * Step 2. We've loaded the pages into the buffer,
2514 * we have to figure out if we can still have B_CACHE
2515 * set. Note that B_CACHE is set according to the
2516 * byte-granular range ( bcount and size ), new the
2517 * aligned range ( newbsize ).
2518 *
2519 * The VM test is against m->valid, which is DEV_BSIZE
2520 * aligned. Needless to say, the validity of the data
2521 * needs to also be DEV_BSIZE aligned. Note that this
2522 * fails with NFS if the server or some other client
2523 * extends the file's EOF. If our buffer is resized,
2524 * B_CACHE may remain set! XXX
2525 */
2526
2527 toff = bp->b_bcount;
2528 tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2529
2530 while ((bp->b_flags & B_CACHE) && toff < size) {
2531 vm_pindex_t pi;
2532
2533 if (tinc > (size - toff))
2534 tinc = size - toff;
2535
2536 pi = ((bp->b_offset & PAGE_MASK) + toff) >>
2537 PAGE_SHIFT;
2538
2539 vfs_buf_test_cache(
2540 bp,
2541 bp->b_offset,
2542 toff,
2543 tinc,
2544 bp->b_pages[pi]
2545 );
2546 toff += tinc;
2547 tinc = PAGE_SIZE;
2548 }
2549
2550 /*
2551 * Step 3, fixup the KVM pmap. Remember that
2552 * bp->b_data is relative to bp->b_offset, but
2553 * bp->b_offset may be offset into the first page.
2554 */
2555
2556 bp->b_data = (caddr_t)
2557 trunc_page((vm_offset_t)bp->b_data);
2558 pmap_qenter(
2559 (vm_offset_t)bp->b_data,
2560 bp->b_pages,
2561 bp->b_npages
2562 );
2563 bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
2564 (vm_offset_t)(bp->b_offset & PAGE_MASK));
2565 }
2566 }
2567 runningbufspace += (newbsize - bp->b_bufsize);
2568 if (newbsize < bp->b_bufsize)
2569 bufspacewakeup();
2570 bp->b_bufsize = newbsize; /* actual buffer allocation */
2571 bp->b_bcount = size; /* requested buffer size */
2572 return 1;
2573}
2574
2575/*
2576 * bufwait:
2577 *
2578 * Wait for buffer I/O completion, returning error status. The buffer
2579 * is left locked and B_DONE on return. B_EINTR is converted into a EINTR
2580 * error and cleared.
2581 */
2582int
2583bufwait(register struct buf * bp)
2584{
2585 int s;
2586
2587 s = splbio();
2588 while ((bp->b_flags & B_DONE) == 0) {
2589 if (bp->b_iocmd == BIO_READ)
2590 tsleep(bp, PRIBIO, "biord", 0);
2591 else
2592 tsleep(bp, PRIBIO, "biowr", 0);
2593 }
2594 splx(s);
2595 if (bp->b_flags & B_EINTR) {
2596 bp->b_flags &= ~B_EINTR;
2597 return (EINTR);
2598 }
2599 if (bp->b_ioflags & BIO_ERROR) {
2600 return (bp->b_error ? bp->b_error : EIO);
2601 } else {
2602 return (0);
2603 }
2604}
2605
2606 /*
2607 * Call back function from struct bio back up to struct buf.
2608 * The corresponding initialization lives in sys/conf.h:DEV_STRATEGY().
2609 */
2610void
2611bufdonebio(struct bio *bp)
2612{
2613 bufdone(bp->bio_caller2);
2614}
2615
2616/*
2617 * bufdone:
2618 *
2619 * Finish I/O on a buffer, optionally calling a completion function.
2620 * This is usually called from an interrupt so process blocking is
2621 * not allowed.
2622 *
2623 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2624 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
2625 * assuming B_INVAL is clear.
2626 *
2627 * For the VMIO case, we set B_CACHE if the op was a read and no
2628 * read error occured, or if the op was a write. B_CACHE is never
2629 * set if the buffer is invalid or otherwise uncacheable.
2630 *
2631 * biodone does not mess with B_INVAL, allowing the I/O routine or the
2632 * initiator to leave B_INVAL set to brelse the buffer out of existance
2633 * in the biodone routine.
2634 */
2635void
2636bufdone(struct buf *bp)
2637{
2638 int s;
2639 void (*biodone) __P((struct buf *));
2640
2641 s = splbio();
2642
2643 KASSERT(BUF_REFCNT(bp) > 0, ("biodone: bp %p not busy %d", bp, BUF_REFCNT(bp)));
2644 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
2645
2646 bp->b_flags |= B_DONE;
2647
2648 if (bp->b_iocmd == BIO_DELETE) {
2649 brelse(bp);
2650 splx(s);
2651 return;
2652 }
2653
2654 if (bp->b_iocmd == BIO_WRITE) {
2655 vwakeup(bp);
2656 }
2657
2658 /* call optional completion function if requested */
2659 if (bp->b_iodone != NULL) {
2660 biodone = bp->b_iodone;
2661 bp->b_iodone = NULL;
2662 (*biodone) (bp);
2663 splx(s);
2664 return;
2665 }
2666 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
2667 (*bioops.io_complete)(bp);
2668
2669 if (bp->b_flags & B_VMIO) {
2670 int i, resid;
2671 vm_ooffset_t foff;
2672 vm_page_t m;
2673 vm_object_t obj;
2674 int iosize;
2675 struct vnode *vp = bp->b_vp;
2676
2677 obj = vp->v_object;
2678
2679#if defined(VFS_BIO_DEBUG)
2680 if (vp->v_usecount == 0) {
2681 panic("biodone: zero vnode ref count");
2682 }
2683
2684 if (vp->v_object == NULL) {
2685 panic("biodone: missing VM object");
2686 }
2687
2688 if ((vp->v_flag & VOBJBUF) == 0) {
2689 panic("biodone: vnode is not setup for merged cache");
2690 }
2691#endif
2692
2693 foff = bp->b_offset;
2694 KASSERT(bp->b_offset != NOOFFSET,
2695 ("biodone: no buffer offset"));
2696
2697 if (!obj) {
2698 panic("biodone: no object");
2699 }
2700#if defined(VFS_BIO_DEBUG)
2701 if (obj->paging_in_progress < bp->b_npages) {
2702 printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n",
2703 obj->paging_in_progress, bp->b_npages);
2704 }
2705#endif
2706
2707 /*
2708 * Set B_CACHE if the op was a normal read and no error
2709 * occured. B_CACHE is set for writes in the b*write()
2710 * routines.
2711 */
2712 iosize = bp->b_bcount - bp->b_resid;
2713 if (bp->b_iocmd == BIO_READ &&
2714 !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
2715 !(bp->b_ioflags & BIO_ERROR)) {
2716 bp->b_flags |= B_CACHE;
2717 }
2718
2719 for (i = 0; i < bp->b_npages; i++) {
2720 int bogusflag = 0;
2721 m = bp->b_pages[i];
2722 if (m == bogus_page) {
2723 bogusflag = 1;
2724 m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2725 if (!m) {
2726#if defined(VFS_BIO_DEBUG)
2727 printf("biodone: page disappeared\n");
2728#endif
2729 vm_object_pip_subtract(obj, 1);
2730 bp->b_flags &= ~B_CACHE;
2731 continue;
2732 }
2733 bp->b_pages[i] = m;
2734 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2735 }
2736#if defined(VFS_BIO_DEBUG)
2737 if (OFF_TO_IDX(foff) != m->pindex) {
2738 printf(
2739"biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2740 (unsigned long)foff, m->pindex);
2741 }
2742#endif
2743 resid = IDX_TO_OFF(m->pindex + 1) - foff;
2744 if (resid > iosize)
2745 resid = iosize;
2746
2747 /*
2748 * In the write case, the valid and clean bits are
2749 * already changed correctly ( see bdwrite() ), so we
2750 * only need to do this here in the read case.
2751 */
2752 if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
2753 vfs_page_set_valid(bp, foff, i, m);
2754 }
2755 vm_page_flag_clear(m, PG_ZERO);
2756
2757 /*
2758 * when debugging new filesystems or buffer I/O methods, this
2759 * is the most common error that pops up. if you see this, you
2760 * have not set the page busy flag correctly!!!
2761 */
2762 if (m->busy == 0) {
2763 printf("biodone: page busy < 0, "
2764 "pindex: %d, foff: 0x(%x,%x), "
2765 "resid: %d, index: %d\n",
2766 (int) m->pindex, (int)(foff >> 32),
2767 (int) foff & 0xffffffff, resid, i);
2768 if (!vn_isdisk(vp, NULL))
2769 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2770 bp->b_vp->v_mount->mnt_stat.f_iosize,
2771 (int) bp->b_lblkno,
2772 bp->b_flags, bp->b_npages);
2773 else
2774 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2775 (int) bp->b_lblkno,
2776 bp->b_flags, bp->b_npages);
2777 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2778 m->valid, m->dirty, m->wire_count);
2779 panic("biodone: page busy < 0\n");
2780 }
2781 vm_page_io_finish(m);
2782 vm_object_pip_subtract(obj, 1);
2783 foff += resid;
2784 iosize -= resid;
2785 }
2786 if (obj)
2787 vm_object_pip_wakeupn(obj, 0);
2788 }
2789 /*
2790 * For asynchronous completions, release the buffer now. The brelse
2791 * will do a wakeup there if necessary - so no need to do a wakeup
2792 * here in the async case. The sync case always needs to do a wakeup.
2793 */
2794
2795 if (bp->b_flags & B_ASYNC) {
2796 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
2797 brelse(bp);
2798 else
2799 bqrelse(bp);
2800 } else {
2801 wakeup(bp);
2802 }
2803 splx(s);
2804}
2805
2806/*
2807 * This routine is called in lieu of iodone in the case of
2808 * incomplete I/O. This keeps the busy status for pages
2809 * consistant.
2810 */
2811void
2812vfs_unbusy_pages(struct buf * bp)
2813{
2814 int i;
2815
2816 if (bp->b_flags & B_VMIO) {
2817 struct vnode *vp = bp->b_vp;
2818 vm_object_t obj = vp->v_object;
2819
2820 for (i = 0; i < bp->b_npages; i++) {
2821 vm_page_t m = bp->b_pages[i];
2822
2823 if (m == bogus_page) {
2824 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
2825 if (!m) {
2826 panic("vfs_unbusy_pages: page missing\n");
2827 }
2828 bp->b_pages[i] = m;
2829 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2830 }
2831 vm_object_pip_subtract(obj, 1);
2832 vm_page_flag_clear(m, PG_ZERO);
2833 vm_page_io_finish(m);
2834 }
2835 vm_object_pip_wakeupn(obj, 0);
2836 }
2837}
2838
2839/*
2840 * vfs_page_set_valid:
2841 *
2842 * Set the valid bits in a page based on the supplied offset. The
2843 * range is restricted to the buffer's size.
2844 *
2845 * This routine is typically called after a read completes.
2846 */
2847static void
2848vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m)
2849{
2850 vm_ooffset_t soff, eoff;
2851
2852 /*
2853 * Start and end offsets in buffer. eoff - soff may not cross a
2854 * page boundry or cross the end of the buffer. The end of the
2855 * buffer, in this case, is our file EOF, not the allocation size
2856 * of the buffer.
2857 */
2858 soff = off;
2859 eoff = (off + PAGE_SIZE) & ~PAGE_MASK;
2860 if (eoff > bp->b_offset + bp->b_bcount)
2861 eoff = bp->b_offset + bp->b_bcount;
2862
2863 /*
2864 * Set valid range. This is typically the entire buffer and thus the
2865 * entire page.
2866 */
2867 if (eoff > soff) {
2868 vm_page_set_validclean(
2869 m,
2870 (vm_offset_t) (soff & PAGE_MASK),
2871 (vm_offset_t) (eoff - soff)
2872 );
2873 }
2874}
2875
2876/*
2877 * This routine is called before a device strategy routine.
2878 * It is used to tell the VM system that paging I/O is in
2879 * progress, and treat the pages associated with the buffer
2880 * almost as being PG_BUSY. Also the object paging_in_progress
2881 * flag is handled to make sure that the object doesn't become
2882 * inconsistant.
2883 *
2884 * Since I/O has not been initiated yet, certain buffer flags
2885 * such as BIO_ERROR or B_INVAL may be in an inconsistant state
2886 * and should be ignored.
2887 */
2888void
2889vfs_busy_pages(struct buf * bp, int clear_modify)
2890{
2891 int i, bogus;
2892
2893 if (bp->b_flags & B_VMIO) {
2894 struct vnode *vp = bp->b_vp;
2895 vm_object_t obj = vp->v_object;
2896 vm_ooffset_t foff;
2897
2898 foff = bp->b_offset;
2899 KASSERT(bp->b_offset != NOOFFSET,
2900 ("vfs_busy_pages: no buffer offset"));
2901 vfs_setdirty(bp);
2902
2903retry:
2904 for (i = 0; i < bp->b_npages; i++) {
2905 vm_page_t m = bp->b_pages[i];
2906 if (vm_page_sleep_busy(m, FALSE, "vbpage"))
2907 goto retry;
2908 }
2909
2910 bogus = 0;
2911 for (i = 0; i < bp->b_npages; i++) {
2912 vm_page_t m = bp->b_pages[i];
2913
2914 vm_page_flag_clear(m, PG_ZERO);
2915 if ((bp->b_flags & B_CLUSTER) == 0) {
2916 vm_object_pip_add(obj, 1);
2917 vm_page_io_start(m);
2918 }
2919
2920 /*
2921 * When readying a buffer for a read ( i.e
2922 * clear_modify == 0 ), it is important to do
2923 * bogus_page replacement for valid pages in
2924 * partially instantiated buffers. Partially
2925 * instantiated buffers can, in turn, occur when
2926 * reconstituting a buffer from its VM backing store
2927 * base. We only have to do this if B_CACHE is
2928 * clear ( which causes the I/O to occur in the
2929 * first place ). The replacement prevents the read
2930 * I/O from overwriting potentially dirty VM-backed
2931 * pages. XXX bogus page replacement is, uh, bogus.
2932 * It may not work properly with small-block devices.
2933 * We need to find a better way.
2934 */
2935
2936 vm_page_protect(m, VM_PROT_NONE);
2937 if (clear_modify)
2938 vfs_page_set_valid(bp, foff, i, m);
2939 else if (m->valid == VM_PAGE_BITS_ALL &&
2940 (bp->b_flags & B_CACHE) == 0) {
2941 bp->b_pages[i] = bogus_page;
2942 bogus++;
2943 }
2944 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2945 }
2946 if (bogus)
2947 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2948 }
2949}
2950
2951/*
2952 * Tell the VM system that the pages associated with this buffer
2953 * are clean. This is used for delayed writes where the data is
2954 * going to go to disk eventually without additional VM intevention.
2955 *
2956 * Note that while we only really need to clean through to b_bcount, we
2957 * just go ahead and clean through to b_bufsize.
2958 */
2959static void
2960vfs_clean_pages(struct buf * bp)
2961{
2962 int i;
2963
2964 if (bp->b_flags & B_VMIO) {
2965 vm_ooffset_t foff;
2966
2967 foff = bp->b_offset;
2968 KASSERT(bp->b_offset != NOOFFSET,
2969 ("vfs_clean_pages: no buffer offset"));
2970 for (i = 0; i < bp->b_npages; i++) {
2971 vm_page_t m = bp->b_pages[i];
2972 vm_ooffset_t noff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2973 vm_ooffset_t eoff = noff;
2974
2975 if (eoff > bp->b_offset + bp->b_bufsize)
2976 eoff = bp->b_offset + bp->b_bufsize;
2977 vfs_page_set_valid(bp, foff, i, m);
2978 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
2979 foff = noff;
2980 }
2981 }
2982}
2983
2984/*
2985 * vfs_bio_set_validclean:
2986 *
2987 * Set the range within the buffer to valid and clean. The range is
2988 * relative to the beginning of the buffer, b_offset. Note that b_offset
2989 * itself may be offset from the beginning of the first page.
2990 */
2991
2992void
2993vfs_bio_set_validclean(struct buf *bp, int base, int size)
2994{
2995 if (bp->b_flags & B_VMIO) {
2996 int i;
2997 int n;
2998
2999 /*
3000 * Fixup base to be relative to beginning of first page.
3001 * Set initial n to be the maximum number of bytes in the
3002 * first page that can be validated.
3003 */
3004
3005 base += (bp->b_offset & PAGE_MASK);
3006 n = PAGE_SIZE - (base & PAGE_MASK);
3007
3008 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
3009 vm_page_t m = bp->b_pages[i];
3010
3011 if (n > size)
3012 n = size;
3013
3014 vm_page_set_validclean(m, base & PAGE_MASK, n);
3015 base += n;
3016 size -= n;
3017 n = PAGE_SIZE;
3018 }
3019 }
3020}
3021
3022/*
3023 * vfs_bio_clrbuf:
3024 *
3025 * clear a buffer. This routine essentially fakes an I/O, so we need
3026 * to clear BIO_ERROR and B_INVAL.
3027 *
3028 * Note that while we only theoretically need to clear through b_bcount,
3029 * we go ahead and clear through b_bufsize.
3030 */
3031
3032void
3033vfs_bio_clrbuf(struct buf *bp) {
3034 int i, mask = 0;
3035 caddr_t sa, ea;
3036 if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) {
3037 bp->b_flags &= ~B_INVAL;
3038 bp->b_ioflags &= ~BIO_ERROR;
3039 if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
3040 (bp->b_offset & PAGE_MASK) == 0) {
3041 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
3042 if (((bp->b_pages[0]->flags & PG_ZERO) == 0) &&
3043 ((bp->b_pages[0]->valid & mask) != mask)) {
3044 bzero(bp->b_data, bp->b_bufsize);
3045 }
3046 bp->b_pages[0]->valid |= mask;
3047 bp->b_resid = 0;
3048 return;
3049 }
3050 ea = sa = bp->b_data;
3051 for(i=0;i<bp->b_npages;i++,sa=ea) {
3052 int j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
3053 ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE);
3054 ea = (caddr_t)(vm_offset_t)ulmin(
3055 (u_long)(vm_offset_t)ea,
3056 (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize);
3057 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
3058 if ((bp->b_pages[i]->valid & mask) == mask)
3059 continue;
3060 if ((bp->b_pages[i]->valid & mask) == 0) {
3061 if ((bp->b_pages[i]->flags & PG_ZERO) == 0) {
3062 bzero(sa, ea - sa);
3063 }
3064 } else {
3065 for (; sa < ea; sa += DEV_BSIZE, j++) {
3066 if (((bp->b_pages[i]->flags & PG_ZERO) == 0) &&
3067 (bp->b_pages[i]->valid & (1<<j)) == 0)
3068 bzero(sa, DEV_BSIZE);
3069 }
3070 }
3071 bp->b_pages[i]->valid |= mask;
3072 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
3073 }
3074 bp->b_resid = 0;
3075 } else {
3076 clrbuf(bp);
3077 }
3078}
3079
3080/*
3081 * vm_hold_load_pages and vm_hold_unload pages get pages into
3082 * a buffers address space. The pages are anonymous and are
3083 * not associated with a file object.
3084 */
3085void
3086vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3087{
3088 vm_offset_t pg;
3089 vm_page_t p;
3090 int index;
3091
3092 to = round_page(to);
3093 from = round_page(from);
3094 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3095
3096 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3097
3098tryagain:
3099
3100 p = vm_page_alloc(kernel_object,
3101 ((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
3102 VM_ALLOC_NORMAL);
3103 if (!p) {
3104 vm_pageout_deficit += (to - from) >> PAGE_SHIFT;
3105 VM_WAIT;
3106 goto tryagain;
3107 }
3108 vm_page_wire(p);
3109 p->valid = VM_PAGE_BITS_ALL;
3110 vm_page_flag_clear(p, PG_ZERO);
3111 pmap_kenter(pg, VM_PAGE_TO_PHYS(p));
3112 bp->b_pages[index] = p;
3113 vm_page_wakeup(p);
3114 }
3115 bp->b_npages = index;
3116}
3117
3118void
3119vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3120{
3121 vm_offset_t pg;
3122 vm_page_t p;
3123 int index, newnpages;
3124
3125 from = round_page(from);
3126 to = round_page(to);
3127 newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3128
3129 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3130 p = bp->b_pages[index];
3131 if (p && (index < bp->b_npages)) {
3132 if (p->busy) {
3133 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3134 bp->b_blkno, bp->b_lblkno);
3135 }
3136 bp->b_pages[index] = NULL;
3137 pmap_kremove(pg);
3138 vm_page_busy(p);
3139 vm_page_unwire(p, 0);
3140 vm_page_free(p);
3141 }
3142 }
3143 bp->b_npages = newnpages;
3144}
3145
3146
3147#include "opt_ddb.h"
3148#ifdef DDB
3149#include <ddb/ddb.h>
3150
3151DB_SHOW_COMMAND(buffer, db_show_buffer)
3152{
3153 /* get args */
3154 struct buf *bp = (struct buf *)addr;
3155
3156 if (!have_addr) {
3157 db_printf("usage: show buffer <addr>\n");
3158 return;
3159 }
3160
3161 db_printf("b_flags = 0x%b\n", (u_int)bp->b_flags, PRINT_BUF_FLAGS);
3162 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3163 "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3164 "b_blkno = %d, b_pblkno = %d\n",
3165 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
3166 major(bp->b_dev), minor(bp->b_dev),
3167 bp->b_data, bp->b_blkno, bp->b_pblkno);
3168 if (bp->b_npages) {
3169 int i;
3170 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
3171 for (i = 0; i < bp->b_npages; i++) {
3172 vm_page_t m;
3173 m = bp->b_pages[i];
3174 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
3175 (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
3176 if ((i + 1) < bp->b_npages)
3177 db_printf(",");
3178 }
3179 db_printf("\n");
3180 }
3181}
3182#endif /* DDB */
| 33#include <sys/buf.h>
34#include <sys/eventhandler.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mount.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/proc.h>
41#include <sys/reboot.h>
42#include <sys/resourcevar.h>
43#include <sys/sysctl.h>
44#include <sys/vmmeter.h>
45#include <sys/vnode.h>
46#include <vm/vm.h>
47#include <vm/vm_param.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_pageout.h>
50#include <vm/vm_page.h>
51#include <vm/vm_object.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_map.h>
54
55static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
56
57struct bio_ops bioops; /* I/O operation notification */
58
59struct buf *buf; /* buffer header pool */
60struct swqueue bswlist;
61struct simplelock buftimelock; /* Interlock on setting prio and timo */
62
63static void vm_hold_free_pages(struct buf * bp, vm_offset_t from,
64 vm_offset_t to);
65static void vm_hold_load_pages(struct buf * bp, vm_offset_t from,
66 vm_offset_t to);
67static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off,
68 int pageno, vm_page_t m);
69static void vfs_clean_pages(struct buf * bp);
70static void vfs_setdirty(struct buf *bp);
71static void vfs_vmio_release(struct buf *bp);
72static void vfs_backgroundwritedone(struct buf *bp);
73static int flushbufqueues(void);
74
75static int bd_request;
76
77static void buf_daemon __P((void));
78/*
79 * bogus page -- for I/O to/from partially complete buffers
80 * this is a temporary solution to the problem, but it is not
81 * really that bad. it would be better to split the buffer
82 * for input in the case of buffers partially already in memory,
83 * but the code is intricate enough already.
84 */
85vm_page_t bogus_page;
86int runningbufspace;
87int vmiodirenable = FALSE;
88static vm_offset_t bogus_offset;
89
90static int bufspace, maxbufspace,
91 bufmallocspace, maxbufmallocspace, lobufspace, hibufspace;
92static int bufreusecnt, bufdefragcnt, buffreekvacnt;
93static int maxbdrun;
94static int needsbuffer;
95static int numdirtybuffers, hidirtybuffers;
96static int numfreebuffers, lofreebuffers, hifreebuffers;
97static int getnewbufcalls;
98static int getnewbufrestarts;
99
100SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD,
101 &numdirtybuffers, 0, "");
102SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW,
103 &hidirtybuffers, 0, "");
104SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD,
105 &numfreebuffers, 0, "");
106SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW,
107 &lofreebuffers, 0, "");
108SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW,
109 &hifreebuffers, 0, "");
110SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD,
111 &runningbufspace, 0, "");
112SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD,
113 &maxbufspace, 0, "");
114SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD,
115 &hibufspace, 0, "");
116SYSCTL_INT(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD,
117 &lobufspace, 0, "");
118SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD,
119 &bufspace, 0, "");
120SYSCTL_INT(_vfs, OID_AUTO, maxbdrun, CTLFLAG_RW,
121 &maxbdrun, 0, "");
122SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW,
123 &maxbufmallocspace, 0, "");
124SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD,
125 &bufmallocspace, 0, "");
126SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW,
127 &getnewbufcalls, 0, "");
128SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW,
129 &getnewbufrestarts, 0, "");
130SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW,
131 &vmiodirenable, 0, "");
132SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW,
133 &bufdefragcnt, 0, "");
134SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW,
135 &buffreekvacnt, 0, "");
136SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW,
137 &bufreusecnt, 0, "");
138
139static int bufhashmask;
140static LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
141struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } };
142char *buf_wmesg = BUF_WMESG;
143
144extern int vm_swap_size;
145
146#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
147#define VFS_BIO_NEED_DIRTYFLUSH 0x02 /* waiting for dirty buffer flush */
148#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
149#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
150
151/*
152 * Buffer hash table code. Note that the logical block scans linearly, which
153 * gives us some L1 cache locality.
154 */
155
156static __inline
157struct bufhashhdr *
158bufhash(struct vnode *vnp, daddr_t bn)
159{
160 return(&bufhashtbl[(((uintptr_t)(vnp) >> 7) + (int)bn) & bufhashmask]);
161}
162
163/*
164 * numdirtywakeup:
165 *
166 * If someone is blocked due to there being too many dirty buffers,
167 * and numdirtybuffers is now reasonable, wake them up.
168 */
169
170static __inline void
171numdirtywakeup(void)
172{
173 if (numdirtybuffers < hidirtybuffers) {
174 if (needsbuffer & VFS_BIO_NEED_DIRTYFLUSH) {
175 needsbuffer &= ~VFS_BIO_NEED_DIRTYFLUSH;
176 wakeup(&needsbuffer);
177 }
178 }
179}
180
181/*
182 * bufspacewakeup:
183 *
184 * Called when buffer space is potentially available for recovery.
185 * getnewbuf() will block on this flag when it is unable to free
186 * sufficient buffer space. Buffer space becomes recoverable when
187 * bp's get placed back in the queues.
188 */
189
190static __inline void
191bufspacewakeup(void)
192{
193 /*
194 * If someone is waiting for BUF space, wake them up. Even
195 * though we haven't freed the kva space yet, the waiting
196 * process will be able to now.
197 */
198 if (needsbuffer & VFS_BIO_NEED_BUFSPACE) {
199 needsbuffer &= ~VFS_BIO_NEED_BUFSPACE;
200 wakeup(&needsbuffer);
201 }
202}
203
204/*
205 * bufcountwakeup:
206 *
207 * Called when a buffer has been added to one of the free queues to
208 * account for the buffer and to wakeup anyone waiting for free buffers.
209 * This typically occurs when large amounts of metadata are being handled
210 * by the buffer cache ( else buffer space runs out first, usually ).
211 */
212
213static __inline void
214bufcountwakeup(void)
215{
216 ++numfreebuffers;
217 if (needsbuffer) {
218 needsbuffer &= ~VFS_BIO_NEED_ANY;
219 if (numfreebuffers >= hifreebuffers)
220 needsbuffer &= ~VFS_BIO_NEED_FREE;
221 wakeup(&needsbuffer);
222 }
223}
224
225/*
226 * vfs_buf_test_cache:
227 *
228 * Called when a buffer is extended. This function clears the B_CACHE
229 * bit if the newly extended portion of the buffer does not contain
230 * valid data.
231 */
232static __inline__
233void
234vfs_buf_test_cache(struct buf *bp,
235 vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
236 vm_page_t m)
237{
238 if (bp->b_flags & B_CACHE) {
239 int base = (foff + off) & PAGE_MASK;
240 if (vm_page_is_valid(m, base, size) == 0)
241 bp->b_flags &= ~B_CACHE;
242 }
243}
244
245static __inline__
246void
247bd_wakeup(int dirtybuflevel)
248{
249 if (numdirtybuffers >= dirtybuflevel && bd_request == 0) {
250 bd_request = 1;
251 wakeup(&bd_request);
252 }
253}
254
255/*
256 * bd_speedup - speedup the buffer cache flushing code
257 */
258
259static __inline__
260void
261bd_speedup(void)
262{
263 bd_wakeup(1);
264}
265
266/*
267 * Initialize buffer headers and related structures.
268 */
269
270caddr_t
271bufhashinit(caddr_t vaddr)
272{
273 /* first, make a null hash table */
274 for (bufhashmask = 8; bufhashmask < nbuf / 4; bufhashmask <<= 1)
275 ;
276 bufhashtbl = (void *)vaddr;
277 vaddr = vaddr + sizeof(*bufhashtbl) * bufhashmask;
278 --bufhashmask;
279 return(vaddr);
280}
281
282void
283bufinit(void)
284{
285 struct buf *bp;
286 int i;
287
288 TAILQ_INIT(&bswlist);
289 LIST_INIT(&invalhash);
290 simple_lock_init(&buftimelock);
291
292 for (i = 0; i <= bufhashmask; i++)
293 LIST_INIT(&bufhashtbl[i]);
294
295 /* next, make a null set of free lists */
296 for (i = 0; i < BUFFER_QUEUES; i++)
297 TAILQ_INIT(&bufqueues[i]);
298
299 /* finally, initialize each buffer header and stick on empty q */
300 for (i = 0; i < nbuf; i++) {
301 bp = &buf[i];
302 bzero(bp, sizeof *bp);
303 bp->b_flags = B_INVAL; /* we're just an empty header */
304 bp->b_dev = NODEV;
305 bp->b_rcred = NOCRED;
306 bp->b_wcred = NOCRED;
307 bp->b_qindex = QUEUE_EMPTY;
308 bp->b_xflags = 0;
309 LIST_INIT(&bp->b_dep);
310 BUF_LOCKINIT(bp);
311 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
312 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
313 }
314
315 /*
316 * maxbufspace is the absolute maximum amount of buffer space we are
317 * allowed to reserve in KVM and in real terms. The absolute maximum
318 * is nominally used by buf_daemon. hibufspace is the nominal maximum
319 * used by most other processes. The differential is required to
320 * ensure that buf_daemon is able to run when other processes might
321 * be blocked waiting for buffer space.
322 *
323 * maxbufspace is based on BKVASIZE. Allocating buffers larger then
324 * this may result in KVM fragmentation which is not handled optimally
325 * by the system.
326 */
327 maxbufspace = nbuf * BKVASIZE;
328 hibufspace = imax(3 * maxbufspace / 4, maxbufspace - MAXBSIZE * 10);
329 lobufspace = hibufspace - MAXBSIZE;
330
331/*
332 * Limit the amount of malloc memory since it is wired permanently into
333 * the kernel space. Even though this is accounted for in the buffer
334 * allocation, we don't want the malloced region to grow uncontrolled.
335 * The malloc scheme improves memory utilization significantly on average
336 * (small) directories.
337 */
338 maxbufmallocspace = hibufspace / 20;
339
340/*
341 * Reduce the chance of a deadlock occuring by limiting the number
342 * of delayed-write dirty buffers we allow to stack up.
343 */
344 hidirtybuffers = nbuf / 4 + 20;
345 numdirtybuffers = 0;
346/*
347 * To support extreme low-memory systems, make sure hidirtybuffers cannot
348 * eat up all available buffer space. This occurs when our minimum cannot
349 * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
350 * BKVASIZE'd (8K) buffers.
351 */
352 while (hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
353 hidirtybuffers >>= 1;
354 }
355
356/*
357 * Try to keep the number of free buffers in the specified range,
358 * and give special processes (e.g. like buf_daemon) access to an
359 * emergency reserve.
360 */
361 lofreebuffers = nbuf / 18 + 5;
362 hifreebuffers = 2 * lofreebuffers;
363 numfreebuffers = nbuf;
364
365/*
366 * Maximum number of async ops initiated per buf_daemon loop. This is
367 * somewhat of a hack at the moment, we really need to limit ourselves
368 * based on the number of bytes of I/O in-transit that were initiated
369 * from buf_daemon.
370 */
371 if ((maxbdrun = nswbuf / 4) < 4)
372 maxbdrun = 4;
373
374 bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
375 bogus_page = vm_page_alloc(kernel_object,
376 ((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
377 VM_ALLOC_NORMAL);
378 cnt.v_wire_count++;
379
380}
381
382/*
383 * bfreekva() - free the kva allocation for a buffer.
384 *
385 * Must be called at splbio() or higher as this is the only locking for
386 * buffer_map.
387 *
388 * Since this call frees up buffer space, we call bufspacewakeup().
389 */
390static void
391bfreekva(struct buf * bp)
392{
393 if (bp->b_kvasize) {
394 ++buffreekvacnt;
395 bufspace -= bp->b_kvasize;
396 vm_map_delete(buffer_map,
397 (vm_offset_t) bp->b_kvabase,
398 (vm_offset_t) bp->b_kvabase + bp->b_kvasize
399 );
400 bp->b_kvasize = 0;
401 bufspacewakeup();
402 }
403}
404
405/*
406 * bremfree:
407 *
408 * Remove the buffer from the appropriate free list.
409 */
410void
411bremfree(struct buf * bp)
412{
413 int s = splbio();
414 int old_qindex = bp->b_qindex;
415
416 if (bp->b_qindex != QUEUE_NONE) {
417 KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
418 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
419 bp->b_qindex = QUEUE_NONE;
420 runningbufspace += bp->b_bufsize;
421 } else {
422 if (BUF_REFCNT(bp) <= 1)
423 panic("bremfree: removing a buffer not on a queue");
424 }
425
426 /*
427 * Fixup numfreebuffers count. If the buffer is invalid or not
428 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
429 * the buffer was free and we must decrement numfreebuffers.
430 */
431 if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
432 switch(old_qindex) {
433 case QUEUE_DIRTY:
434 case QUEUE_CLEAN:
435 case QUEUE_EMPTY:
436 case QUEUE_EMPTYKVA:
437 --numfreebuffers;
438 break;
439 default:
440 break;
441 }
442 }
443 splx(s);
444}
445
446
447/*
448 * Get a buffer with the specified data. Look in the cache first. We
449 * must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
450 * is set, the buffer is valid and we do not have to do anything ( see
451 * getblk() ).
452 */
453int
454bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred,
455 struct buf ** bpp)
456{
457 struct buf *bp;
458
459 bp = getblk(vp, blkno, size, 0, 0);
460 *bpp = bp;
461
462 /* if not found in cache, do some I/O */
463 if ((bp->b_flags & B_CACHE) == 0) {
464 if (curproc != NULL)
465 curproc->p_stats->p_ru.ru_inblock++;
466 KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp));
467 bp->b_iocmd = BIO_READ;
468 bp->b_flags &= ~B_INVAL;
469 bp->b_ioflags &= ~BIO_ERROR;
470 if (bp->b_rcred == NOCRED) {
471 if (cred != NOCRED)
472 crhold(cred);
473 bp->b_rcred = cred;
474 }
475 vfs_busy_pages(bp, 0);
476 VOP_STRATEGY(vp, bp);
477 return (bufwait(bp));
478 }
479 return (0);
480}
481
482/*
483 * Operates like bread, but also starts asynchronous I/O on
484 * read-ahead blocks. We must clear BIO_ERROR and B_INVAL prior
485 * to initiating I/O . If B_CACHE is set, the buffer is valid
486 * and we do not have to do anything.
487 */
488int
489breadn(struct vnode * vp, daddr_t blkno, int size,
490 daddr_t * rablkno, int *rabsize,
491 int cnt, struct ucred * cred, struct buf ** bpp)
492{
493 struct buf *bp, *rabp;
494 int i;
495 int rv = 0, readwait = 0;
496
497 *bpp = bp = getblk(vp, blkno, size, 0, 0);
498
499 /* if not found in cache, do some I/O */
500 if ((bp->b_flags & B_CACHE) == 0) {
501 if (curproc != NULL)
502 curproc->p_stats->p_ru.ru_inblock++;
503 bp->b_iocmd = BIO_READ;
504 bp->b_flags &= ~B_INVAL;
505 bp->b_ioflags &= ~BIO_ERROR;
506 if (bp->b_rcred == NOCRED) {
507 if (cred != NOCRED)
508 crhold(cred);
509 bp->b_rcred = cred;
510 }
511 vfs_busy_pages(bp, 0);
512 VOP_STRATEGY(vp, bp);
513 ++readwait;
514 }
515
516 for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
517 if (inmem(vp, *rablkno))
518 continue;
519 rabp = getblk(vp, *rablkno, *rabsize, 0, 0);
520
521 if ((rabp->b_flags & B_CACHE) == 0) {
522 if (curproc != NULL)
523 curproc->p_stats->p_ru.ru_inblock++;
524 rabp->b_flags |= B_ASYNC;
525 rabp->b_flags &= ~B_INVAL;
526 rabp->b_ioflags &= ~BIO_ERROR;
527 rabp->b_iocmd = BIO_READ;
528 if (rabp->b_rcred == NOCRED) {
529 if (cred != NOCRED)
530 crhold(cred);
531 rabp->b_rcred = cred;
532 }
533 vfs_busy_pages(rabp, 0);
534 BUF_KERNPROC(rabp);
535 VOP_STRATEGY(vp, rabp);
536 } else {
537 brelse(rabp);
538 }
539 }
540
541 if (readwait) {
542 rv = bufwait(bp);
543 }
544 return (rv);
545}
546
547/*
548 * Write, release buffer on completion. (Done by iodone
549 * if async). Do not bother writing anything if the buffer
550 * is invalid.
551 *
552 * Note that we set B_CACHE here, indicating that buffer is
553 * fully valid and thus cacheable. This is true even of NFS
554 * now so we set it generally. This could be set either here
555 * or in biodone() since the I/O is synchronous. We put it
556 * here.
557 */
558int
559bwrite(struct buf * bp)
560{
561 int oldflags, s;
562 struct buf *newbp;
563
564 if (bp->b_flags & B_INVAL) {
565 brelse(bp);
566 return (0);
567 }
568
569 oldflags = bp->b_flags;
570
571 if (BUF_REFCNT(bp) == 0)
572 panic("bwrite: buffer is not busy???");
573 s = splbio();
574 /*
575 * If a background write is already in progress, delay
576 * writing this block if it is asynchronous. Otherwise
577 * wait for the background write to complete.
578 */
579 if (bp->b_xflags & BX_BKGRDINPROG) {
580 if (bp->b_flags & B_ASYNC) {
581 splx(s);
582 bdwrite(bp);
583 return (0);
584 }
585 bp->b_xflags |= BX_BKGRDWAIT;
586 tsleep(&bp->b_xflags, PRIBIO, "biord", 0);
587 if (bp->b_xflags & BX_BKGRDINPROG)
588 panic("bwrite: still writing");
589 }
590
591 /* Mark the buffer clean */
592 bundirty(bp);
593
594 /*
595 * If this buffer is marked for background writing and we
596 * do not have to wait for it, make a copy and write the
597 * copy so as to leave this buffer ready for further use.
598 */
599 if ((bp->b_xflags & BX_BKGRDWRITE) && (bp->b_flags & B_ASYNC)) {
600 if (bp->b_iodone != NULL) {
601 printf("bp->b_iodone = %p\n", bp->b_iodone);
602 panic("bwrite: need chained iodone");
603 }
604
605 /* get a new block */
606 newbp = geteblk(bp->b_bufsize);
607
608 /* set it to be identical to the old block */
609 memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
610 bgetvp(bp->b_vp, newbp);
611 newbp->b_lblkno = bp->b_lblkno;
612 newbp->b_blkno = bp->b_blkno;
613 newbp->b_offset = bp->b_offset;
614 newbp->b_iodone = vfs_backgroundwritedone;
615 newbp->b_flags |= B_ASYNC;
616 newbp->b_flags &= ~B_INVAL;
617
618 /* move over the dependencies */
619 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
620 (*bioops.io_movedeps)(bp, newbp);
621
622 /*
623 * Initiate write on the copy, release the original to
624 * the B_LOCKED queue so that it cannot go away until
625 * the background write completes. If not locked it could go
626 * away and then be reconstituted while it was being written.
627 * If the reconstituted buffer were written, we could end up
628 * with two background copies being written at the same time.
629 */
630 bp->b_xflags |= BX_BKGRDINPROG;
631 bp->b_flags |= B_LOCKED;
632 bqrelse(bp);
633 bp = newbp;
634 }
635
636 bp->b_flags &= ~B_DONE;
637 bp->b_ioflags &= ~BIO_ERROR;
638 bp->b_flags |= B_WRITEINPROG | B_CACHE;
639 bp->b_iocmd = BIO_WRITE;
640
641 bp->b_vp->v_numoutput++;
642 vfs_busy_pages(bp, 1);
643 if (curproc != NULL)
644 curproc->p_stats->p_ru.ru_oublock++;
645 splx(s);
646 if (oldflags & B_ASYNC)
647 BUF_KERNPROC(bp);
648 BUF_STRATEGY(bp);
649
650 if ((oldflags & B_ASYNC) == 0) {
651 int rtval = bufwait(bp);
652 brelse(bp);
653 return (rtval);
654 }
655
656 return (0);
657}
658
659/*
660 * Complete a background write started from bwrite.
661 */
662static void
663vfs_backgroundwritedone(bp)
664 struct buf *bp;
665{
666 struct buf *origbp;
667
668 /*
669 * Find the original buffer that we are writing.
670 */
671 if ((origbp = gbincore(bp->b_vp, bp->b_lblkno)) == NULL)
672 panic("backgroundwritedone: lost buffer");
673 /*
674 * Process dependencies then return any unfinished ones.
675 */
676 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
677 (*bioops.io_complete)(bp);
678 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
679 (*bioops.io_movedeps)(bp, origbp);
680 /*
681 * Clear the BX_BKGRDINPROG flag in the original buffer
682 * and awaken it if it is waiting for the write to complete.
683 */
684 origbp->b_xflags &= ~BX_BKGRDINPROG;
685 if (origbp->b_xflags & BX_BKGRDWAIT) {
686 origbp->b_xflags &= ~BX_BKGRDWAIT;
687 wakeup(&origbp->b_xflags);
688 }
689 /*
690 * Clear the B_LOCKED flag and remove it from the locked
691 * queue if it currently resides there.
692 */
693 origbp->b_flags &= ~B_LOCKED;
694 if (BUF_LOCK(origbp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
695 bremfree(origbp);
696 bqrelse(origbp);
697 }
698 /*
699 * This buffer is marked B_NOCACHE, so when it is released
700 * by biodone, it will be tossed. We mark it with BIO_READ
701 * to avoid biodone doing a second vwakeup.
702 */
703 bp->b_flags |= B_NOCACHE;
704 bp->b_iocmd = BIO_READ;
705 bp->b_flags &= ~(B_CACHE | B_DONE);
706 bp->b_iodone = 0;
707 bufdone(bp);
708}
709
710/*
711 * Delayed write. (Buffer is marked dirty). Do not bother writing
712 * anything if the buffer is marked invalid.
713 *
714 * Note that since the buffer must be completely valid, we can safely
715 * set B_CACHE. In fact, we have to set B_CACHE here rather then in
716 * biodone() in order to prevent getblk from writing the buffer
717 * out synchronously.
718 */
719void
720bdwrite(struct buf * bp)
721{
722 if (BUF_REFCNT(bp) == 0)
723 panic("bdwrite: buffer is not busy");
724
725 if (bp->b_flags & B_INVAL) {
726 brelse(bp);
727 return;
728 }
729 bdirty(bp);
730
731 /*
732 * Set B_CACHE, indicating that the buffer is fully valid. This is
733 * true even of NFS now.
734 */
735 bp->b_flags |= B_CACHE;
736
737 /*
738 * This bmap keeps the system from needing to do the bmap later,
739 * perhaps when the system is attempting to do a sync. Since it
740 * is likely that the indirect block -- or whatever other datastructure
741 * that the filesystem needs is still in memory now, it is a good
742 * thing to do this. Note also, that if the pageout daemon is
743 * requesting a sync -- there might not be enough memory to do
744 * the bmap then... So, this is important to do.
745 */
746 if (bp->b_lblkno == bp->b_blkno) {
747 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
748 }
749
750 /*
751 * Set the *dirty* buffer range based upon the VM system dirty pages.
752 */
753 vfs_setdirty(bp);
754
755 /*
756 * We need to do this here to satisfy the vnode_pager and the
757 * pageout daemon, so that it thinks that the pages have been
758 * "cleaned". Note that since the pages are in a delayed write
759 * buffer -- the VFS layer "will" see that the pages get written
760 * out on the next sync, or perhaps the cluster will be completed.
761 */
762 vfs_clean_pages(bp);
763 bqrelse(bp);
764
765 /*
766 * Wakeup the buffer flushing daemon if we have saturated the
767 * buffer cache.
768 */
769
770 bd_wakeup(hidirtybuffers);
771
772 /*
773 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
774 * due to the softdep code.
775 */
776}
777
778/*
779 * bdirty:
780 *
781 * Turn buffer into delayed write request. We must clear BIO_READ and
782 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
783 * itself to properly update it in the dirty/clean lists. We mark it
784 * B_DONE to ensure that any asynchronization of the buffer properly
785 * clears B_DONE ( else a panic will occur later ).
786 *
787 * bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
788 * might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
789 * should only be called if the buffer is known-good.
790 *
791 * Since the buffer is not on a queue, we do not update the numfreebuffers
792 * count.
793 *
794 * Must be called at splbio().
795 * The buffer must be on QUEUE_NONE.
796 */
797void
798bdirty(bp)
799 struct buf *bp;
800{
801 KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
802 bp->b_flags &= ~(B_RELBUF);
803 bp->b_iocmd = BIO_WRITE;
804
805 if ((bp->b_flags & B_DELWRI) == 0) {
806 bp->b_flags |= B_DONE | B_DELWRI;
807 reassignbuf(bp, bp->b_vp);
808 ++numdirtybuffers;
809 bd_wakeup(hidirtybuffers);
810 }
811}
812
813/*
814 * bundirty:
815 *
816 * Clear B_DELWRI for buffer.
817 *
818 * Since the buffer is not on a queue, we do not update the numfreebuffers
819 * count.
820 *
821 * Must be called at splbio().
822 * The buffer must be on QUEUE_NONE.
823 */
824
825void
826bundirty(bp)
827 struct buf *bp;
828{
829 KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
830
831 if (bp->b_flags & B_DELWRI) {
832 bp->b_flags &= ~B_DELWRI;
833 reassignbuf(bp, bp->b_vp);
834 --numdirtybuffers;
835 numdirtywakeup();
836 }
837 /*
838 * Since it is now being written, we can clear its deferred write flag.
839 */
840 bp->b_flags &= ~B_DEFERRED;
841}
842
843/*
844 * bawrite:
845 *
846 * Asynchronous write. Start output on a buffer, but do not wait for
847 * it to complete. The buffer is released when the output completes.
848 *
849 * bwrite() ( or the VOP routine anyway ) is responsible for handling
850 * B_INVAL buffers. Not us.
851 */
852void
853bawrite(struct buf * bp)
854{
855 bp->b_flags |= B_ASYNC;
856 (void) BUF_WRITE(bp);
857}
858
859/*
860 * bowrite:
861 *
862 * Ordered write. Start output on a buffer, and flag it so that the
863 * device will write it in the order it was queued. The buffer is
864 * released when the output completes. bwrite() ( or the VOP routine
865 * anyway ) is responsible for handling B_INVAL buffers.
866 */
867int
868bowrite(struct buf * bp)
869{
870 bp->b_ioflags |= BIO_ORDERED;
871 bp->b_flags |= B_ASYNC;
872 return (BUF_WRITE(bp));
873}
874
875/*
876 * bwillwrite:
877 *
878 * Called prior to the locking of any vnodes when we are expecting to
879 * write. We do not want to starve the buffer cache with too many
880 * dirty buffers so we block here. By blocking prior to the locking
881 * of any vnodes we attempt to avoid the situation where a locked vnode
882 * prevents the various system daemons from flushing related buffers.
883 */
884
885void
886bwillwrite(void)
887{
888 int slop = hidirtybuffers / 10;
889
890 if (numdirtybuffers > hidirtybuffers + slop) {
891 int s;
892
893 s = splbio();
894 while (numdirtybuffers > hidirtybuffers) {
895 bd_wakeup(hidirtybuffers);
896 needsbuffer |= VFS_BIO_NEED_DIRTYFLUSH;
897 tsleep(&needsbuffer, (PRIBIO + 4), "flswai", 0);
898 }
899 splx(s);
900 }
901}
902
903/*
904 * brelse:
905 *
906 * Release a busy buffer and, if requested, free its resources. The
907 * buffer will be stashed in the appropriate bufqueue[] allowing it
908 * to be accessed later as a cache entity or reused for other purposes.
909 */
910void
911brelse(struct buf * bp)
912{
913 int s;
914
915 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
916
917 s = splbio();
918
919 if (bp->b_flags & B_LOCKED)
920 bp->b_ioflags &= ~BIO_ERROR;
921
922 if (bp->b_iocmd == BIO_WRITE &&
923 (bp->b_ioflags & BIO_ERROR) &&
924 !(bp->b_flags & B_INVAL)) {
925 /*
926 * Failed write, redirty. Must clear BIO_ERROR to prevent
927 * pages from being scrapped. If B_INVAL is set then
928 * this case is not run and the next case is run to
929 * destroy the buffer. B_INVAL can occur if the buffer
930 * is outside the range supported by the underlying device.
931 */
932 bp->b_ioflags &= ~BIO_ERROR;
933 bdirty(bp);
934 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
935 (bp->b_ioflags & BIO_ERROR) ||
936 bp->b_iocmd == BIO_DELETE || (bp->b_bufsize <= 0)) {
937 /*
938 * Either a failed I/O or we were asked to free or not
939 * cache the buffer.
940 */
941 bp->b_flags |= B_INVAL;
942 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
943 (*bioops.io_deallocate)(bp);
944 if (bp->b_flags & B_DELWRI) {
945 --numdirtybuffers;
946 numdirtywakeup();
947 }
948 bp->b_flags &= ~(B_DELWRI | B_CACHE);
949 if ((bp->b_flags & B_VMIO) == 0) {
950 if (bp->b_bufsize)
951 allocbuf(bp, 0);
952 if (bp->b_vp)
953 brelvp(bp);
954 }
955 }
956
957 /*
958 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release()
959 * is called with B_DELWRI set, the underlying pages may wind up
960 * getting freed causing a previous write (bdwrite()) to get 'lost'
961 * because pages associated with a B_DELWRI bp are marked clean.
962 *
963 * We still allow the B_INVAL case to call vfs_vmio_release(), even
964 * if B_DELWRI is set.
965 */
966
967 if (bp->b_flags & B_DELWRI)
968 bp->b_flags &= ~B_RELBUF;
969
970 /*
971 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
972 * constituted, not even NFS buffers now. Two flags effect this. If
973 * B_INVAL, the struct buf is invalidated but the VM object is kept
974 * around ( i.e. so it is trivial to reconstitute the buffer later ).
975 *
976 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
977 * invalidated. BIO_ERROR cannot be set for a failed write unless the
978 * buffer is also B_INVAL because it hits the re-dirtying code above.
979 *
980 * Normally we can do this whether a buffer is B_DELWRI or not. If
981 * the buffer is an NFS buffer, it is tracking piecemeal writes or
982 * the commit state and we cannot afford to lose the buffer. If the
983 * buffer has a background write in progress, we need to keep it
984 * around to prevent it from being reconstituted and starting a second
985 * background write.
986 */
987 if ((bp->b_flags & B_VMIO)
988 && !(bp->b_vp->v_tag == VT_NFS &&
989 !vn_isdisk(bp->b_vp, NULL) &&
990 (bp->b_flags & B_DELWRI) &&
991 (bp->b_xflags & BX_BKGRDINPROG))
992 ) {
993
994 int i, j, resid;
995 vm_page_t m;
996 off_t foff;
997 vm_pindex_t poff;
998 vm_object_t obj;
999 struct vnode *vp;
1000
1001 vp = bp->b_vp;
1002
1003 /*
1004 * Get the base offset and length of the buffer. Note that
1005 * for block sizes that are less then PAGE_SIZE, the b_data
1006 * base of the buffer does not represent exactly b_offset and
1007 * neither b_offset nor b_size are necessarily page aligned.
1008 * Instead, the starting position of b_offset is:
1009 *
1010 * b_data + (b_offset & PAGE_MASK)
1011 *
1012 * block sizes less then DEV_BSIZE (usually 512) are not
1013 * supported due to the page granularity bits (m->valid,
1014 * m->dirty, etc...).
1015 *
1016 * See man buf(9) for more information
1017 */
1018
1019 resid = bp->b_bufsize;
1020 foff = bp->b_offset;
1021
1022 for (i = 0; i < bp->b_npages; i++) {
1023 m = bp->b_pages[i];
1024 vm_page_flag_clear(m, PG_ZERO);
1025 if (m == bogus_page) {
1026
1027 obj = (vm_object_t) vp->v_object;
1028 poff = OFF_TO_IDX(bp->b_offset);
1029
1030 for (j = i; j < bp->b_npages; j++) {
1031 m = bp->b_pages[j];
1032 if (m == bogus_page) {
1033 m = vm_page_lookup(obj, poff + j);
1034 if (!m) {
1035 panic("brelse: page missing\n");
1036 }
1037 bp->b_pages[j] = m;
1038 }
1039 }
1040
1041 if ((bp->b_flags & B_INVAL) == 0) {
1042 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
1043 }
1044 }
1045 if ((bp->b_flags & B_NOCACHE) || (bp->b_ioflags & BIO_ERROR)) {
1046 int poffset = foff & PAGE_MASK;
1047 int presid = resid > (PAGE_SIZE - poffset) ?
1048 (PAGE_SIZE - poffset) : resid;
1049
1050 KASSERT(presid >= 0, ("brelse: extra page"));
1051 vm_page_set_invalid(m, poffset, presid);
1052 }
1053 resid -= PAGE_SIZE - (foff & PAGE_MASK);
1054 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
1055 }
1056
1057 if (bp->b_flags & (B_INVAL | B_RELBUF))
1058 vfs_vmio_release(bp);
1059
1060 } else if (bp->b_flags & B_VMIO) {
1061
1062 if (bp->b_flags & (B_INVAL | B_RELBUF))
1063 vfs_vmio_release(bp);
1064
1065 }
1066
1067 if (bp->b_qindex != QUEUE_NONE)
1068 panic("brelse: free buffer onto another queue???");
1069 if (BUF_REFCNT(bp) > 1) {
1070 /* Temporary panic to verify exclusive locking */
1071 /* This panic goes away when we allow shared refs */
1072 panic("brelse: multiple refs");
1073 /* do not release to free list */
1074 BUF_UNLOCK(bp);
1075 splx(s);
1076 return;
1077 }
1078
1079 /* enqueue */
1080
1081 /* buffers with no memory */
1082 if (bp->b_bufsize == 0) {
1083 bp->b_flags |= B_INVAL;
1084 bp->b_xflags &= ~BX_BKGRDWRITE;
1085 if (bp->b_xflags & BX_BKGRDINPROG)
1086 panic("losing buffer 1");
1087 if (bp->b_kvasize) {
1088 bp->b_qindex = QUEUE_EMPTYKVA;
1089 } else {
1090 bp->b_qindex = QUEUE_EMPTY;
1091 }
1092 TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1093 LIST_REMOVE(bp, b_hash);
1094 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1095 bp->b_dev = NODEV;
1096 /* buffers with junk contents */
1097 } else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) || (bp->b_ioflags & BIO_ERROR)) {
1098 bp->b_flags |= B_INVAL;
1099 bp->b_xflags &= ~BX_BKGRDWRITE;
1100 if (bp->b_xflags & BX_BKGRDINPROG)
1101 panic("losing buffer 2");
1102 bp->b_qindex = QUEUE_CLEAN;
1103 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1104 LIST_REMOVE(bp, b_hash);
1105 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1106 bp->b_dev = NODEV;
1107
1108 /* buffers that are locked */
1109 } else if (bp->b_flags & B_LOCKED) {
1110 bp->b_qindex = QUEUE_LOCKED;
1111 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1112
1113 /* remaining buffers */
1114 } else {
1115 switch(bp->b_flags & (B_DELWRI|B_AGE)) {
1116 case B_DELWRI | B_AGE:
1117 bp->b_qindex = QUEUE_DIRTY;
1118 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1119 break;
1120 case B_DELWRI:
1121 bp->b_qindex = QUEUE_DIRTY;
1122 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1123 break;
1124 case B_AGE:
1125 bp->b_qindex = QUEUE_CLEAN;
1126 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1127 break;
1128 default:
1129 bp->b_qindex = QUEUE_CLEAN;
1130 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1131 break;
1132 }
1133 }
1134
1135 /*
1136 * If B_INVAL, clear B_DELWRI. We've already placed the buffer
1137 * on the correct queue.
1138 */
1139 if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) {
1140 bp->b_flags &= ~B_DELWRI;
1141 --numdirtybuffers;
1142 numdirtywakeup();
1143 }
1144
1145 runningbufspace -= bp->b_bufsize;
1146
1147 /*
1148 * Fixup numfreebuffers count. The bp is on an appropriate queue
1149 * unless locked. We then bump numfreebuffers if it is not B_DELWRI.
1150 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1151 * if B_INVAL is set ).
1152 */
1153
1154 if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI))
1155 bufcountwakeup();
1156
1157 /*
1158 * Something we can maybe free.
1159 */
1160
1161 if (bp->b_bufsize || bp->b_kvasize)
1162 bufspacewakeup();
1163
1164 /* unlock */
1165 BUF_UNLOCK(bp);
1166 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1167 bp->b_ioflags &= ~BIO_ORDERED;
1168 splx(s);
1169}
1170
1171/*
1172 * Release a buffer back to the appropriate queue but do not try to free
1173 * it.
1174 *
1175 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1176 * biodone() to requeue an async I/O on completion. It is also used when
1177 * known good buffers need to be requeued but we think we may need the data
1178 * again soon.
1179 */
1180void
1181bqrelse(struct buf * bp)
1182{
1183 int s;
1184
1185 s = splbio();
1186
1187 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1188
1189 if (bp->b_qindex != QUEUE_NONE)
1190 panic("bqrelse: free buffer onto another queue???");
1191 if (BUF_REFCNT(bp) > 1) {
1192 /* do not release to free list */
1193 panic("bqrelse: multiple refs");
1194 BUF_UNLOCK(bp);
1195 splx(s);
1196 return;
1197 }
1198 if (bp->b_flags & B_LOCKED) {
1199 bp->b_ioflags &= ~BIO_ERROR;
1200 bp->b_qindex = QUEUE_LOCKED;
1201 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1202 /* buffers with stale but valid contents */
1203 } else if (bp->b_flags & B_DELWRI) {
1204 bp->b_qindex = QUEUE_DIRTY;
1205 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1206 } else {
1207 bp->b_qindex = QUEUE_CLEAN;
1208 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1209 }
1210
1211 runningbufspace -= bp->b_bufsize;
1212
1213 if ((bp->b_flags & B_LOCKED) == 0 &&
1214 ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))) {
1215 bufcountwakeup();
1216 }
1217
1218 /*
1219 * Something we can maybe wakeup
1220 */
1221 if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1222 bufspacewakeup();
1223
1224 /* unlock */
1225 BUF_UNLOCK(bp);
1226 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1227 bp->b_ioflags &= ~BIO_ORDERED;
1228 splx(s);
1229}
1230
1231static void
1232vfs_vmio_release(bp)
1233 struct buf *bp;
1234{
1235 int i, s;
1236 vm_page_t m;
1237
1238 s = splvm();
1239 for (i = 0; i < bp->b_npages; i++) {
1240 m = bp->b_pages[i];
1241 bp->b_pages[i] = NULL;
1242 /*
1243 * In order to keep page LRU ordering consistent, put
1244 * everything on the inactive queue.
1245 */
1246 vm_page_unwire(m, 0);
1247 /*
1248 * We don't mess with busy pages, it is
1249 * the responsibility of the process that
1250 * busied the pages to deal with them.
1251 */
1252 if ((m->flags & PG_BUSY) || (m->busy != 0))
1253 continue;
1254
1255 if (m->wire_count == 0) {
1256 vm_page_flag_clear(m, PG_ZERO);
1257 /*
1258 * Might as well free the page if we can and it has
1259 * no valid data.
1260 */
1261 if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) {
1262 vm_page_busy(m);
1263 vm_page_protect(m, VM_PROT_NONE);
1264 vm_page_free(m);
1265 }
1266 }
1267 }
1268 runningbufspace -= bp->b_bufsize;
1269 splx(s);
1270 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
1271 if (bp->b_bufsize)
1272 bufspacewakeup();
1273 bp->b_npages = 0;
1274 bp->b_bufsize = 0;
1275 bp->b_flags &= ~B_VMIO;
1276 if (bp->b_vp)
1277 brelvp(bp);
1278}
1279
1280/*
1281 * Check to see if a block is currently memory resident.
1282 */
1283struct buf *
1284gbincore(struct vnode * vp, daddr_t blkno)
1285{
1286 struct buf *bp;
1287 struct bufhashhdr *bh;
1288
1289 bh = bufhash(vp, blkno);
1290
1291 /* Search hash chain */
1292 LIST_FOREACH(bp, bh, b_hash) {
1293 /* hit */
1294 if (bp->b_vp == vp && bp->b_lblkno == blkno &&
1295 (bp->b_flags & B_INVAL) == 0) {
1296 break;
1297 }
1298 }
1299 return (bp);
1300}
1301
1302/*
1303 * vfs_bio_awrite:
1304 *
1305 * Implement clustered async writes for clearing out B_DELWRI buffers.
1306 * This is much better then the old way of writing only one buffer at
1307 * a time. Note that we may not be presented with the buffers in the
1308 * correct order, so we search for the cluster in both directions.
1309 */
1310int
1311vfs_bio_awrite(struct buf * bp)
1312{
1313 int i;
1314 int j;
1315 daddr_t lblkno = bp->b_lblkno;
1316 struct vnode *vp = bp->b_vp;
1317 int s;
1318 int ncl;
1319 struct buf *bpa;
1320 int nwritten;
1321 int size;
1322 int maxcl;
1323
1324 s = splbio();
1325 /*
1326 * right now we support clustered writing only to regular files. If
1327 * we find a clusterable block we could be in the middle of a cluster
1328 * rather then at the beginning.
1329 */
1330 if ((vp->v_type == VREG) &&
1331 (vp->v_mount != 0) && /* Only on nodes that have the size info */
1332 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
1333
1334 size = vp->v_mount->mnt_stat.f_iosize;
1335 maxcl = MAXPHYS / size;
1336
1337 for (i = 1; i < maxcl; i++) {
1338 if ((bpa = gbincore(vp, lblkno + i)) &&
1339 BUF_REFCNT(bpa) == 0 &&
1340 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1341 (B_DELWRI | B_CLUSTEROK)) &&
1342 (bpa->b_bufsize == size)) {
1343 if ((bpa->b_blkno == bpa->b_lblkno) ||
1344 (bpa->b_blkno !=
1345 bp->b_blkno + ((i * size) >> DEV_BSHIFT)))
1346 break;
1347 } else {
1348 break;
1349 }
1350 }
1351 for (j = 1; i + j <= maxcl && j <= lblkno; j++) {
1352 if ((bpa = gbincore(vp, lblkno - j)) &&
1353 BUF_REFCNT(bpa) == 0 &&
1354 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1355 (B_DELWRI | B_CLUSTEROK)) &&
1356 (bpa->b_bufsize == size)) {
1357 if ((bpa->b_blkno == bpa->b_lblkno) ||
1358 (bpa->b_blkno !=
1359 bp->b_blkno - ((j * size) >> DEV_BSHIFT)))
1360 break;
1361 } else {
1362 break;
1363 }
1364 }
1365 --j;
1366 ncl = i + j;
1367 /*
1368 * this is a possible cluster write
1369 */
1370 if (ncl != 1) {
1371 nwritten = cluster_wbuild(vp, size, lblkno - j, ncl);
1372 splx(s);
1373 return nwritten;
1374 }
1375 }
1376
1377 BUF_LOCK(bp, LK_EXCLUSIVE);
1378 bremfree(bp);
1379 bp->b_flags |= B_ASYNC;
1380
1381 splx(s);
1382 /*
1383 * default (old) behavior, writing out only one block
1384 *
1385 * XXX returns b_bufsize instead of b_bcount for nwritten?
1386 */
1387 nwritten = bp->b_bufsize;
1388 (void) BUF_WRITE(bp);
1389
1390 return nwritten;
1391}
1392
1393/*
1394 * getnewbuf:
1395 *
1396 * Find and initialize a new buffer header, freeing up existing buffers
1397 * in the bufqueues as necessary. The new buffer is returned locked.
1398 *
1399 * Important: B_INVAL is not set. If the caller wishes to throw the
1400 * buffer away, the caller must set B_INVAL prior to calling brelse().
1401 *
1402 * We block if:
1403 * We have insufficient buffer headers
1404 * We have insufficient buffer space
1405 * buffer_map is too fragmented ( space reservation fails )
1406 * If we have to flush dirty buffers ( but we try to avoid this )
1407 *
1408 * To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1409 * Instead we ask the buf daemon to do it for us. We attempt to
1410 * avoid piecemeal wakeups of the pageout daemon.
1411 */
1412
1413static struct buf *
1414getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
1415{
1416 struct buf *bp;
1417 struct buf *nbp;
1418 int defrag = 0;
1419 int nqindex;
1420 int isspecial;
1421 static int flushingbufs;
1422
1423 if (curproc && (curproc->p_flag & P_BUFEXHAUST) == 0)
1424 isspecial = 0;
1425 else
1426 isspecial = 1;
1427
1428 ++getnewbufcalls;
1429 --getnewbufrestarts;
1430restart:
1431 ++getnewbufrestarts;
1432
1433 /*
1434 * Setup for scan. If we do not have enough free buffers,
1435 * we setup a degenerate case that immediately fails. Note
1436 * that if we are specially marked process, we are allowed to
1437 * dip into our reserves.
1438 *
1439 * The scanning sequence is nominally: EMPTY->EMPTYKVA->CLEAN
1440 *
1441 * We start with EMPTYKVA. If the list is empty we backup to EMPTY.
1442 * However, there are a number of cases (defragging, reusing, ...)
1443 * where we cannot backup.
1444 */
1445
1446 if (isspecial == 0 && numfreebuffers < lofreebuffers) {
1447 /*
1448 * This will cause an immediate failure
1449 */
1450 nqindex = QUEUE_CLEAN;
1451 nbp = NULL;
1452 } else {
1453 /*
1454 * Locate a buffer which already has KVA assigned. First
1455 * try EMPTYKVA buffers.
1456 */
1457 nqindex = QUEUE_EMPTYKVA;
1458 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA]);
1459
1460 if (nbp == NULL) {
1461 /*
1462 * If no EMPTYKVA buffers and we are either
1463 * defragging or reusing, locate a CLEAN buffer
1464 * to free or reuse. If bufspace useage is low
1465 * skip this step so we can allocate a new buffer.
1466 */
1467 if (defrag || bufspace >= lobufspace) {
1468 nqindex = QUEUE_CLEAN;
1469 nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
1470 }
1471
1472 /*
1473 * Nada. If we are allowed to allocate an EMPTY
1474 * buffer, go get one.
1475 */
1476 if (nbp == NULL && defrag == 0 &&
1477 (isspecial || bufspace < hibufspace)) {
1478 nqindex = QUEUE_EMPTY;
1479 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1480 }
1481 }
1482 }
1483
1484 /*
1485 * Run scan, possibly freeing data and/or kva mappings on the fly
1486 * depending.
1487 */
1488
1489 while ((bp = nbp) != NULL) {
1490 int qindex = nqindex;
1491
1492 /*
1493 * Calculate next bp ( we can only use it if we do not block
1494 * or do other fancy things ).
1495 */
1496 if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
1497 switch(qindex) {
1498 case QUEUE_EMPTY:
1499 nqindex = QUEUE_EMPTYKVA;
1500 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA])))
1501 break;
1502 /* fall through */
1503 case QUEUE_EMPTYKVA:
1504 nqindex = QUEUE_CLEAN;
1505 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN])))
1506 break;
1507 /* fall through */
1508 case QUEUE_CLEAN:
1509 /*
1510 * nbp is NULL.
1511 */
1512 break;
1513 }
1514 }
1515
1516 /*
1517 * Sanity Checks
1518 */
1519 KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp));
1520
1521 /*
1522 * Note: we no longer distinguish between VMIO and non-VMIO
1523 * buffers.
1524 */
1525
1526 KASSERT((bp->b_flags & B_DELWRI) == 0, ("delwri buffer %p found in queue %d", bp, qindex));
1527
1528 /*
1529 * If we are defragging then we need a buffer with
1530 * b_kvasize != 0. XXX this situation should no longer
1531 * occur, if defrag is non-zero the buffer's b_kvasize
1532 * should also be non-zero at this point. XXX
1533 */
1534 if (defrag && bp->b_kvasize == 0) {
1535 printf("Warning: defrag empty buffer %p\n", bp);
1536 continue;
1537 }
1538
1539 /*
1540 * Start freeing the bp. This is somewhat involved. nbp
1541 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1542 */
1543
1544 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1545 panic("getnewbuf: locked buf");
1546 bremfree(bp);
1547
1548 if (qindex == QUEUE_CLEAN) {
1549 if (bp->b_flags & B_VMIO) {
1550 bp->b_flags &= ~B_ASYNC;
1551 vfs_vmio_release(bp);
1552 }
1553 if (bp->b_vp)
1554 brelvp(bp);
1555 }
1556
1557 /*
1558 * NOTE: nbp is now entirely invalid. We can only restart
1559 * the scan from this point on.
1560 *
1561 * Get the rest of the buffer freed up. b_kva* is still
1562 * valid after this operation.
1563 */
1564
1565 if (bp->b_rcred != NOCRED) {
1566 crfree(bp->b_rcred);
1567 bp->b_rcred = NOCRED;
1568 }
1569 if (bp->b_wcred != NOCRED) {
1570 crfree(bp->b_wcred);
1571 bp->b_wcred = NOCRED;
1572 }
1573 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
1574 (*bioops.io_deallocate)(bp);
1575 if (bp->b_xflags & BX_BKGRDINPROG)
1576 panic("losing buffer 3");
1577 LIST_REMOVE(bp, b_hash);
1578 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1579
1580 if (bp->b_bufsize)
1581 allocbuf(bp, 0);
1582
1583 bp->b_flags = 0;
1584 bp->b_ioflags = 0;
1585 bp->b_xflags = 0;
1586 bp->b_dev = NODEV;
1587 bp->b_vp = NULL;
1588 bp->b_blkno = bp->b_lblkno = 0;
1589 bp->b_offset = NOOFFSET;
1590 bp->b_iodone = 0;
1591 bp->b_error = 0;
1592 bp->b_resid = 0;
1593 bp->b_bcount = 0;
1594 bp->b_npages = 0;
1595 bp->b_dirtyoff = bp->b_dirtyend = 0;
1596
1597 LIST_INIT(&bp->b_dep);
1598
1599 /*
1600 * If we are defragging then free the buffer.
1601 */
1602 if (defrag) {
1603 bp->b_flags |= B_INVAL;
1604 bfreekva(bp);
1605 brelse(bp);
1606 defrag = 0;
1607 goto restart;
1608 }
1609
1610 /*
1611 * If we are a normal process then deal with bufspace
1612 * hysteresis. A normal process tries to keep bufspace
1613 * between lobufspace and hibufspace. Note: if we encounter
1614 * a buffer with b_kvasize == 0 then it means we started
1615 * our scan on the EMPTY list and should allocate a new
1616 * buffer.
1617 */
1618 if (isspecial == 0) {
1619 if (bufspace > hibufspace)
1620 flushingbufs = 1;
1621 if (flushingbufs && bp->b_kvasize != 0) {
1622 bp->b_flags |= B_INVAL;
1623 bfreekva(bp);
1624 brelse(bp);
1625 goto restart;
1626 }
1627 if (bufspace < lobufspace)
1628 flushingbufs = 0;
1629 }
1630 break;
1631 }
1632
1633 /*
1634 * If we exhausted our list, sleep as appropriate. We may have to
1635 * wakeup various daemons and write out some dirty buffers.
1636 *
1637 * Generally we are sleeping due to insufficient buffer space.
1638 */
1639
1640 if (bp == NULL) {
1641 int flags;
1642 char *waitmsg;
1643
1644 if (defrag) {
1645 flags = VFS_BIO_NEED_BUFSPACE;
1646 waitmsg = "nbufkv";
1647 } else if (bufspace >= hibufspace) {
1648 waitmsg = "nbufbs";
1649 flags = VFS_BIO_NEED_BUFSPACE;
1650 } else {
1651 waitmsg = "newbuf";
1652 flags = VFS_BIO_NEED_ANY;
1653 }
1654
1655 bd_speedup(); /* heeeelp */
1656
1657 needsbuffer |= flags;
1658 while (needsbuffer & flags) {
1659 if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag,
1660 waitmsg, slptimeo))
1661 return (NULL);
1662 }
1663 } else {
1664 /*
1665 * We finally have a valid bp. We aren't quite out of the
1666 * woods, we still have to reserve kva space. In order
1667 * to keep fragmentation sane we only allocate kva in
1668 * BKVASIZE chunks.
1669 */
1670 maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
1671
1672 if (maxsize != bp->b_kvasize) {
1673 vm_offset_t addr = 0;
1674
1675 bfreekva(bp);
1676
1677 if (vm_map_findspace(buffer_map,
1678 vm_map_min(buffer_map), maxsize, &addr)) {
1679 /*
1680 * Uh oh. Buffer map is to fragmented. We
1681 * must defragment the map.
1682 */
1683 ++bufdefragcnt;
1684 defrag = 1;
1685 bp->b_flags |= B_INVAL;
1686 brelse(bp);
1687 goto restart;
1688 }
1689 if (addr) {
1690 vm_map_insert(buffer_map, NULL, 0,
1691 addr, addr + maxsize,
1692 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
1693
1694 bp->b_kvabase = (caddr_t) addr;
1695 bp->b_kvasize = maxsize;
1696 bufspace += bp->b_kvasize;
1697 ++bufreusecnt;
1698 }
1699 }
1700 bp->b_data = bp->b_kvabase;
1701 }
1702 return(bp);
1703}
1704
1705/*
1706 * waitfreebuffers:
1707 *
1708 * Wait for sufficient free buffers. Only called from normal processes.
1709 */
1710
1711static void
1712waitfreebuffers(int slpflag, int slptimeo)
1713{
1714 while (numfreebuffers < hifreebuffers) {
1715 if (numfreebuffers >= hifreebuffers)
1716 break;
1717 needsbuffer |= VFS_BIO_NEED_FREE;
1718 if (tsleep(&needsbuffer, (PRIBIO + 4)|slpflag, "biofre", slptimeo))
1719 break;
1720 }
1721}
1722
1723/*
1724 * buf_daemon:
1725 *
1726 * buffer flushing daemon. Buffers are normally flushed by the
1727 * update daemon but if it cannot keep up this process starts to
1728 * take the load in an attempt to prevent getnewbuf() from blocking.
1729 */
1730
1731static struct proc *bufdaemonproc;
1732static int bd_interval;
1733static int bd_flushto;
1734static int bd_flushinc;
1735
1736static struct kproc_desc buf_kp = {
1737 "bufdaemon",
1738 buf_daemon,
1739 &bufdaemonproc
1740};
1741SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp)
1742
1743static void
1744buf_daemon()
1745{
1746 int s;
1747
1748 /*
1749 * This process needs to be suspended prior to shutdown sync.
1750 */
1751 EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, bufdaemonproc,
1752 SHUTDOWN_PRI_LAST);
1753
1754 /*
1755 * This process is allowed to take the buffer cache to the limit
1756 */
1757 curproc->p_flag |= P_BUFEXHAUST;
1758 s = splbio();
1759
1760 bd_interval = 5 * hz; /* dynamically adjusted */
1761 bd_flushto = hidirtybuffers; /* dynamically adjusted */
1762 bd_flushinc = 1;
1763
1764 for (;;) {
1765 kproc_suspend_loop(bufdaemonproc);
1766
1767 bd_request = 0;
1768
1769 /*
1770 * Do the flush. Limit the number of buffers we flush in one
1771 * go. The failure condition occurs when processes are writing
1772 * buffers faster then we can dispose of them. In this case
1773 * we may be flushing so often that the previous set of flushes
1774 * have not had time to complete, causing us to run out of
1775 * physical buffers and block.
1776 */
1777 {
1778 int runcount = maxbdrun;
1779
1780 while (numdirtybuffers > bd_flushto && runcount) {
1781 --runcount;
1782 if (flushbufqueues() == 0)
1783 break;
1784 }
1785 }
1786
1787 if (bd_request ||
1788 tsleep(&bd_request, PVM, "psleep", bd_interval) == 0) {
1789 /*
1790 * Another request is pending or we were woken up
1791 * without timing out. Flush more.
1792 */
1793 --bd_flushto;
1794 if (bd_flushto >= numdirtybuffers - 5) {
1795 bd_flushto = numdirtybuffers - 10;
1796 bd_flushinc = 1;
1797 }
1798 if (bd_flushto < 2)
1799 bd_flushto = 2;
1800 } else {
1801 /*
1802 * We slept and timed out, we can slow down.
1803 */
1804 bd_flushto += bd_flushinc;
1805 if (bd_flushto > hidirtybuffers)
1806 bd_flushto = hidirtybuffers;
1807 ++bd_flushinc;
1808 if (bd_flushinc > hidirtybuffers / 20 + 1)
1809 bd_flushinc = hidirtybuffers / 20 + 1;
1810 }
1811
1812 /*
1813 * Set the interval on a linear scale based on hidirtybuffers
1814 * with a maximum frequency of 1/10 second.
1815 */
1816 bd_interval = bd_flushto * 5 * hz / hidirtybuffers;
1817 if (bd_interval < hz / 10)
1818 bd_interval = hz / 10;
1819 }
1820}
1821
1822/*
1823 * flushbufqueues:
1824 *
1825 * Try to flush a buffer in the dirty queue. We must be careful to
1826 * free up B_INVAL buffers instead of write them, which NFS is
1827 * particularly sensitive to.
1828 */
1829
1830static int
1831flushbufqueues(void)
1832{
1833 struct buf *bp;
1834 int r = 0;
1835
1836 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1837
1838 while (bp) {
1839 KASSERT((bp->b_flags & B_DELWRI), ("unexpected clean buffer %p", bp));
1840 if ((bp->b_flags & B_DELWRI) != 0 &&
1841 (bp->b_xflags & BX_BKGRDINPROG) == 0) {
1842 if (bp->b_flags & B_INVAL) {
1843 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1844 panic("flushbufqueues: locked buf");
1845 bremfree(bp);
1846 brelse(bp);
1847 ++r;
1848 break;
1849 }
1850 if (LIST_FIRST(&bp->b_dep) != NULL &&
1851 bioops.io_countdeps &&
1852 (bp->b_flags & B_DEFERRED) == 0 &&
1853 (*bioops.io_countdeps)(bp, 0)) {
1854 TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY],
1855 bp, b_freelist);
1856 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY],
1857 bp, b_freelist);
1858 bp->b_flags |= B_DEFERRED;
1859 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1860 continue;
1861 }
1862 vfs_bio_awrite(bp);
1863 ++r;
1864 break;
1865 }
1866 bp = TAILQ_NEXT(bp, b_freelist);
1867 }
1868 return (r);
1869}
1870
1871/*
1872 * Check to see if a block is currently memory resident.
1873 */
1874struct buf *
1875incore(struct vnode * vp, daddr_t blkno)
1876{
1877 struct buf *bp;
1878
1879 int s = splbio();
1880 bp = gbincore(vp, blkno);
1881 splx(s);
1882 return (bp);
1883}
1884
1885/*
1886 * Returns true if no I/O is needed to access the
1887 * associated VM object. This is like incore except
1888 * it also hunts around in the VM system for the data.
1889 */
1890
1891int
1892inmem(struct vnode * vp, daddr_t blkno)
1893{
1894 vm_object_t obj;
1895 vm_offset_t toff, tinc, size;
1896 vm_page_t m;
1897 vm_ooffset_t off;
1898
1899 if (incore(vp, blkno))
1900 return 1;
1901 if (vp->v_mount == NULL)
1902 return 0;
1903 if ((vp->v_object == NULL) || (vp->v_flag & VOBJBUF) == 0)
1904 return 0;
1905
1906 obj = vp->v_object;
1907 size = PAGE_SIZE;
1908 if (size > vp->v_mount->mnt_stat.f_iosize)
1909 size = vp->v_mount->mnt_stat.f_iosize;
1910 off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
1911
1912 for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
1913 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
1914 if (!m)
1915 return 0;
1916 tinc = size;
1917 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
1918 tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
1919 if (vm_page_is_valid(m,
1920 (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
1921 return 0;
1922 }
1923 return 1;
1924}
1925
1926/*
1927 * vfs_setdirty:
1928 *
1929 * Sets the dirty range for a buffer based on the status of the dirty
1930 * bits in the pages comprising the buffer.
1931 *
1932 * The range is limited to the size of the buffer.
1933 *
1934 * This routine is primarily used by NFS, but is generalized for the
1935 * B_VMIO case.
1936 */
1937static void
1938vfs_setdirty(struct buf *bp)
1939{
1940 int i;
1941 vm_object_t object;
1942
1943 /*
1944 * Degenerate case - empty buffer
1945 */
1946
1947 if (bp->b_bufsize == 0)
1948 return;
1949
1950 /*
1951 * We qualify the scan for modified pages on whether the
1952 * object has been flushed yet. The OBJ_WRITEABLE flag
1953 * is not cleared simply by protecting pages off.
1954 */
1955
1956 if ((bp->b_flags & B_VMIO) == 0)
1957 return;
1958
1959 object = bp->b_pages[0]->object;
1960
1961 if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY))
1962 printf("Warning: object %p writeable but not mightbedirty\n", object);
1963 if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY))
1964 printf("Warning: object %p mightbedirty but not writeable\n", object);
1965
1966 if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) {
1967 vm_offset_t boffset;
1968 vm_offset_t eoffset;
1969
1970 /*
1971 * test the pages to see if they have been modified directly
1972 * by users through the VM system.
1973 */
1974 for (i = 0; i < bp->b_npages; i++) {
1975 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
1976 vm_page_test_dirty(bp->b_pages[i]);
1977 }
1978
1979 /*
1980 * Calculate the encompassing dirty range, boffset and eoffset,
1981 * (eoffset - boffset) bytes.
1982 */
1983
1984 for (i = 0; i < bp->b_npages; i++) {
1985 if (bp->b_pages[i]->dirty)
1986 break;
1987 }
1988 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1989
1990 for (i = bp->b_npages - 1; i >= 0; --i) {
1991 if (bp->b_pages[i]->dirty) {
1992 break;
1993 }
1994 }
1995 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1996
1997 /*
1998 * Fit it to the buffer.
1999 */
2000
2001 if (eoffset > bp->b_bcount)
2002 eoffset = bp->b_bcount;
2003
2004 /*
2005 * If we have a good dirty range, merge with the existing
2006 * dirty range.
2007 */
2008
2009 if (boffset < eoffset) {
2010 if (bp->b_dirtyoff > boffset)
2011 bp->b_dirtyoff = boffset;
2012 if (bp->b_dirtyend < eoffset)
2013 bp->b_dirtyend = eoffset;
2014 }
2015 }
2016}
2017
2018/*
2019 * getblk:
2020 *
2021 * Get a block given a specified block and offset into a file/device.
2022 * The buffers B_DONE bit will be cleared on return, making it almost
2023 * ready for an I/O initiation. B_INVAL may or may not be set on
2024 * return. The caller should clear B_INVAL prior to initiating a
2025 * READ.
2026 *
2027 * For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2028 * an existing buffer.
2029 *
2030 * For a VMIO buffer, B_CACHE is modified according to the backing VM.
2031 * If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2032 * and then cleared based on the backing VM. If the previous buffer is
2033 * non-0-sized but invalid, B_CACHE will be cleared.
2034 *
2035 * If getblk() must create a new buffer, the new buffer is returned with
2036 * both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2037 * case it is returned with B_INVAL clear and B_CACHE set based on the
2038 * backing VM.
2039 *
2040 * getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2041 * B_CACHE bit is clear.
2042 *
2043 * What this means, basically, is that the caller should use B_CACHE to
2044 * determine whether the buffer is fully valid or not and should clear
2045 * B_INVAL prior to issuing a read. If the caller intends to validate
2046 * the buffer by loading its data area with something, the caller needs
2047 * to clear B_INVAL. If the caller does this without issuing an I/O,
2048 * the caller should set B_CACHE ( as an optimization ), else the caller
2049 * should issue the I/O and biodone() will set B_CACHE if the I/O was
2050 * a write attempt or if it was a successfull read. If the caller
2051 * intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
2052 * prior to issuing the READ. biodone() will *not* clear B_INVAL.
2053 */
2054struct buf *
2055getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo)
2056{
2057 struct buf *bp;
2058 int s;
2059 struct bufhashhdr *bh;
2060
2061 if (size > MAXBSIZE)
2062 panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE);
2063
2064 s = splbio();
2065loop:
2066 /*
2067 * Block if we are low on buffers. Certain processes are allowed
2068 * to completely exhaust the buffer cache.
2069 *
2070 * If this check ever becomes a bottleneck it may be better to
2071 * move it into the else, when gbincore() fails. At the moment
2072 * it isn't a problem.
2073 */
2074 if (!curproc || (curproc->p_flag & P_BUFEXHAUST)) {
2075 if (numfreebuffers == 0) {
2076 if (!curproc)
2077 return NULL;
2078 needsbuffer |= VFS_BIO_NEED_ANY;
2079 tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, "newbuf",
2080 slptimeo);
2081 }
2082 } else if (numfreebuffers < lofreebuffers) {
2083 waitfreebuffers(slpflag, slptimeo);
2084 }
2085
2086 if ((bp = gbincore(vp, blkno))) {
2087 /*
2088 * Buffer is in-core. If the buffer is not busy, it must
2089 * be on a queue.
2090 */
2091
2092 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
2093 if (BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL,
2094 "getblk", slpflag, slptimeo) == ENOLCK)
2095 goto loop;
2096 splx(s);
2097 return (struct buf *) NULL;
2098 }
2099
2100 /*
2101 * The buffer is locked. B_CACHE is cleared if the buffer is
2102 * invalid. Ohterwise, for a non-VMIO buffer, B_CACHE is set
2103 * and for a VMIO buffer B_CACHE is adjusted according to the
2104 * backing VM cache.
2105 */
2106 if (bp->b_flags & B_INVAL)
2107 bp->b_flags &= ~B_CACHE;
2108 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
2109 bp->b_flags |= B_CACHE;
2110 bremfree(bp);
2111
2112 /*
2113 * check for size inconsistancies for non-VMIO case.
2114 */
2115
2116 if (bp->b_bcount != size) {
2117 if ((bp->b_flags & B_VMIO) == 0 ||
2118 (size > bp->b_kvasize)) {
2119 if (bp->b_flags & B_DELWRI) {
2120 bp->b_flags |= B_NOCACHE;
2121 BUF_WRITE(bp);
2122 } else {
2123 if ((bp->b_flags & B_VMIO) &&
2124 (LIST_FIRST(&bp->b_dep) == NULL)) {
2125 bp->b_flags |= B_RELBUF;
2126 brelse(bp);
2127 } else {
2128 bp->b_flags |= B_NOCACHE;
2129 BUF_WRITE(bp);
2130 }
2131 }
2132 goto loop;
2133 }
2134 }
2135
2136 /*
2137 * If the size is inconsistant in the VMIO case, we can resize
2138 * the buffer. This might lead to B_CACHE getting set or
2139 * cleared. If the size has not changed, B_CACHE remains
2140 * unchanged from its previous state.
2141 */
2142
2143 if (bp->b_bcount != size)
2144 allocbuf(bp, size);
2145
2146 KASSERT(bp->b_offset != NOOFFSET,
2147 ("getblk: no buffer offset"));
2148
2149 /*
2150 * A buffer with B_DELWRI set and B_CACHE clear must
2151 * be committed before we can return the buffer in
2152 * order to prevent the caller from issuing a read
2153 * ( due to B_CACHE not being set ) and overwriting
2154 * it.
2155 *
2156 * Most callers, including NFS and FFS, need this to
2157 * operate properly either because they assume they
2158 * can issue a read if B_CACHE is not set, or because
2159 * ( for example ) an uncached B_DELWRI might loop due
2160 * to softupdates re-dirtying the buffer. In the latter
2161 * case, B_CACHE is set after the first write completes,
2162 * preventing further loops.
2163 */
2164
2165 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
2166 BUF_WRITE(bp);
2167 goto loop;
2168 }
2169
2170 splx(s);
2171 bp->b_flags &= ~B_DONE;
2172 } else {
2173 /*
2174 * Buffer is not in-core, create new buffer. The buffer
2175 * returned by getnewbuf() is locked. Note that the returned
2176 * buffer is also considered valid (not marked B_INVAL).
2177 */
2178 int bsize, maxsize, vmio;
2179 off_t offset;
2180
2181 if (vn_isdisk(vp, NULL))
2182 bsize = DEV_BSIZE;
2183 else if (vp->v_mountedhere)
2184 bsize = vp->v_mountedhere->mnt_stat.f_iosize;
2185 else if (vp->v_mount)
2186 bsize = vp->v_mount->mnt_stat.f_iosize;
2187 else
2188 bsize = size;
2189
2190 offset = (off_t)blkno * bsize;
2191 vmio = (vp->v_object != 0) && (vp->v_flag & VOBJBUF);
2192 maxsize = vmio ? size + (offset & PAGE_MASK) : size;
2193 maxsize = imax(maxsize, bsize);
2194
2195 if ((bp = getnewbuf(slpflag, slptimeo, size, maxsize)) == NULL) {
2196 if (slpflag || slptimeo) {
2197 splx(s);
2198 return NULL;
2199 }
2200 goto loop;
2201 }
2202
2203 /*
2204 * This code is used to make sure that a buffer is not
2205 * created while the getnewbuf routine is blocked.
2206 * This can be a problem whether the vnode is locked or not.
2207 * If the buffer is created out from under us, we have to
2208 * throw away the one we just created. There is now window
2209 * race because we are safely running at splbio() from the
2210 * point of the duplicate buffer creation through to here,
2211 * and we've locked the buffer.
2212 */
2213 if (gbincore(vp, blkno)) {
2214 bp->b_flags |= B_INVAL;
2215 brelse(bp);
2216 goto loop;
2217 }
2218
2219 /*
2220 * Insert the buffer into the hash, so that it can
2221 * be found by incore.
2222 */
2223 bp->b_blkno = bp->b_lblkno = blkno;
2224 bp->b_offset = offset;
2225
2226 bgetvp(vp, bp);
2227 LIST_REMOVE(bp, b_hash);
2228 bh = bufhash(vp, blkno);
2229 LIST_INSERT_HEAD(bh, bp, b_hash);
2230
2231 /*
2232 * set B_VMIO bit. allocbuf() the buffer bigger. Since the
2233 * buffer size starts out as 0, B_CACHE will be set by
2234 * allocbuf() for the VMIO case prior to it testing the
2235 * backing store for validity.
2236 */
2237
2238 if (vmio) {
2239 bp->b_flags |= B_VMIO;
2240#if defined(VFS_BIO_DEBUG)
2241 if (vp->v_type != VREG && vp->v_type != VBLK)
2242 printf("getblk: vmioing file type %d???\n", vp->v_type);
2243#endif
2244 } else {
2245 bp->b_flags &= ~B_VMIO;
2246 }
2247
2248 allocbuf(bp, size);
2249
2250 splx(s);
2251 bp->b_flags &= ~B_DONE;
2252 }
2253 return (bp);
2254}
2255
2256/*
2257 * Get an empty, disassociated buffer of given size. The buffer is initially
2258 * set to B_INVAL.
2259 */
2260struct buf *
2261geteblk(int size)
2262{
2263 struct buf *bp;
2264 int s;
2265 int maxsize;
2266
2267 maxsize = (size + BKVAMASK) & ~BKVAMASK;
2268
2269 s = splbio();
2270 while ((bp = getnewbuf(0, 0, size, maxsize)) == 0);
2271 splx(s);
2272 allocbuf(bp, size);
2273 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
2274 return (bp);
2275}
2276
2277
2278/*
2279 * This code constitutes the buffer memory from either anonymous system
2280 * memory (in the case of non-VMIO operations) or from an associated
2281 * VM object (in the case of VMIO operations). This code is able to
2282 * resize a buffer up or down.
2283 *
2284 * Note that this code is tricky, and has many complications to resolve
2285 * deadlock or inconsistant data situations. Tread lightly!!!
2286 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2287 * the caller. Calling this code willy nilly can result in the loss of data.
2288 *
2289 * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
2290 * B_CACHE for the non-VMIO case.
2291 */
2292
2293int
2294allocbuf(struct buf *bp, int size)
2295{
2296 int newbsize, mbsize;
2297 int i;
2298
2299 if (BUF_REFCNT(bp) == 0)
2300 panic("allocbuf: buffer not busy");
2301
2302 if (bp->b_kvasize < size)
2303 panic("allocbuf: buffer too small");
2304
2305 if ((bp->b_flags & B_VMIO) == 0) {
2306 caddr_t origbuf;
2307 int origbufsize;
2308 /*
2309 * Just get anonymous memory from the kernel. Don't
2310 * mess with B_CACHE.
2311 */
2312 mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2313#if !defined(NO_B_MALLOC)
2314 if (bp->b_flags & B_MALLOC)
2315 newbsize = mbsize;
2316 else
2317#endif
2318 newbsize = round_page(size);
2319
2320 if (newbsize < bp->b_bufsize) {
2321#if !defined(NO_B_MALLOC)
2322 /*
2323 * malloced buffers are not shrunk
2324 */
2325 if (bp->b_flags & B_MALLOC) {
2326 if (newbsize) {
2327 bp->b_bcount = size;
2328 } else {
2329 free(bp->b_data, M_BIOBUF);
2330 bufmallocspace -= bp->b_bufsize;
2331 runningbufspace -= bp->b_bufsize;
2332 if (bp->b_bufsize)
2333 bufspacewakeup();
2334 bp->b_data = bp->b_kvabase;
2335 bp->b_bufsize = 0;
2336 bp->b_bcount = 0;
2337 bp->b_flags &= ~B_MALLOC;
2338 }
2339 return 1;
2340 }
2341#endif
2342 vm_hold_free_pages(
2343 bp,
2344 (vm_offset_t) bp->b_data + newbsize,
2345 (vm_offset_t) bp->b_data + bp->b_bufsize);
2346 } else if (newbsize > bp->b_bufsize) {
2347#if !defined(NO_B_MALLOC)
2348 /*
2349 * We only use malloced memory on the first allocation.
2350 * and revert to page-allocated memory when the buffer
2351 * grows.
2352 */
2353 if ( (bufmallocspace < maxbufmallocspace) &&
2354 (bp->b_bufsize == 0) &&
2355 (mbsize <= PAGE_SIZE/2)) {
2356
2357 bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
2358 bp->b_bufsize = mbsize;
2359 bp->b_bcount = size;
2360 bp->b_flags |= B_MALLOC;
2361 bufmallocspace += mbsize;
2362 runningbufspace += bp->b_bufsize;
2363 return 1;
2364 }
2365#endif
2366 origbuf = NULL;
2367 origbufsize = 0;
2368#if !defined(NO_B_MALLOC)
2369 /*
2370 * If the buffer is growing on its other-than-first allocation,
2371 * then we revert to the page-allocation scheme.
2372 */
2373 if (bp->b_flags & B_MALLOC) {
2374 origbuf = bp->b_data;
2375 origbufsize = bp->b_bufsize;
2376 bp->b_data = bp->b_kvabase;
2377 bufmallocspace -= bp->b_bufsize;
2378 runningbufspace -= bp->b_bufsize;
2379 if (bp->b_bufsize)
2380 bufspacewakeup();
2381 bp->b_bufsize = 0;
2382 bp->b_flags &= ~B_MALLOC;
2383 newbsize = round_page(newbsize);
2384 }
2385#endif
2386 vm_hold_load_pages(
2387 bp,
2388 (vm_offset_t) bp->b_data + bp->b_bufsize,
2389 (vm_offset_t) bp->b_data + newbsize);
2390#if !defined(NO_B_MALLOC)
2391 if (origbuf) {
2392 bcopy(origbuf, bp->b_data, origbufsize);
2393 free(origbuf, M_BIOBUF);
2394 }
2395#endif
2396 }
2397 } else {
2398 vm_page_t m;
2399 int desiredpages;
2400
2401 newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2402 desiredpages = (size == 0) ? 0 :
2403 num_pages((bp->b_offset & PAGE_MASK) + newbsize);
2404
2405#if !defined(NO_B_MALLOC)
2406 if (bp->b_flags & B_MALLOC)
2407 panic("allocbuf: VMIO buffer can't be malloced");
2408#endif
2409 /*
2410 * Set B_CACHE initially if buffer is 0 length or will become
2411 * 0-length.
2412 */
2413 if (size == 0 || bp->b_bufsize == 0)
2414 bp->b_flags |= B_CACHE;
2415
2416 if (newbsize < bp->b_bufsize) {
2417 /*
2418 * DEV_BSIZE aligned new buffer size is less then the
2419 * DEV_BSIZE aligned existing buffer size. Figure out
2420 * if we have to remove any pages.
2421 */
2422 if (desiredpages < bp->b_npages) {
2423 for (i = desiredpages; i < bp->b_npages; i++) {
2424 /*
2425 * the page is not freed here -- it
2426 * is the responsibility of
2427 * vnode_pager_setsize
2428 */
2429 m = bp->b_pages[i];
2430 KASSERT(m != bogus_page,
2431 ("allocbuf: bogus page found"));
2432 while (vm_page_sleep_busy(m, TRUE, "biodep"))
2433 ;
2434
2435 bp->b_pages[i] = NULL;
2436 vm_page_unwire(m, 0);
2437 }
2438 pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) +
2439 (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages));
2440 bp->b_npages = desiredpages;
2441 }
2442 } else if (size > bp->b_bcount) {
2443 /*
2444 * We are growing the buffer, possibly in a
2445 * byte-granular fashion.
2446 */
2447 struct vnode *vp;
2448 vm_object_t obj;
2449 vm_offset_t toff;
2450 vm_offset_t tinc;
2451
2452 /*
2453 * Step 1, bring in the VM pages from the object,
2454 * allocating them if necessary. We must clear
2455 * B_CACHE if these pages are not valid for the
2456 * range covered by the buffer.
2457 */
2458
2459 vp = bp->b_vp;
2460 obj = vp->v_object;
2461
2462 while (bp->b_npages < desiredpages) {
2463 vm_page_t m;
2464 vm_pindex_t pi;
2465
2466 pi = OFF_TO_IDX(bp->b_offset) + bp->b_npages;
2467 if ((m = vm_page_lookup(obj, pi)) == NULL) {
2468 m = vm_page_alloc(obj, pi, VM_ALLOC_NORMAL);
2469 if (m == NULL) {
2470 VM_WAIT;
2471 vm_pageout_deficit += desiredpages - bp->b_npages;
2472 } else {
2473 vm_page_wire(m);
2474 vm_page_wakeup(m);
2475 bp->b_flags &= ~B_CACHE;
2476 bp->b_pages[bp->b_npages] = m;
2477 ++bp->b_npages;
2478 }
2479 continue;
2480 }
2481
2482 /*
2483 * We found a page. If we have to sleep on it,
2484 * retry because it might have gotten freed out
2485 * from under us.
2486 *
2487 * We can only test PG_BUSY here. Blocking on
2488 * m->busy might lead to a deadlock:
2489 *
2490 * vm_fault->getpages->cluster_read->allocbuf
2491 *
2492 */
2493
2494 if (vm_page_sleep_busy(m, FALSE, "pgtblk"))
2495 continue;
2496
2497 /*
2498 * We have a good page. Should we wakeup the
2499 * page daemon?
2500 */
2501 if ((curproc != pageproc) &&
2502 ((m->queue - m->pc) == PQ_CACHE) &&
2503 ((cnt.v_free_count + cnt.v_cache_count) <
2504 (cnt.v_free_min + cnt.v_cache_min))) {
2505 pagedaemon_wakeup();
2506 }
2507 vm_page_flag_clear(m, PG_ZERO);
2508 vm_page_wire(m);
2509 bp->b_pages[bp->b_npages] = m;
2510 ++bp->b_npages;
2511 }
2512
2513 /*
2514 * Step 2. We've loaded the pages into the buffer,
2515 * we have to figure out if we can still have B_CACHE
2516 * set. Note that B_CACHE is set according to the
2517 * byte-granular range ( bcount and size ), new the
2518 * aligned range ( newbsize ).
2519 *
2520 * The VM test is against m->valid, which is DEV_BSIZE
2521 * aligned. Needless to say, the validity of the data
2522 * needs to also be DEV_BSIZE aligned. Note that this
2523 * fails with NFS if the server or some other client
2524 * extends the file's EOF. If our buffer is resized,
2525 * B_CACHE may remain set! XXX
2526 */
2527
2528 toff = bp->b_bcount;
2529 tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2530
2531 while ((bp->b_flags & B_CACHE) && toff < size) {
2532 vm_pindex_t pi;
2533
2534 if (tinc > (size - toff))
2535 tinc = size - toff;
2536
2537 pi = ((bp->b_offset & PAGE_MASK) + toff) >>
2538 PAGE_SHIFT;
2539
2540 vfs_buf_test_cache(
2541 bp,
2542 bp->b_offset,
2543 toff,
2544 tinc,
2545 bp->b_pages[pi]
2546 );
2547 toff += tinc;
2548 tinc = PAGE_SIZE;
2549 }
2550
2551 /*
2552 * Step 3, fixup the KVM pmap. Remember that
2553 * bp->b_data is relative to bp->b_offset, but
2554 * bp->b_offset may be offset into the first page.
2555 */
2556
2557 bp->b_data = (caddr_t)
2558 trunc_page((vm_offset_t)bp->b_data);
2559 pmap_qenter(
2560 (vm_offset_t)bp->b_data,
2561 bp->b_pages,
2562 bp->b_npages
2563 );
2564 bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
2565 (vm_offset_t)(bp->b_offset & PAGE_MASK));
2566 }
2567 }
2568 runningbufspace += (newbsize - bp->b_bufsize);
2569 if (newbsize < bp->b_bufsize)
2570 bufspacewakeup();
2571 bp->b_bufsize = newbsize; /* actual buffer allocation */
2572 bp->b_bcount = size; /* requested buffer size */
2573 return 1;
2574}
2575
2576/*
2577 * bufwait:
2578 *
2579 * Wait for buffer I/O completion, returning error status. The buffer
2580 * is left locked and B_DONE on return. B_EINTR is converted into a EINTR
2581 * error and cleared.
2582 */
2583int
2584bufwait(register struct buf * bp)
2585{
2586 int s;
2587
2588 s = splbio();
2589 while ((bp->b_flags & B_DONE) == 0) {
2590 if (bp->b_iocmd == BIO_READ)
2591 tsleep(bp, PRIBIO, "biord", 0);
2592 else
2593 tsleep(bp, PRIBIO, "biowr", 0);
2594 }
2595 splx(s);
2596 if (bp->b_flags & B_EINTR) {
2597 bp->b_flags &= ~B_EINTR;
2598 return (EINTR);
2599 }
2600 if (bp->b_ioflags & BIO_ERROR) {
2601 return (bp->b_error ? bp->b_error : EIO);
2602 } else {
2603 return (0);
2604 }
2605}
2606
2607 /*
2608 * Call back function from struct bio back up to struct buf.
2609 * The corresponding initialization lives in sys/conf.h:DEV_STRATEGY().
2610 */
2611void
2612bufdonebio(struct bio *bp)
2613{
2614 bufdone(bp->bio_caller2);
2615}
2616
2617/*
2618 * bufdone:
2619 *
2620 * Finish I/O on a buffer, optionally calling a completion function.
2621 * This is usually called from an interrupt so process blocking is
2622 * not allowed.
2623 *
2624 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2625 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
2626 * assuming B_INVAL is clear.
2627 *
2628 * For the VMIO case, we set B_CACHE if the op was a read and no
2629 * read error occured, or if the op was a write. B_CACHE is never
2630 * set if the buffer is invalid or otherwise uncacheable.
2631 *
2632 * biodone does not mess with B_INVAL, allowing the I/O routine or the
2633 * initiator to leave B_INVAL set to brelse the buffer out of existance
2634 * in the biodone routine.
2635 */
2636void
2637bufdone(struct buf *bp)
2638{
2639 int s;
2640 void (*biodone) __P((struct buf *));
2641
2642 s = splbio();
2643
2644 KASSERT(BUF_REFCNT(bp) > 0, ("biodone: bp %p not busy %d", bp, BUF_REFCNT(bp)));
2645 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
2646
2647 bp->b_flags |= B_DONE;
2648
2649 if (bp->b_iocmd == BIO_DELETE) {
2650 brelse(bp);
2651 splx(s);
2652 return;
2653 }
2654
2655 if (bp->b_iocmd == BIO_WRITE) {
2656 vwakeup(bp);
2657 }
2658
2659 /* call optional completion function if requested */
2660 if (bp->b_iodone != NULL) {
2661 biodone = bp->b_iodone;
2662 bp->b_iodone = NULL;
2663 (*biodone) (bp);
2664 splx(s);
2665 return;
2666 }
2667 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
2668 (*bioops.io_complete)(bp);
2669
2670 if (bp->b_flags & B_VMIO) {
2671 int i, resid;
2672 vm_ooffset_t foff;
2673 vm_page_t m;
2674 vm_object_t obj;
2675 int iosize;
2676 struct vnode *vp = bp->b_vp;
2677
2678 obj = vp->v_object;
2679
2680#if defined(VFS_BIO_DEBUG)
2681 if (vp->v_usecount == 0) {
2682 panic("biodone: zero vnode ref count");
2683 }
2684
2685 if (vp->v_object == NULL) {
2686 panic("biodone: missing VM object");
2687 }
2688
2689 if ((vp->v_flag & VOBJBUF) == 0) {
2690 panic("biodone: vnode is not setup for merged cache");
2691 }
2692#endif
2693
2694 foff = bp->b_offset;
2695 KASSERT(bp->b_offset != NOOFFSET,
2696 ("biodone: no buffer offset"));
2697
2698 if (!obj) {
2699 panic("biodone: no object");
2700 }
2701#if defined(VFS_BIO_DEBUG)
2702 if (obj->paging_in_progress < bp->b_npages) {
2703 printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n",
2704 obj->paging_in_progress, bp->b_npages);
2705 }
2706#endif
2707
2708 /*
2709 * Set B_CACHE if the op was a normal read and no error
2710 * occured. B_CACHE is set for writes in the b*write()
2711 * routines.
2712 */
2713 iosize = bp->b_bcount - bp->b_resid;
2714 if (bp->b_iocmd == BIO_READ &&
2715 !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
2716 !(bp->b_ioflags & BIO_ERROR)) {
2717 bp->b_flags |= B_CACHE;
2718 }
2719
2720 for (i = 0; i < bp->b_npages; i++) {
2721 int bogusflag = 0;
2722 m = bp->b_pages[i];
2723 if (m == bogus_page) {
2724 bogusflag = 1;
2725 m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2726 if (!m) {
2727#if defined(VFS_BIO_DEBUG)
2728 printf("biodone: page disappeared\n");
2729#endif
2730 vm_object_pip_subtract(obj, 1);
2731 bp->b_flags &= ~B_CACHE;
2732 continue;
2733 }
2734 bp->b_pages[i] = m;
2735 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2736 }
2737#if defined(VFS_BIO_DEBUG)
2738 if (OFF_TO_IDX(foff) != m->pindex) {
2739 printf(
2740"biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2741 (unsigned long)foff, m->pindex);
2742 }
2743#endif
2744 resid = IDX_TO_OFF(m->pindex + 1) - foff;
2745 if (resid > iosize)
2746 resid = iosize;
2747
2748 /*
2749 * In the write case, the valid and clean bits are
2750 * already changed correctly ( see bdwrite() ), so we
2751 * only need to do this here in the read case.
2752 */
2753 if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
2754 vfs_page_set_valid(bp, foff, i, m);
2755 }
2756 vm_page_flag_clear(m, PG_ZERO);
2757
2758 /*
2759 * when debugging new filesystems or buffer I/O methods, this
2760 * is the most common error that pops up. if you see this, you
2761 * have not set the page busy flag correctly!!!
2762 */
2763 if (m->busy == 0) {
2764 printf("biodone: page busy < 0, "
2765 "pindex: %d, foff: 0x(%x,%x), "
2766 "resid: %d, index: %d\n",
2767 (int) m->pindex, (int)(foff >> 32),
2768 (int) foff & 0xffffffff, resid, i);
2769 if (!vn_isdisk(vp, NULL))
2770 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2771 bp->b_vp->v_mount->mnt_stat.f_iosize,
2772 (int) bp->b_lblkno,
2773 bp->b_flags, bp->b_npages);
2774 else
2775 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2776 (int) bp->b_lblkno,
2777 bp->b_flags, bp->b_npages);
2778 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2779 m->valid, m->dirty, m->wire_count);
2780 panic("biodone: page busy < 0\n");
2781 }
2782 vm_page_io_finish(m);
2783 vm_object_pip_subtract(obj, 1);
2784 foff += resid;
2785 iosize -= resid;
2786 }
2787 if (obj)
2788 vm_object_pip_wakeupn(obj, 0);
2789 }
2790 /*
2791 * For asynchronous completions, release the buffer now. The brelse
2792 * will do a wakeup there if necessary - so no need to do a wakeup
2793 * here in the async case. The sync case always needs to do a wakeup.
2794 */
2795
2796 if (bp->b_flags & B_ASYNC) {
2797 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
2798 brelse(bp);
2799 else
2800 bqrelse(bp);
2801 } else {
2802 wakeup(bp);
2803 }
2804 splx(s);
2805}
2806
2807/*
2808 * This routine is called in lieu of iodone in the case of
2809 * incomplete I/O. This keeps the busy status for pages
2810 * consistant.
2811 */
2812void
2813vfs_unbusy_pages(struct buf * bp)
2814{
2815 int i;
2816
2817 if (bp->b_flags & B_VMIO) {
2818 struct vnode *vp = bp->b_vp;
2819 vm_object_t obj = vp->v_object;
2820
2821 for (i = 0; i < bp->b_npages; i++) {
2822 vm_page_t m = bp->b_pages[i];
2823
2824 if (m == bogus_page) {
2825 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
2826 if (!m) {
2827 panic("vfs_unbusy_pages: page missing\n");
2828 }
2829 bp->b_pages[i] = m;
2830 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2831 }
2832 vm_object_pip_subtract(obj, 1);
2833 vm_page_flag_clear(m, PG_ZERO);
2834 vm_page_io_finish(m);
2835 }
2836 vm_object_pip_wakeupn(obj, 0);
2837 }
2838}
2839
2840/*
2841 * vfs_page_set_valid:
2842 *
2843 * Set the valid bits in a page based on the supplied offset. The
2844 * range is restricted to the buffer's size.
2845 *
2846 * This routine is typically called after a read completes.
2847 */
2848static void
2849vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m)
2850{
2851 vm_ooffset_t soff, eoff;
2852
2853 /*
2854 * Start and end offsets in buffer. eoff - soff may not cross a
2855 * page boundry or cross the end of the buffer. The end of the
2856 * buffer, in this case, is our file EOF, not the allocation size
2857 * of the buffer.
2858 */
2859 soff = off;
2860 eoff = (off + PAGE_SIZE) & ~PAGE_MASK;
2861 if (eoff > bp->b_offset + bp->b_bcount)
2862 eoff = bp->b_offset + bp->b_bcount;
2863
2864 /*
2865 * Set valid range. This is typically the entire buffer and thus the
2866 * entire page.
2867 */
2868 if (eoff > soff) {
2869 vm_page_set_validclean(
2870 m,
2871 (vm_offset_t) (soff & PAGE_MASK),
2872 (vm_offset_t) (eoff - soff)
2873 );
2874 }
2875}
2876
2877/*
2878 * This routine is called before a device strategy routine.
2879 * It is used to tell the VM system that paging I/O is in
2880 * progress, and treat the pages associated with the buffer
2881 * almost as being PG_BUSY. Also the object paging_in_progress
2882 * flag is handled to make sure that the object doesn't become
2883 * inconsistant.
2884 *
2885 * Since I/O has not been initiated yet, certain buffer flags
2886 * such as BIO_ERROR or B_INVAL may be in an inconsistant state
2887 * and should be ignored.
2888 */
2889void
2890vfs_busy_pages(struct buf * bp, int clear_modify)
2891{
2892 int i, bogus;
2893
2894 if (bp->b_flags & B_VMIO) {
2895 struct vnode *vp = bp->b_vp;
2896 vm_object_t obj = vp->v_object;
2897 vm_ooffset_t foff;
2898
2899 foff = bp->b_offset;
2900 KASSERT(bp->b_offset != NOOFFSET,
2901 ("vfs_busy_pages: no buffer offset"));
2902 vfs_setdirty(bp);
2903
2904retry:
2905 for (i = 0; i < bp->b_npages; i++) {
2906 vm_page_t m = bp->b_pages[i];
2907 if (vm_page_sleep_busy(m, FALSE, "vbpage"))
2908 goto retry;
2909 }
2910
2911 bogus = 0;
2912 for (i = 0; i < bp->b_npages; i++) {
2913 vm_page_t m = bp->b_pages[i];
2914
2915 vm_page_flag_clear(m, PG_ZERO);
2916 if ((bp->b_flags & B_CLUSTER) == 0) {
2917 vm_object_pip_add(obj, 1);
2918 vm_page_io_start(m);
2919 }
2920
2921 /*
2922 * When readying a buffer for a read ( i.e
2923 * clear_modify == 0 ), it is important to do
2924 * bogus_page replacement for valid pages in
2925 * partially instantiated buffers. Partially
2926 * instantiated buffers can, in turn, occur when
2927 * reconstituting a buffer from its VM backing store
2928 * base. We only have to do this if B_CACHE is
2929 * clear ( which causes the I/O to occur in the
2930 * first place ). The replacement prevents the read
2931 * I/O from overwriting potentially dirty VM-backed
2932 * pages. XXX bogus page replacement is, uh, bogus.
2933 * It may not work properly with small-block devices.
2934 * We need to find a better way.
2935 */
2936
2937 vm_page_protect(m, VM_PROT_NONE);
2938 if (clear_modify)
2939 vfs_page_set_valid(bp, foff, i, m);
2940 else if (m->valid == VM_PAGE_BITS_ALL &&
2941 (bp->b_flags & B_CACHE) == 0) {
2942 bp->b_pages[i] = bogus_page;
2943 bogus++;
2944 }
2945 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2946 }
2947 if (bogus)
2948 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2949 }
2950}
2951
2952/*
2953 * Tell the VM system that the pages associated with this buffer
2954 * are clean. This is used for delayed writes where the data is
2955 * going to go to disk eventually without additional VM intevention.
2956 *
2957 * Note that while we only really need to clean through to b_bcount, we
2958 * just go ahead and clean through to b_bufsize.
2959 */
2960static void
2961vfs_clean_pages(struct buf * bp)
2962{
2963 int i;
2964
2965 if (bp->b_flags & B_VMIO) {
2966 vm_ooffset_t foff;
2967
2968 foff = bp->b_offset;
2969 KASSERT(bp->b_offset != NOOFFSET,
2970 ("vfs_clean_pages: no buffer offset"));
2971 for (i = 0; i < bp->b_npages; i++) {
2972 vm_page_t m = bp->b_pages[i];
2973 vm_ooffset_t noff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2974 vm_ooffset_t eoff = noff;
2975
2976 if (eoff > bp->b_offset + bp->b_bufsize)
2977 eoff = bp->b_offset + bp->b_bufsize;
2978 vfs_page_set_valid(bp, foff, i, m);
2979 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
2980 foff = noff;
2981 }
2982 }
2983}
2984
2985/*
2986 * vfs_bio_set_validclean:
2987 *
2988 * Set the range within the buffer to valid and clean. The range is
2989 * relative to the beginning of the buffer, b_offset. Note that b_offset
2990 * itself may be offset from the beginning of the first page.
2991 */
2992
2993void
2994vfs_bio_set_validclean(struct buf *bp, int base, int size)
2995{
2996 if (bp->b_flags & B_VMIO) {
2997 int i;
2998 int n;
2999
3000 /*
3001 * Fixup base to be relative to beginning of first page.
3002 * Set initial n to be the maximum number of bytes in the
3003 * first page that can be validated.
3004 */
3005
3006 base += (bp->b_offset & PAGE_MASK);
3007 n = PAGE_SIZE - (base & PAGE_MASK);
3008
3009 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
3010 vm_page_t m = bp->b_pages[i];
3011
3012 if (n > size)
3013 n = size;
3014
3015 vm_page_set_validclean(m, base & PAGE_MASK, n);
3016 base += n;
3017 size -= n;
3018 n = PAGE_SIZE;
3019 }
3020 }
3021}
3022
3023/*
3024 * vfs_bio_clrbuf:
3025 *
3026 * clear a buffer. This routine essentially fakes an I/O, so we need
3027 * to clear BIO_ERROR and B_INVAL.
3028 *
3029 * Note that while we only theoretically need to clear through b_bcount,
3030 * we go ahead and clear through b_bufsize.
3031 */
3032
3033void
3034vfs_bio_clrbuf(struct buf *bp) {
3035 int i, mask = 0;
3036 caddr_t sa, ea;
3037 if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) {
3038 bp->b_flags &= ~B_INVAL;
3039 bp->b_ioflags &= ~BIO_ERROR;
3040 if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
3041 (bp->b_offset & PAGE_MASK) == 0) {
3042 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
3043 if (((bp->b_pages[0]->flags & PG_ZERO) == 0) &&
3044 ((bp->b_pages[0]->valid & mask) != mask)) {
3045 bzero(bp->b_data, bp->b_bufsize);
3046 }
3047 bp->b_pages[0]->valid |= mask;
3048 bp->b_resid = 0;
3049 return;
3050 }
3051 ea = sa = bp->b_data;
3052 for(i=0;i<bp->b_npages;i++,sa=ea) {
3053 int j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
3054 ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE);
3055 ea = (caddr_t)(vm_offset_t)ulmin(
3056 (u_long)(vm_offset_t)ea,
3057 (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize);
3058 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
3059 if ((bp->b_pages[i]->valid & mask) == mask)
3060 continue;
3061 if ((bp->b_pages[i]->valid & mask) == 0) {
3062 if ((bp->b_pages[i]->flags & PG_ZERO) == 0) {
3063 bzero(sa, ea - sa);
3064 }
3065 } else {
3066 for (; sa < ea; sa += DEV_BSIZE, j++) {
3067 if (((bp->b_pages[i]->flags & PG_ZERO) == 0) &&
3068 (bp->b_pages[i]->valid & (1<<j)) == 0)
3069 bzero(sa, DEV_BSIZE);
3070 }
3071 }
3072 bp->b_pages[i]->valid |= mask;
3073 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
3074 }
3075 bp->b_resid = 0;
3076 } else {
3077 clrbuf(bp);
3078 }
3079}
3080
3081/*
3082 * vm_hold_load_pages and vm_hold_unload pages get pages into
3083 * a buffers address space. The pages are anonymous and are
3084 * not associated with a file object.
3085 */
3086void
3087vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3088{
3089 vm_offset_t pg;
3090 vm_page_t p;
3091 int index;
3092
3093 to = round_page(to);
3094 from = round_page(from);
3095 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3096
3097 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3098
3099tryagain:
3100
3101 p = vm_page_alloc(kernel_object,
3102 ((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
3103 VM_ALLOC_NORMAL);
3104 if (!p) {
3105 vm_pageout_deficit += (to - from) >> PAGE_SHIFT;
3106 VM_WAIT;
3107 goto tryagain;
3108 }
3109 vm_page_wire(p);
3110 p->valid = VM_PAGE_BITS_ALL;
3111 vm_page_flag_clear(p, PG_ZERO);
3112 pmap_kenter(pg, VM_PAGE_TO_PHYS(p));
3113 bp->b_pages[index] = p;
3114 vm_page_wakeup(p);
3115 }
3116 bp->b_npages = index;
3117}
3118
3119void
3120vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3121{
3122 vm_offset_t pg;
3123 vm_page_t p;
3124 int index, newnpages;
3125
3126 from = round_page(from);
3127 to = round_page(to);
3128 newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3129
3130 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3131 p = bp->b_pages[index];
3132 if (p && (index < bp->b_npages)) {
3133 if (p->busy) {
3134 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3135 bp->b_blkno, bp->b_lblkno);
3136 }
3137 bp->b_pages[index] = NULL;
3138 pmap_kremove(pg);
3139 vm_page_busy(p);
3140 vm_page_unwire(p, 0);
3141 vm_page_free(p);
3142 }
3143 }
3144 bp->b_npages = newnpages;
3145}
3146
3147
3148#include "opt_ddb.h"
3149#ifdef DDB
3150#include <ddb/ddb.h>
3151
3152DB_SHOW_COMMAND(buffer, db_show_buffer)
3153{
3154 /* get args */
3155 struct buf *bp = (struct buf *)addr;
3156
3157 if (!have_addr) {
3158 db_printf("usage: show buffer <addr>\n");
3159 return;
3160 }
3161
3162 db_printf("b_flags = 0x%b\n", (u_int)bp->b_flags, PRINT_BUF_FLAGS);
3163 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3164 "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3165 "b_blkno = %d, b_pblkno = %d\n",
3166 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
3167 major(bp->b_dev), minor(bp->b_dev),
3168 bp->b_data, bp->b_blkno, bp->b_pblkno);
3169 if (bp->b_npages) {
3170 int i;
3171 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
3172 for (i = 0; i < bp->b_npages; i++) {
3173 vm_page_t m;
3174 m = bp->b_pages[i];
3175 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
3176 (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
3177 if ((i + 1) < bp->b_npages)
3178 db_printf(",");
3179 }
3180 db_printf("\n");
3181 }
3182}
3183#endif /* DDB */
|