1/*
2 * Copyright (c) 1994,1997 John S. Dyson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Absolutely no warranty of function or purpose is made by the author
12 * John S. Dyson.
13 *
| 1/*
2 * Copyright (c) 1994,1997 John S. Dyson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Absolutely no warranty of function or purpose is made by the author
12 * John S. Dyson.
13 *
|
14 * $FreeBSD: head/sys/kern/vfs_bio.c 58934 2000-04-02 15:24:56Z phk $
| 14 * $FreeBSD: head/sys/kern/vfs_bio.c 59249 2000-04-15 05:54:02Z phk $
|
15 */
16
17/*
18 * this file contains a new buffer I/O scheme implementing a coherent
19 * VM object and buffer cache scheme. Pains have been taken to make
20 * sure that the performance degradation associated with schemes such
21 * as this is not realized.
22 *
23 * Author: John S. Dyson
24 * Significant help during the development and debugging phases
25 * had been provided by David Greenman, also of the FreeBSD core team.
26 *
27 * see man buf(9) for more info.
28 */
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/buf.h>
33#include <sys/conf.h>
34#include <sys/eventhandler.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mount.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/proc.h>
41#include <sys/reboot.h>
42#include <sys/resourcevar.h>
43#include <sys/sysctl.h>
44#include <sys/vmmeter.h>
45#include <sys/vnode.h>
46#include <vm/vm.h>
47#include <vm/vm_param.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_pageout.h>
50#include <vm/vm_page.h>
51#include <vm/vm_object.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_map.h>
54
55static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
56
57struct bio_ops bioops; /* I/O operation notification */
58
59struct buf *buf; /* buffer header pool */
60struct swqueue bswlist;
61
62static void vm_hold_free_pages(struct buf * bp, vm_offset_t from,
63 vm_offset_t to);
64static void vm_hold_load_pages(struct buf * bp, vm_offset_t from,
65 vm_offset_t to);
66static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off,
67 int pageno, vm_page_t m);
68static void vfs_clean_pages(struct buf * bp);
69static void vfs_setdirty(struct buf *bp);
70static void vfs_vmio_release(struct buf *bp);
71static void vfs_backgroundwritedone(struct buf *bp);
72static int flushbufqueues(void);
73
74static int bd_request;
75
76static void buf_daemon __P((void));
77/*
78 * bogus page -- for I/O to/from partially complete buffers
79 * this is a temporary solution to the problem, but it is not
80 * really that bad. it would be better to split the buffer
81 * for input in the case of buffers partially already in memory,
82 * but the code is intricate enough already.
83 */
84vm_page_t bogus_page;
85int runningbufspace;
86int vmiodirenable = FALSE;
87static vm_offset_t bogus_offset;
88
89static int bufspace, maxbufspace,
90 bufmallocspace, maxbufmallocspace, lobufspace, hibufspace;
91static int bufreusecnt, bufdefragcnt, buffreekvacnt;
92static int maxbdrun;
93static int needsbuffer;
94static int numdirtybuffers, hidirtybuffers;
95static int numfreebuffers, lofreebuffers, hifreebuffers;
96static int getnewbufcalls;
97static int getnewbufrestarts;
98
99SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD,
100 &numdirtybuffers, 0, "");
101SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW,
102 &hidirtybuffers, 0, "");
103SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD,
104 &numfreebuffers, 0, "");
105SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW,
106 &lofreebuffers, 0, "");
107SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW,
108 &hifreebuffers, 0, "");
109SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD,
110 &runningbufspace, 0, "");
111SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD,
112 &maxbufspace, 0, "");
113SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD,
114 &hibufspace, 0, "");
115SYSCTL_INT(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD,
116 &lobufspace, 0, "");
117SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD,
118 &bufspace, 0, "");
119SYSCTL_INT(_vfs, OID_AUTO, maxbdrun, CTLFLAG_RW,
120 &maxbdrun, 0, "");
121SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW,
122 &maxbufmallocspace, 0, "");
123SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD,
124 &bufmallocspace, 0, "");
125SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW,
126 &getnewbufcalls, 0, "");
127SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW,
128 &getnewbufrestarts, 0, "");
129SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW,
130 &vmiodirenable, 0, "");
131SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW,
132 &bufdefragcnt, 0, "");
133SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW,
134 &buffreekvacnt, 0, "");
135SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW,
136 &bufreusecnt, 0, "");
137
138static int bufhashmask;
139static LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
140struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } };
141char *buf_wmesg = BUF_WMESG;
142
143extern int vm_swap_size;
144
145#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
146#define VFS_BIO_NEED_DIRTYFLUSH 0x02 /* waiting for dirty buffer flush */
147#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
148#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
149
150/*
151 * Buffer hash table code. Note that the logical block scans linearly, which
152 * gives us some L1 cache locality.
153 */
154
155static __inline
156struct bufhashhdr *
157bufhash(struct vnode *vnp, daddr_t bn)
158{
159 return(&bufhashtbl[(((uintptr_t)(vnp) >> 7) + (int)bn) & bufhashmask]);
160}
161
162/*
163 * numdirtywakeup:
164 *
165 * If someone is blocked due to there being too many dirty buffers,
166 * and numdirtybuffers is now reasonable, wake them up.
167 */
168
169static __inline void
170numdirtywakeup(void)
171{
172 if (numdirtybuffers < hidirtybuffers) {
173 if (needsbuffer & VFS_BIO_NEED_DIRTYFLUSH) {
174 needsbuffer &= ~VFS_BIO_NEED_DIRTYFLUSH;
175 wakeup(&needsbuffer);
176 }
177 }
178}
179
180/*
181 * bufspacewakeup:
182 *
183 * Called when buffer space is potentially available for recovery.
184 * getnewbuf() will block on this flag when it is unable to free
185 * sufficient buffer space. Buffer space becomes recoverable when
186 * bp's get placed back in the queues.
187 */
188
189static __inline void
190bufspacewakeup(void)
191{
192 /*
193 * If someone is waiting for BUF space, wake them up. Even
194 * though we haven't freed the kva space yet, the waiting
195 * process will be able to now.
196 */
197 if (needsbuffer & VFS_BIO_NEED_BUFSPACE) {
198 needsbuffer &= ~VFS_BIO_NEED_BUFSPACE;
199 wakeup(&needsbuffer);
200 }
201}
202
203/*
204 * bufcountwakeup:
205 *
206 * Called when a buffer has been added to one of the free queues to
207 * account for the buffer and to wakeup anyone waiting for free buffers.
208 * This typically occurs when large amounts of metadata are being handled
209 * by the buffer cache ( else buffer space runs out first, usually ).
210 */
211
212static __inline void
213bufcountwakeup(void)
214{
215 ++numfreebuffers;
216 if (needsbuffer) {
217 needsbuffer &= ~VFS_BIO_NEED_ANY;
218 if (numfreebuffers >= hifreebuffers)
219 needsbuffer &= ~VFS_BIO_NEED_FREE;
220 wakeup(&needsbuffer);
221 }
222}
223
224/*
225 * vfs_buf_test_cache:
226 *
227 * Called when a buffer is extended. This function clears the B_CACHE
228 * bit if the newly extended portion of the buffer does not contain
229 * valid data.
230 */
231static __inline__
232void
233vfs_buf_test_cache(struct buf *bp,
234 vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
235 vm_page_t m)
236{
237 if (bp->b_flags & B_CACHE) {
238 int base = (foff + off) & PAGE_MASK;
239 if (vm_page_is_valid(m, base, size) == 0)
240 bp->b_flags &= ~B_CACHE;
241 }
242}
243
244static __inline__
245void
246bd_wakeup(int dirtybuflevel)
247{
248 if (numdirtybuffers >= dirtybuflevel && bd_request == 0) {
249 bd_request = 1;
250 wakeup(&bd_request);
251 }
252}
253
254/*
255 * bd_speedup - speedup the buffer cache flushing code
256 */
257
258static __inline__
259void
260bd_speedup(void)
261{
262 bd_wakeup(1);
263}
264
265/*
266 * Initialize buffer headers and related structures.
267 */
268
269caddr_t
270bufhashinit(caddr_t vaddr)
271{
272 /* first, make a null hash table */
273 for (bufhashmask = 8; bufhashmask < nbuf / 4; bufhashmask <<= 1)
274 ;
275 bufhashtbl = (void *)vaddr;
276 vaddr = vaddr + sizeof(*bufhashtbl) * bufhashmask;
277 --bufhashmask;
278 return(vaddr);
279}
280
281void
282bufinit(void)
283{
284 struct buf *bp;
285 int i;
286
287 TAILQ_INIT(&bswlist);
288 LIST_INIT(&invalhash);
289 simple_lock_init(&buftimelock);
290
291 for (i = 0; i <= bufhashmask; i++)
292 LIST_INIT(&bufhashtbl[i]);
293
294 /* next, make a null set of free lists */
295 for (i = 0; i < BUFFER_QUEUES; i++)
296 TAILQ_INIT(&bufqueues[i]);
297
298 /* finally, initialize each buffer header and stick on empty q */
299 for (i = 0; i < nbuf; i++) {
300 bp = &buf[i];
301 bzero(bp, sizeof *bp);
302 bp->b_flags = B_INVAL; /* we're just an empty header */
303 bp->b_dev = NODEV;
304 bp->b_rcred = NOCRED;
305 bp->b_wcred = NOCRED;
306 bp->b_qindex = QUEUE_EMPTY;
307 bp->b_xflags = 0;
308 LIST_INIT(&bp->b_dep);
309 BUF_LOCKINIT(bp);
310 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
311 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
312 }
313
314 /*
315 * maxbufspace is the absolute maximum amount of buffer space we are
316 * allowed to reserve in KVM and in real terms. The absolute maximum
317 * is nominally used by buf_daemon. hibufspace is the nominal maximum
318 * used by most other processes. The differential is required to
319 * ensure that buf_daemon is able to run when other processes might
320 * be blocked waiting for buffer space.
321 *
322 * maxbufspace is based on BKVASIZE. Allocating buffers larger then
323 * this may result in KVM fragmentation which is not handled optimally
324 * by the system.
325 */
326 maxbufspace = nbuf * BKVASIZE;
327 hibufspace = imax(3 * maxbufspace / 4, maxbufspace - MAXBSIZE * 10);
328 lobufspace = hibufspace - MAXBSIZE;
329
330/*
331 * Limit the amount of malloc memory since it is wired permanently into
332 * the kernel space. Even though this is accounted for in the buffer
333 * allocation, we don't want the malloced region to grow uncontrolled.
334 * The malloc scheme improves memory utilization significantly on average
335 * (small) directories.
336 */
337 maxbufmallocspace = hibufspace / 20;
338
339/*
340 * Reduce the chance of a deadlock occuring by limiting the number
341 * of delayed-write dirty buffers we allow to stack up.
342 */
343 hidirtybuffers = nbuf / 4 + 20;
344 numdirtybuffers = 0;
345/*
346 * To support extreme low-memory systems, make sure hidirtybuffers cannot
347 * eat up all available buffer space. This occurs when our minimum cannot
348 * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
349 * BKVASIZE'd (8K) buffers.
350 */
351 while (hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
352 hidirtybuffers >>= 1;
353 }
354
355/*
356 * Try to keep the number of free buffers in the specified range,
357 * and give special processes (e.g. like buf_daemon) access to an
358 * emergency reserve.
359 */
360 lofreebuffers = nbuf / 18 + 5;
361 hifreebuffers = 2 * lofreebuffers;
362 numfreebuffers = nbuf;
363
364/*
365 * Maximum number of async ops initiated per buf_daemon loop. This is
366 * somewhat of a hack at the moment, we really need to limit ourselves
367 * based on the number of bytes of I/O in-transit that were initiated
368 * from buf_daemon.
369 */
370 if ((maxbdrun = nswbuf / 4) < 4)
371 maxbdrun = 4;
372
373 bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
374 bogus_page = vm_page_alloc(kernel_object,
375 ((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
376 VM_ALLOC_NORMAL);
377 cnt.v_wire_count++;
378
379}
380
381/*
382 * bfreekva() - free the kva allocation for a buffer.
383 *
384 * Must be called at splbio() or higher as this is the only locking for
385 * buffer_map.
386 *
387 * Since this call frees up buffer space, we call bufspacewakeup().
388 */
389static void
390bfreekva(struct buf * bp)
391{
392 if (bp->b_kvasize) {
393 ++buffreekvacnt;
394 bufspace -= bp->b_kvasize;
395 vm_map_delete(buffer_map,
396 (vm_offset_t) bp->b_kvabase,
397 (vm_offset_t) bp->b_kvabase + bp->b_kvasize
398 );
399 bp->b_kvasize = 0;
400 bufspacewakeup();
401 }
402}
403
404/*
405 * bremfree:
406 *
407 * Remove the buffer from the appropriate free list.
408 */
409void
410bremfree(struct buf * bp)
411{
412 int s = splbio();
413 int old_qindex = bp->b_qindex;
414
415 if (bp->b_qindex != QUEUE_NONE) {
416 KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
417 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
418 bp->b_qindex = QUEUE_NONE;
419 runningbufspace += bp->b_bufsize;
420 } else {
421 if (BUF_REFCNT(bp) <= 1)
422 panic("bremfree: removing a buffer not on a queue");
423 }
424
425 /*
426 * Fixup numfreebuffers count. If the buffer is invalid or not
427 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
428 * the buffer was free and we must decrement numfreebuffers.
429 */
430 if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
431 switch(old_qindex) {
432 case QUEUE_DIRTY:
433 case QUEUE_CLEAN:
434 case QUEUE_EMPTY:
435 case QUEUE_EMPTYKVA:
436 --numfreebuffers;
437 break;
438 default:
439 break;
440 }
441 }
442 splx(s);
443}
444
445
446/*
447 * Get a buffer with the specified data. Look in the cache first. We
448 * must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
449 * is set, the buffer is valid and we do not have to do anything ( see
450 * getblk() ).
451 */
452int
453bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred,
454 struct buf ** bpp)
455{
456 struct buf *bp;
457
458 bp = getblk(vp, blkno, size, 0, 0);
459 *bpp = bp;
460
461 /* if not found in cache, do some I/O */
462 if ((bp->b_flags & B_CACHE) == 0) {
463 if (curproc != NULL)
464 curproc->p_stats->p_ru.ru_inblock++;
465 KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp));
466 bp->b_iocmd = BIO_READ;
467 bp->b_flags &= ~B_INVAL;
468 bp->b_ioflags &= ~BIO_ERROR;
469 if (bp->b_rcred == NOCRED) {
470 if (cred != NOCRED)
471 crhold(cred);
472 bp->b_rcred = cred;
473 }
474 vfs_busy_pages(bp, 0);
475 VOP_STRATEGY(vp, bp);
476 return (biowait(bp));
477 }
478 return (0);
479}
480
481/*
482 * Operates like bread, but also starts asynchronous I/O on
483 * read-ahead blocks. We must clear BIO_ERROR and B_INVAL prior
484 * to initiating I/O . If B_CACHE is set, the buffer is valid
485 * and we do not have to do anything.
486 */
487int
488breadn(struct vnode * vp, daddr_t blkno, int size,
489 daddr_t * rablkno, int *rabsize,
490 int cnt, struct ucred * cred, struct buf ** bpp)
491{
492 struct buf *bp, *rabp;
493 int i;
494 int rv = 0, readwait = 0;
495
496 *bpp = bp = getblk(vp, blkno, size, 0, 0);
497
498 /* if not found in cache, do some I/O */
499 if ((bp->b_flags & B_CACHE) == 0) {
500 if (curproc != NULL)
501 curproc->p_stats->p_ru.ru_inblock++;
502 bp->b_iocmd = BIO_READ;
503 bp->b_flags &= ~B_INVAL;
504 bp->b_ioflags &= ~BIO_ERROR;
505 if (bp->b_rcred == NOCRED) {
506 if (cred != NOCRED)
507 crhold(cred);
508 bp->b_rcred = cred;
509 }
510 vfs_busy_pages(bp, 0);
511 VOP_STRATEGY(vp, bp);
512 ++readwait;
513 }
514
515 for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
516 if (inmem(vp, *rablkno))
517 continue;
518 rabp = getblk(vp, *rablkno, *rabsize, 0, 0);
519
520 if ((rabp->b_flags & B_CACHE) == 0) {
521 if (curproc != NULL)
522 curproc->p_stats->p_ru.ru_inblock++;
523 rabp->b_flags |= B_ASYNC;
524 rabp->b_flags &= ~B_INVAL;
525 rabp->b_ioflags &= ~BIO_ERROR;
526 rabp->b_iocmd = BIO_READ;
527 if (rabp->b_rcred == NOCRED) {
528 if (cred != NOCRED)
529 crhold(cred);
530 rabp->b_rcred = cred;
531 }
532 vfs_busy_pages(rabp, 0);
533 BUF_KERNPROC(rabp);
534 VOP_STRATEGY(vp, rabp);
535 } else {
536 brelse(rabp);
537 }
538 }
539
540 if (readwait) {
541 rv = biowait(bp);
542 }
543 return (rv);
544}
545
546/*
547 * Write, release buffer on completion. (Done by iodone
548 * if async). Do not bother writing anything if the buffer
549 * is invalid.
550 *
551 * Note that we set B_CACHE here, indicating that buffer is
552 * fully valid and thus cacheable. This is true even of NFS
553 * now so we set it generally. This could be set either here
554 * or in biodone() since the I/O is synchronous. We put it
555 * here.
556 */
557int
558bwrite(struct buf * bp)
559{
560 int oldflags, s;
561 struct buf *newbp;
562
563 if (bp->b_flags & B_INVAL) {
564 brelse(bp);
565 return (0);
566 }
567
568 oldflags = bp->b_flags;
569
570 if (BUF_REFCNT(bp) == 0)
571 panic("bwrite: buffer is not busy???");
572 s = splbio();
573 /*
574 * If a background write is already in progress, delay
575 * writing this block if it is asynchronous. Otherwise
576 * wait for the background write to complete.
577 */
578 if (bp->b_xflags & BX_BKGRDINPROG) {
579 if (bp->b_flags & B_ASYNC) {
580 splx(s);
581 bdwrite(bp);
582 return (0);
583 }
584 bp->b_xflags |= BX_BKGRDWAIT;
585 tsleep(&bp->b_xflags, PRIBIO, "biord", 0);
586 if (bp->b_xflags & BX_BKGRDINPROG)
587 panic("bwrite: still writing");
588 }
589
590 /* Mark the buffer clean */
591 bundirty(bp);
592
593 /*
594 * If this buffer is marked for background writing and we
595 * do not have to wait for it, make a copy and write the
596 * copy so as to leave this buffer ready for further use.
597 */
598 if ((bp->b_xflags & BX_BKGRDWRITE) && (bp->b_flags & B_ASYNC)) {
599 if (bp->b_iodone != NULL) {
600 printf("bp->b_iodone = %p\n", bp->b_iodone);
601 panic("bwrite: need chained iodone");
602 }
603
604 /* get a new block */
605 newbp = geteblk(bp->b_bufsize);
606
607 /* set it to be identical to the old block */
608 memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
609 bgetvp(bp->b_vp, newbp);
610 newbp->b_lblkno = bp->b_lblkno;
611 newbp->b_blkno = bp->b_blkno;
612 newbp->b_offset = bp->b_offset;
613 newbp->b_iodone = vfs_backgroundwritedone;
614 newbp->b_flags |= B_ASYNC;
615 newbp->b_flags &= ~B_INVAL;
616
617 /* move over the dependencies */
618 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
619 (*bioops.io_movedeps)(bp, newbp);
620
621 /*
622 * Initiate write on the copy, release the original to
623 * the B_LOCKED queue so that it cannot go away until
624 * the background write completes. If not locked it could go
625 * away and then be reconstituted while it was being written.
626 * If the reconstituted buffer were written, we could end up
627 * with two background copies being written at the same time.
628 */
629 bp->b_xflags |= BX_BKGRDINPROG;
630 bp->b_flags |= B_LOCKED;
631 bqrelse(bp);
632 bp = newbp;
633 }
634
635 bp->b_flags &= ~B_DONE;
636 bp->b_ioflags &= ~BIO_ERROR;
637 bp->b_flags |= B_WRITEINPROG | B_CACHE;
638 bp->b_iocmd = BIO_WRITE;
639
640 bp->b_vp->v_numoutput++;
641 vfs_busy_pages(bp, 1);
642 if (curproc != NULL)
643 curproc->p_stats->p_ru.ru_oublock++;
644 splx(s);
645 if (oldflags & B_ASYNC)
646 BUF_KERNPROC(bp);
647 BUF_STRATEGY(bp);
648
649 if ((oldflags & B_ASYNC) == 0) {
650 int rtval = biowait(bp);
651 brelse(bp);
652 return (rtval);
653 }
654
655 return (0);
656}
657
658/*
659 * Complete a background write started from bwrite.
660 */
661static void
662vfs_backgroundwritedone(bp)
663 struct buf *bp;
664{
665 struct buf *origbp;
666
667 /*
668 * Find the original buffer that we are writing.
669 */
670 if ((origbp = gbincore(bp->b_vp, bp->b_lblkno)) == NULL)
671 panic("backgroundwritedone: lost buffer");
672 /*
673 * Process dependencies then return any unfinished ones.
674 */
675 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
676 (*bioops.io_complete)(bp);
677 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
678 (*bioops.io_movedeps)(bp, origbp);
679 /*
680 * Clear the BX_BKGRDINPROG flag in the original buffer
681 * and awaken it if it is waiting for the write to complete.
682 */
683 origbp->b_xflags &= ~BX_BKGRDINPROG;
684 if (origbp->b_xflags & BX_BKGRDWAIT) {
685 origbp->b_xflags &= ~BX_BKGRDWAIT;
686 wakeup(&origbp->b_xflags);
687 }
688 /*
689 * Clear the B_LOCKED flag and remove it from the locked
690 * queue if it currently resides there.
691 */
692 origbp->b_flags &= ~B_LOCKED;
693 if (BUF_LOCK(origbp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
694 bremfree(origbp);
695 bqrelse(origbp);
696 }
697 /*
698 * This buffer is marked B_NOCACHE, so when it is released
699 * by biodone, it will be tossed. We mark it with BIO_READ
700 * to avoid biodone doing a second vwakeup.
701 */
702 bp->b_flags |= B_NOCACHE;
703 bp->b_iocmd = BIO_READ;
704 bp->b_flags &= ~(B_CACHE | B_DONE);
705 bp->b_iodone = 0;
| 15 */
16
17/*
18 * this file contains a new buffer I/O scheme implementing a coherent
19 * VM object and buffer cache scheme. Pains have been taken to make
20 * sure that the performance degradation associated with schemes such
21 * as this is not realized.
22 *
23 * Author: John S. Dyson
24 * Significant help during the development and debugging phases
25 * had been provided by David Greenman, also of the FreeBSD core team.
26 *
27 * see man buf(9) for more info.
28 */
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/buf.h>
33#include <sys/conf.h>
34#include <sys/eventhandler.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mount.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/proc.h>
41#include <sys/reboot.h>
42#include <sys/resourcevar.h>
43#include <sys/sysctl.h>
44#include <sys/vmmeter.h>
45#include <sys/vnode.h>
46#include <vm/vm.h>
47#include <vm/vm_param.h>
48#include <vm/vm_kern.h>
49#include <vm/vm_pageout.h>
50#include <vm/vm_page.h>
51#include <vm/vm_object.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_map.h>
54
55static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
56
57struct bio_ops bioops; /* I/O operation notification */
58
59struct buf *buf; /* buffer header pool */
60struct swqueue bswlist;
61
62static void vm_hold_free_pages(struct buf * bp, vm_offset_t from,
63 vm_offset_t to);
64static void vm_hold_load_pages(struct buf * bp, vm_offset_t from,
65 vm_offset_t to);
66static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off,
67 int pageno, vm_page_t m);
68static void vfs_clean_pages(struct buf * bp);
69static void vfs_setdirty(struct buf *bp);
70static void vfs_vmio_release(struct buf *bp);
71static void vfs_backgroundwritedone(struct buf *bp);
72static int flushbufqueues(void);
73
74static int bd_request;
75
76static void buf_daemon __P((void));
77/*
78 * bogus page -- for I/O to/from partially complete buffers
79 * this is a temporary solution to the problem, but it is not
80 * really that bad. it would be better to split the buffer
81 * for input in the case of buffers partially already in memory,
82 * but the code is intricate enough already.
83 */
84vm_page_t bogus_page;
85int runningbufspace;
86int vmiodirenable = FALSE;
87static vm_offset_t bogus_offset;
88
89static int bufspace, maxbufspace,
90 bufmallocspace, maxbufmallocspace, lobufspace, hibufspace;
91static int bufreusecnt, bufdefragcnt, buffreekvacnt;
92static int maxbdrun;
93static int needsbuffer;
94static int numdirtybuffers, hidirtybuffers;
95static int numfreebuffers, lofreebuffers, hifreebuffers;
96static int getnewbufcalls;
97static int getnewbufrestarts;
98
99SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD,
100 &numdirtybuffers, 0, "");
101SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW,
102 &hidirtybuffers, 0, "");
103SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD,
104 &numfreebuffers, 0, "");
105SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW,
106 &lofreebuffers, 0, "");
107SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW,
108 &hifreebuffers, 0, "");
109SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD,
110 &runningbufspace, 0, "");
111SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD,
112 &maxbufspace, 0, "");
113SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD,
114 &hibufspace, 0, "");
115SYSCTL_INT(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD,
116 &lobufspace, 0, "");
117SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD,
118 &bufspace, 0, "");
119SYSCTL_INT(_vfs, OID_AUTO, maxbdrun, CTLFLAG_RW,
120 &maxbdrun, 0, "");
121SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW,
122 &maxbufmallocspace, 0, "");
123SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD,
124 &bufmallocspace, 0, "");
125SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW,
126 &getnewbufcalls, 0, "");
127SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW,
128 &getnewbufrestarts, 0, "");
129SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW,
130 &vmiodirenable, 0, "");
131SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW,
132 &bufdefragcnt, 0, "");
133SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW,
134 &buffreekvacnt, 0, "");
135SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW,
136 &bufreusecnt, 0, "");
137
138static int bufhashmask;
139static LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
140struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } };
141char *buf_wmesg = BUF_WMESG;
142
143extern int vm_swap_size;
144
145#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
146#define VFS_BIO_NEED_DIRTYFLUSH 0x02 /* waiting for dirty buffer flush */
147#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
148#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
149
150/*
151 * Buffer hash table code. Note that the logical block scans linearly, which
152 * gives us some L1 cache locality.
153 */
154
155static __inline
156struct bufhashhdr *
157bufhash(struct vnode *vnp, daddr_t bn)
158{
159 return(&bufhashtbl[(((uintptr_t)(vnp) >> 7) + (int)bn) & bufhashmask]);
160}
161
162/*
163 * numdirtywakeup:
164 *
165 * If someone is blocked due to there being too many dirty buffers,
166 * and numdirtybuffers is now reasonable, wake them up.
167 */
168
169static __inline void
170numdirtywakeup(void)
171{
172 if (numdirtybuffers < hidirtybuffers) {
173 if (needsbuffer & VFS_BIO_NEED_DIRTYFLUSH) {
174 needsbuffer &= ~VFS_BIO_NEED_DIRTYFLUSH;
175 wakeup(&needsbuffer);
176 }
177 }
178}
179
180/*
181 * bufspacewakeup:
182 *
183 * Called when buffer space is potentially available for recovery.
184 * getnewbuf() will block on this flag when it is unable to free
185 * sufficient buffer space. Buffer space becomes recoverable when
186 * bp's get placed back in the queues.
187 */
188
189static __inline void
190bufspacewakeup(void)
191{
192 /*
193 * If someone is waiting for BUF space, wake them up. Even
194 * though we haven't freed the kva space yet, the waiting
195 * process will be able to now.
196 */
197 if (needsbuffer & VFS_BIO_NEED_BUFSPACE) {
198 needsbuffer &= ~VFS_BIO_NEED_BUFSPACE;
199 wakeup(&needsbuffer);
200 }
201}
202
203/*
204 * bufcountwakeup:
205 *
206 * Called when a buffer has been added to one of the free queues to
207 * account for the buffer and to wakeup anyone waiting for free buffers.
208 * This typically occurs when large amounts of metadata are being handled
209 * by the buffer cache ( else buffer space runs out first, usually ).
210 */
211
212static __inline void
213bufcountwakeup(void)
214{
215 ++numfreebuffers;
216 if (needsbuffer) {
217 needsbuffer &= ~VFS_BIO_NEED_ANY;
218 if (numfreebuffers >= hifreebuffers)
219 needsbuffer &= ~VFS_BIO_NEED_FREE;
220 wakeup(&needsbuffer);
221 }
222}
223
224/*
225 * vfs_buf_test_cache:
226 *
227 * Called when a buffer is extended. This function clears the B_CACHE
228 * bit if the newly extended portion of the buffer does not contain
229 * valid data.
230 */
231static __inline__
232void
233vfs_buf_test_cache(struct buf *bp,
234 vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
235 vm_page_t m)
236{
237 if (bp->b_flags & B_CACHE) {
238 int base = (foff + off) & PAGE_MASK;
239 if (vm_page_is_valid(m, base, size) == 0)
240 bp->b_flags &= ~B_CACHE;
241 }
242}
243
244static __inline__
245void
246bd_wakeup(int dirtybuflevel)
247{
248 if (numdirtybuffers >= dirtybuflevel && bd_request == 0) {
249 bd_request = 1;
250 wakeup(&bd_request);
251 }
252}
253
254/*
255 * bd_speedup - speedup the buffer cache flushing code
256 */
257
258static __inline__
259void
260bd_speedup(void)
261{
262 bd_wakeup(1);
263}
264
265/*
266 * Initialize buffer headers and related structures.
267 */
268
269caddr_t
270bufhashinit(caddr_t vaddr)
271{
272 /* first, make a null hash table */
273 for (bufhashmask = 8; bufhashmask < nbuf / 4; bufhashmask <<= 1)
274 ;
275 bufhashtbl = (void *)vaddr;
276 vaddr = vaddr + sizeof(*bufhashtbl) * bufhashmask;
277 --bufhashmask;
278 return(vaddr);
279}
280
281void
282bufinit(void)
283{
284 struct buf *bp;
285 int i;
286
287 TAILQ_INIT(&bswlist);
288 LIST_INIT(&invalhash);
289 simple_lock_init(&buftimelock);
290
291 for (i = 0; i <= bufhashmask; i++)
292 LIST_INIT(&bufhashtbl[i]);
293
294 /* next, make a null set of free lists */
295 for (i = 0; i < BUFFER_QUEUES; i++)
296 TAILQ_INIT(&bufqueues[i]);
297
298 /* finally, initialize each buffer header and stick on empty q */
299 for (i = 0; i < nbuf; i++) {
300 bp = &buf[i];
301 bzero(bp, sizeof *bp);
302 bp->b_flags = B_INVAL; /* we're just an empty header */
303 bp->b_dev = NODEV;
304 bp->b_rcred = NOCRED;
305 bp->b_wcred = NOCRED;
306 bp->b_qindex = QUEUE_EMPTY;
307 bp->b_xflags = 0;
308 LIST_INIT(&bp->b_dep);
309 BUF_LOCKINIT(bp);
310 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
311 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
312 }
313
314 /*
315 * maxbufspace is the absolute maximum amount of buffer space we are
316 * allowed to reserve in KVM and in real terms. The absolute maximum
317 * is nominally used by buf_daemon. hibufspace is the nominal maximum
318 * used by most other processes. The differential is required to
319 * ensure that buf_daemon is able to run when other processes might
320 * be blocked waiting for buffer space.
321 *
322 * maxbufspace is based on BKVASIZE. Allocating buffers larger then
323 * this may result in KVM fragmentation which is not handled optimally
324 * by the system.
325 */
326 maxbufspace = nbuf * BKVASIZE;
327 hibufspace = imax(3 * maxbufspace / 4, maxbufspace - MAXBSIZE * 10);
328 lobufspace = hibufspace - MAXBSIZE;
329
330/*
331 * Limit the amount of malloc memory since it is wired permanently into
332 * the kernel space. Even though this is accounted for in the buffer
333 * allocation, we don't want the malloced region to grow uncontrolled.
334 * The malloc scheme improves memory utilization significantly on average
335 * (small) directories.
336 */
337 maxbufmallocspace = hibufspace / 20;
338
339/*
340 * Reduce the chance of a deadlock occuring by limiting the number
341 * of delayed-write dirty buffers we allow to stack up.
342 */
343 hidirtybuffers = nbuf / 4 + 20;
344 numdirtybuffers = 0;
345/*
346 * To support extreme low-memory systems, make sure hidirtybuffers cannot
347 * eat up all available buffer space. This occurs when our minimum cannot
348 * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
349 * BKVASIZE'd (8K) buffers.
350 */
351 while (hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
352 hidirtybuffers >>= 1;
353 }
354
355/*
356 * Try to keep the number of free buffers in the specified range,
357 * and give special processes (e.g. like buf_daemon) access to an
358 * emergency reserve.
359 */
360 lofreebuffers = nbuf / 18 + 5;
361 hifreebuffers = 2 * lofreebuffers;
362 numfreebuffers = nbuf;
363
364/*
365 * Maximum number of async ops initiated per buf_daemon loop. This is
366 * somewhat of a hack at the moment, we really need to limit ourselves
367 * based on the number of bytes of I/O in-transit that were initiated
368 * from buf_daemon.
369 */
370 if ((maxbdrun = nswbuf / 4) < 4)
371 maxbdrun = 4;
372
373 bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
374 bogus_page = vm_page_alloc(kernel_object,
375 ((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
376 VM_ALLOC_NORMAL);
377 cnt.v_wire_count++;
378
379}
380
381/*
382 * bfreekva() - free the kva allocation for a buffer.
383 *
384 * Must be called at splbio() or higher as this is the only locking for
385 * buffer_map.
386 *
387 * Since this call frees up buffer space, we call bufspacewakeup().
388 */
389static void
390bfreekva(struct buf * bp)
391{
392 if (bp->b_kvasize) {
393 ++buffreekvacnt;
394 bufspace -= bp->b_kvasize;
395 vm_map_delete(buffer_map,
396 (vm_offset_t) bp->b_kvabase,
397 (vm_offset_t) bp->b_kvabase + bp->b_kvasize
398 );
399 bp->b_kvasize = 0;
400 bufspacewakeup();
401 }
402}
403
404/*
405 * bremfree:
406 *
407 * Remove the buffer from the appropriate free list.
408 */
409void
410bremfree(struct buf * bp)
411{
412 int s = splbio();
413 int old_qindex = bp->b_qindex;
414
415 if (bp->b_qindex != QUEUE_NONE) {
416 KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
417 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
418 bp->b_qindex = QUEUE_NONE;
419 runningbufspace += bp->b_bufsize;
420 } else {
421 if (BUF_REFCNT(bp) <= 1)
422 panic("bremfree: removing a buffer not on a queue");
423 }
424
425 /*
426 * Fixup numfreebuffers count. If the buffer is invalid or not
427 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
428 * the buffer was free and we must decrement numfreebuffers.
429 */
430 if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
431 switch(old_qindex) {
432 case QUEUE_DIRTY:
433 case QUEUE_CLEAN:
434 case QUEUE_EMPTY:
435 case QUEUE_EMPTYKVA:
436 --numfreebuffers;
437 break;
438 default:
439 break;
440 }
441 }
442 splx(s);
443}
444
445
446/*
447 * Get a buffer with the specified data. Look in the cache first. We
448 * must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
449 * is set, the buffer is valid and we do not have to do anything ( see
450 * getblk() ).
451 */
452int
453bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred,
454 struct buf ** bpp)
455{
456 struct buf *bp;
457
458 bp = getblk(vp, blkno, size, 0, 0);
459 *bpp = bp;
460
461 /* if not found in cache, do some I/O */
462 if ((bp->b_flags & B_CACHE) == 0) {
463 if (curproc != NULL)
464 curproc->p_stats->p_ru.ru_inblock++;
465 KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp));
466 bp->b_iocmd = BIO_READ;
467 bp->b_flags &= ~B_INVAL;
468 bp->b_ioflags &= ~BIO_ERROR;
469 if (bp->b_rcred == NOCRED) {
470 if (cred != NOCRED)
471 crhold(cred);
472 bp->b_rcred = cred;
473 }
474 vfs_busy_pages(bp, 0);
475 VOP_STRATEGY(vp, bp);
476 return (biowait(bp));
477 }
478 return (0);
479}
480
481/*
482 * Operates like bread, but also starts asynchronous I/O on
483 * read-ahead blocks. We must clear BIO_ERROR and B_INVAL prior
484 * to initiating I/O . If B_CACHE is set, the buffer is valid
485 * and we do not have to do anything.
486 */
487int
488breadn(struct vnode * vp, daddr_t blkno, int size,
489 daddr_t * rablkno, int *rabsize,
490 int cnt, struct ucred * cred, struct buf ** bpp)
491{
492 struct buf *bp, *rabp;
493 int i;
494 int rv = 0, readwait = 0;
495
496 *bpp = bp = getblk(vp, blkno, size, 0, 0);
497
498 /* if not found in cache, do some I/O */
499 if ((bp->b_flags & B_CACHE) == 0) {
500 if (curproc != NULL)
501 curproc->p_stats->p_ru.ru_inblock++;
502 bp->b_iocmd = BIO_READ;
503 bp->b_flags &= ~B_INVAL;
504 bp->b_ioflags &= ~BIO_ERROR;
505 if (bp->b_rcred == NOCRED) {
506 if (cred != NOCRED)
507 crhold(cred);
508 bp->b_rcred = cred;
509 }
510 vfs_busy_pages(bp, 0);
511 VOP_STRATEGY(vp, bp);
512 ++readwait;
513 }
514
515 for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
516 if (inmem(vp, *rablkno))
517 continue;
518 rabp = getblk(vp, *rablkno, *rabsize, 0, 0);
519
520 if ((rabp->b_flags & B_CACHE) == 0) {
521 if (curproc != NULL)
522 curproc->p_stats->p_ru.ru_inblock++;
523 rabp->b_flags |= B_ASYNC;
524 rabp->b_flags &= ~B_INVAL;
525 rabp->b_ioflags &= ~BIO_ERROR;
526 rabp->b_iocmd = BIO_READ;
527 if (rabp->b_rcred == NOCRED) {
528 if (cred != NOCRED)
529 crhold(cred);
530 rabp->b_rcred = cred;
531 }
532 vfs_busy_pages(rabp, 0);
533 BUF_KERNPROC(rabp);
534 VOP_STRATEGY(vp, rabp);
535 } else {
536 brelse(rabp);
537 }
538 }
539
540 if (readwait) {
541 rv = biowait(bp);
542 }
543 return (rv);
544}
545
546/*
547 * Write, release buffer on completion. (Done by iodone
548 * if async). Do not bother writing anything if the buffer
549 * is invalid.
550 *
551 * Note that we set B_CACHE here, indicating that buffer is
552 * fully valid and thus cacheable. This is true even of NFS
553 * now so we set it generally. This could be set either here
554 * or in biodone() since the I/O is synchronous. We put it
555 * here.
556 */
557int
558bwrite(struct buf * bp)
559{
560 int oldflags, s;
561 struct buf *newbp;
562
563 if (bp->b_flags & B_INVAL) {
564 brelse(bp);
565 return (0);
566 }
567
568 oldflags = bp->b_flags;
569
570 if (BUF_REFCNT(bp) == 0)
571 panic("bwrite: buffer is not busy???");
572 s = splbio();
573 /*
574 * If a background write is already in progress, delay
575 * writing this block if it is asynchronous. Otherwise
576 * wait for the background write to complete.
577 */
578 if (bp->b_xflags & BX_BKGRDINPROG) {
579 if (bp->b_flags & B_ASYNC) {
580 splx(s);
581 bdwrite(bp);
582 return (0);
583 }
584 bp->b_xflags |= BX_BKGRDWAIT;
585 tsleep(&bp->b_xflags, PRIBIO, "biord", 0);
586 if (bp->b_xflags & BX_BKGRDINPROG)
587 panic("bwrite: still writing");
588 }
589
590 /* Mark the buffer clean */
591 bundirty(bp);
592
593 /*
594 * If this buffer is marked for background writing and we
595 * do not have to wait for it, make a copy and write the
596 * copy so as to leave this buffer ready for further use.
597 */
598 if ((bp->b_xflags & BX_BKGRDWRITE) && (bp->b_flags & B_ASYNC)) {
599 if (bp->b_iodone != NULL) {
600 printf("bp->b_iodone = %p\n", bp->b_iodone);
601 panic("bwrite: need chained iodone");
602 }
603
604 /* get a new block */
605 newbp = geteblk(bp->b_bufsize);
606
607 /* set it to be identical to the old block */
608 memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
609 bgetvp(bp->b_vp, newbp);
610 newbp->b_lblkno = bp->b_lblkno;
611 newbp->b_blkno = bp->b_blkno;
612 newbp->b_offset = bp->b_offset;
613 newbp->b_iodone = vfs_backgroundwritedone;
614 newbp->b_flags |= B_ASYNC;
615 newbp->b_flags &= ~B_INVAL;
616
617 /* move over the dependencies */
618 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
619 (*bioops.io_movedeps)(bp, newbp);
620
621 /*
622 * Initiate write on the copy, release the original to
623 * the B_LOCKED queue so that it cannot go away until
624 * the background write completes. If not locked it could go
625 * away and then be reconstituted while it was being written.
626 * If the reconstituted buffer were written, we could end up
627 * with two background copies being written at the same time.
628 */
629 bp->b_xflags |= BX_BKGRDINPROG;
630 bp->b_flags |= B_LOCKED;
631 bqrelse(bp);
632 bp = newbp;
633 }
634
635 bp->b_flags &= ~B_DONE;
636 bp->b_ioflags &= ~BIO_ERROR;
637 bp->b_flags |= B_WRITEINPROG | B_CACHE;
638 bp->b_iocmd = BIO_WRITE;
639
640 bp->b_vp->v_numoutput++;
641 vfs_busy_pages(bp, 1);
642 if (curproc != NULL)
643 curproc->p_stats->p_ru.ru_oublock++;
644 splx(s);
645 if (oldflags & B_ASYNC)
646 BUF_KERNPROC(bp);
647 BUF_STRATEGY(bp);
648
649 if ((oldflags & B_ASYNC) == 0) {
650 int rtval = biowait(bp);
651 brelse(bp);
652 return (rtval);
653 }
654
655 return (0);
656}
657
658/*
659 * Complete a background write started from bwrite.
660 */
661static void
662vfs_backgroundwritedone(bp)
663 struct buf *bp;
664{
665 struct buf *origbp;
666
667 /*
668 * Find the original buffer that we are writing.
669 */
670 if ((origbp = gbincore(bp->b_vp, bp->b_lblkno)) == NULL)
671 panic("backgroundwritedone: lost buffer");
672 /*
673 * Process dependencies then return any unfinished ones.
674 */
675 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
676 (*bioops.io_complete)(bp);
677 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_movedeps)
678 (*bioops.io_movedeps)(bp, origbp);
679 /*
680 * Clear the BX_BKGRDINPROG flag in the original buffer
681 * and awaken it if it is waiting for the write to complete.
682 */
683 origbp->b_xflags &= ~BX_BKGRDINPROG;
684 if (origbp->b_xflags & BX_BKGRDWAIT) {
685 origbp->b_xflags &= ~BX_BKGRDWAIT;
686 wakeup(&origbp->b_xflags);
687 }
688 /*
689 * Clear the B_LOCKED flag and remove it from the locked
690 * queue if it currently resides there.
691 */
692 origbp->b_flags &= ~B_LOCKED;
693 if (BUF_LOCK(origbp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
694 bremfree(origbp);
695 bqrelse(origbp);
696 }
697 /*
698 * This buffer is marked B_NOCACHE, so when it is released
699 * by biodone, it will be tossed. We mark it with BIO_READ
700 * to avoid biodone doing a second vwakeup.
701 */
702 bp->b_flags |= B_NOCACHE;
703 bp->b_iocmd = BIO_READ;
704 bp->b_flags &= ~(B_CACHE | B_DONE);
705 bp->b_iodone = 0;
|
706 biodone(bp);
| 706 bufdone(bp);
|
707}
708
709/*
710 * Delayed write. (Buffer is marked dirty). Do not bother writing
711 * anything if the buffer is marked invalid.
712 *
713 * Note that since the buffer must be completely valid, we can safely
714 * set B_CACHE. In fact, we have to set B_CACHE here rather then in
715 * biodone() in order to prevent getblk from writing the buffer
716 * out synchronously.
717 */
718void
719bdwrite(struct buf * bp)
720{
721 if (BUF_REFCNT(bp) == 0)
722 panic("bdwrite: buffer is not busy");
723
724 if (bp->b_flags & B_INVAL) {
725 brelse(bp);
726 return;
727 }
728 bdirty(bp);
729
730 /*
731 * Set B_CACHE, indicating that the buffer is fully valid. This is
732 * true even of NFS now.
733 */
734 bp->b_flags |= B_CACHE;
735
736 /*
737 * This bmap keeps the system from needing to do the bmap later,
738 * perhaps when the system is attempting to do a sync. Since it
739 * is likely that the indirect block -- or whatever other datastructure
740 * that the filesystem needs is still in memory now, it is a good
741 * thing to do this. Note also, that if the pageout daemon is
742 * requesting a sync -- there might not be enough memory to do
743 * the bmap then... So, this is important to do.
744 */
745 if (bp->b_lblkno == bp->b_blkno) {
746 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
747 }
748
749 /*
750 * Set the *dirty* buffer range based upon the VM system dirty pages.
751 */
752 vfs_setdirty(bp);
753
754 /*
755 * We need to do this here to satisfy the vnode_pager and the
756 * pageout daemon, so that it thinks that the pages have been
757 * "cleaned". Note that since the pages are in a delayed write
758 * buffer -- the VFS layer "will" see that the pages get written
759 * out on the next sync, or perhaps the cluster will be completed.
760 */
761 vfs_clean_pages(bp);
762 bqrelse(bp);
763
764 /*
765 * Wakeup the buffer flushing daemon if we have saturated the
766 * buffer cache.
767 */
768
769 bd_wakeup(hidirtybuffers);
770
771 /*
772 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
773 * due to the softdep code.
774 */
775}
776
777/*
778 * bdirty:
779 *
780 * Turn buffer into delayed write request. We must clear BIO_READ and
781 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
782 * itself to properly update it in the dirty/clean lists. We mark it
783 * B_DONE to ensure that any asynchronization of the buffer properly
784 * clears B_DONE ( else a panic will occur later ).
785 *
786 * bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
787 * might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
788 * should only be called if the buffer is known-good.
789 *
790 * Since the buffer is not on a queue, we do not update the numfreebuffers
791 * count.
792 *
793 * Must be called at splbio().
794 * The buffer must be on QUEUE_NONE.
795 */
796void
797bdirty(bp)
798 struct buf *bp;
799{
800 KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
801 bp->b_flags &= ~(B_RELBUF);
802 bp->b_iocmd = BIO_WRITE;
803
804 if ((bp->b_flags & B_DELWRI) == 0) {
805 bp->b_flags |= B_DONE | B_DELWRI;
806 reassignbuf(bp, bp->b_vp);
807 ++numdirtybuffers;
808 bd_wakeup(hidirtybuffers);
809 }
810}
811
812/*
813 * bundirty:
814 *
815 * Clear B_DELWRI for buffer.
816 *
817 * Since the buffer is not on a queue, we do not update the numfreebuffers
818 * count.
819 *
820 * Must be called at splbio().
821 * The buffer must be on QUEUE_NONE.
822 */
823
824void
825bundirty(bp)
826 struct buf *bp;
827{
828 KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
829
830 if (bp->b_flags & B_DELWRI) {
831 bp->b_flags &= ~B_DELWRI;
832 reassignbuf(bp, bp->b_vp);
833 --numdirtybuffers;
834 numdirtywakeup();
835 }
836 /*
837 * Since it is now being written, we can clear its deferred write flag.
838 */
839 bp->b_flags &= ~B_DEFERRED;
840}
841
842/*
843 * bawrite:
844 *
845 * Asynchronous write. Start output on a buffer, but do not wait for
846 * it to complete. The buffer is released when the output completes.
847 *
848 * bwrite() ( or the VOP routine anyway ) is responsible for handling
849 * B_INVAL buffers. Not us.
850 */
851void
852bawrite(struct buf * bp)
853{
854 bp->b_flags |= B_ASYNC;
855 (void) BUF_WRITE(bp);
856}
857
858/*
859 * bowrite:
860 *
861 * Ordered write. Start output on a buffer, and flag it so that the
862 * device will write it in the order it was queued. The buffer is
863 * released when the output completes. bwrite() ( or the VOP routine
864 * anyway ) is responsible for handling B_INVAL buffers.
865 */
866int
867bowrite(struct buf * bp)
868{
869 bp->b_ioflags |= BIO_ORDERED;
870 bp->b_flags |= B_ASYNC;
871 return (BUF_WRITE(bp));
872}
873
874/*
875 * bwillwrite:
876 *
877 * Called prior to the locking of any vnodes when we are expecting to
878 * write. We do not want to starve the buffer cache with too many
879 * dirty buffers so we block here. By blocking prior to the locking
880 * of any vnodes we attempt to avoid the situation where a locked vnode
881 * prevents the various system daemons from flushing related buffers.
882 */
883
884void
885bwillwrite(void)
886{
887 int slop = hidirtybuffers / 10;
888
889 if (numdirtybuffers > hidirtybuffers + slop) {
890 int s;
891
892 s = splbio();
893 while (numdirtybuffers > hidirtybuffers) {
894 bd_wakeup(hidirtybuffers);
895 needsbuffer |= VFS_BIO_NEED_DIRTYFLUSH;
896 tsleep(&needsbuffer, (PRIBIO + 4), "flswai", 0);
897 }
898 splx(s);
899 }
900}
901
902/*
903 * brelse:
904 *
905 * Release a busy buffer and, if requested, free its resources. The
906 * buffer will be stashed in the appropriate bufqueue[] allowing it
907 * to be accessed later as a cache entity or reused for other purposes.
908 */
909void
910brelse(struct buf * bp)
911{
912 int s;
913
914 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
915
916 s = splbio();
917
918 if (bp->b_flags & B_LOCKED)
919 bp->b_ioflags &= ~BIO_ERROR;
920
921 if (bp->b_iocmd == BIO_WRITE &&
922 (bp->b_ioflags & BIO_ERROR) &&
923 !(bp->b_flags & B_INVAL)) {
924 /*
925 * Failed write, redirty. Must clear BIO_ERROR to prevent
926 * pages from being scrapped. If B_INVAL is set then
927 * this case is not run and the next case is run to
928 * destroy the buffer. B_INVAL can occur if the buffer
929 * is outside the range supported by the underlying device.
930 */
931 bp->b_ioflags &= ~BIO_ERROR;
932 bdirty(bp);
933 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
934 (bp->b_ioflags & BIO_ERROR) ||
935 bp->b_iocmd == BIO_DELETE || (bp->b_bufsize <= 0)) {
936 /*
937 * Either a failed I/O or we were asked to free or not
938 * cache the buffer.
939 */
940 bp->b_flags |= B_INVAL;
941 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
942 (*bioops.io_deallocate)(bp);
943 if (bp->b_flags & B_DELWRI) {
944 --numdirtybuffers;
945 numdirtywakeup();
946 }
947 bp->b_flags &= ~(B_DELWRI | B_CACHE);
948 if ((bp->b_flags & B_VMIO) == 0) {
949 if (bp->b_bufsize)
950 allocbuf(bp, 0);
951 if (bp->b_vp)
952 brelvp(bp);
953 }
954 }
955
956 /*
957 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release()
958 * is called with B_DELWRI set, the underlying pages may wind up
959 * getting freed causing a previous write (bdwrite()) to get 'lost'
960 * because pages associated with a B_DELWRI bp are marked clean.
961 *
962 * We still allow the B_INVAL case to call vfs_vmio_release(), even
963 * if B_DELWRI is set.
964 */
965
966 if (bp->b_flags & B_DELWRI)
967 bp->b_flags &= ~B_RELBUF;
968
969 /*
970 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
971 * constituted, not even NFS buffers now. Two flags effect this. If
972 * B_INVAL, the struct buf is invalidated but the VM object is kept
973 * around ( i.e. so it is trivial to reconstitute the buffer later ).
974 *
975 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
976 * invalidated. BIO_ERROR cannot be set for a failed write unless the
977 * buffer is also B_INVAL because it hits the re-dirtying code above.
978 *
979 * Normally we can do this whether a buffer is B_DELWRI or not. If
980 * the buffer is an NFS buffer, it is tracking piecemeal writes or
981 * the commit state and we cannot afford to lose the buffer. If the
982 * buffer has a background write in progress, we need to keep it
983 * around to prevent it from being reconstituted and starting a second
984 * background write.
985 */
986 if ((bp->b_flags & B_VMIO)
987 && !(bp->b_vp->v_tag == VT_NFS &&
988 !vn_isdisk(bp->b_vp, NULL) &&
989 (bp->b_flags & B_DELWRI) &&
990 (bp->b_xflags & BX_BKGRDINPROG))
991 ) {
992
993 int i, j, resid;
994 vm_page_t m;
995 off_t foff;
996 vm_pindex_t poff;
997 vm_object_t obj;
998 struct vnode *vp;
999
1000 vp = bp->b_vp;
1001
1002 /*
1003 * Get the base offset and length of the buffer. Note that
1004 * for block sizes that are less then PAGE_SIZE, the b_data
1005 * base of the buffer does not represent exactly b_offset and
1006 * neither b_offset nor b_size are necessarily page aligned.
1007 * Instead, the starting position of b_offset is:
1008 *
1009 * b_data + (b_offset & PAGE_MASK)
1010 *
1011 * block sizes less then DEV_BSIZE (usually 512) are not
1012 * supported due to the page granularity bits (m->valid,
1013 * m->dirty, etc...).
1014 *
1015 * See man buf(9) for more information
1016 */
1017
1018 resid = bp->b_bufsize;
1019 foff = bp->b_offset;
1020
1021 for (i = 0; i < bp->b_npages; i++) {
1022 m = bp->b_pages[i];
1023 vm_page_flag_clear(m, PG_ZERO);
1024 if (m == bogus_page) {
1025
1026 obj = (vm_object_t) vp->v_object;
1027 poff = OFF_TO_IDX(bp->b_offset);
1028
1029 for (j = i; j < bp->b_npages; j++) {
1030 m = bp->b_pages[j];
1031 if (m == bogus_page) {
1032 m = vm_page_lookup(obj, poff + j);
1033 if (!m) {
1034 panic("brelse: page missing\n");
1035 }
1036 bp->b_pages[j] = m;
1037 }
1038 }
1039
1040 if ((bp->b_flags & B_INVAL) == 0) {
1041 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
1042 }
1043 }
1044 if ((bp->b_flags & B_NOCACHE) || (bp->b_ioflags & BIO_ERROR)) {
1045 int poffset = foff & PAGE_MASK;
1046 int presid = resid > (PAGE_SIZE - poffset) ?
1047 (PAGE_SIZE - poffset) : resid;
1048
1049 KASSERT(presid >= 0, ("brelse: extra page"));
1050 vm_page_set_invalid(m, poffset, presid);
1051 }
1052 resid -= PAGE_SIZE - (foff & PAGE_MASK);
1053 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
1054 }
1055
1056 if (bp->b_flags & (B_INVAL | B_RELBUF))
1057 vfs_vmio_release(bp);
1058
1059 } else if (bp->b_flags & B_VMIO) {
1060
1061 if (bp->b_flags & (B_INVAL | B_RELBUF))
1062 vfs_vmio_release(bp);
1063
1064 }
1065
1066 if (bp->b_qindex != QUEUE_NONE)
1067 panic("brelse: free buffer onto another queue???");
1068 if (BUF_REFCNT(bp) > 1) {
1069 /* Temporary panic to verify exclusive locking */
1070 /* This panic goes away when we allow shared refs */
1071 panic("brelse: multiple refs");
1072 /* do not release to free list */
1073 BUF_UNLOCK(bp);
1074 splx(s);
1075 return;
1076 }
1077
1078 /* enqueue */
1079
1080 /* buffers with no memory */
1081 if (bp->b_bufsize == 0) {
1082 bp->b_flags |= B_INVAL;
1083 bp->b_xflags &= ~BX_BKGRDWRITE;
1084 if (bp->b_xflags & BX_BKGRDINPROG)
1085 panic("losing buffer 1");
1086 if (bp->b_kvasize) {
1087 bp->b_qindex = QUEUE_EMPTYKVA;
1088 } else {
1089 bp->b_qindex = QUEUE_EMPTY;
1090 }
1091 TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1092 LIST_REMOVE(bp, b_hash);
1093 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1094 bp->b_dev = NODEV;
1095 /* buffers with junk contents */
1096 } else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) || (bp->b_ioflags & BIO_ERROR)) {
1097 bp->b_flags |= B_INVAL;
1098 bp->b_xflags &= ~BX_BKGRDWRITE;
1099 if (bp->b_xflags & BX_BKGRDINPROG)
1100 panic("losing buffer 2");
1101 bp->b_qindex = QUEUE_CLEAN;
1102 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1103 LIST_REMOVE(bp, b_hash);
1104 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1105 bp->b_dev = NODEV;
1106
1107 /* buffers that are locked */
1108 } else if (bp->b_flags & B_LOCKED) {
1109 bp->b_qindex = QUEUE_LOCKED;
1110 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1111
1112 /* remaining buffers */
1113 } else {
1114 switch(bp->b_flags & (B_DELWRI|B_AGE)) {
1115 case B_DELWRI | B_AGE:
1116 bp->b_qindex = QUEUE_DIRTY;
1117 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1118 break;
1119 case B_DELWRI:
1120 bp->b_qindex = QUEUE_DIRTY;
1121 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1122 break;
1123 case B_AGE:
1124 bp->b_qindex = QUEUE_CLEAN;
1125 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1126 break;
1127 default:
1128 bp->b_qindex = QUEUE_CLEAN;
1129 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1130 break;
1131 }
1132 }
1133
1134 /*
1135 * If B_INVAL, clear B_DELWRI. We've already placed the buffer
1136 * on the correct queue.
1137 */
1138 if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) {
1139 bp->b_flags &= ~B_DELWRI;
1140 --numdirtybuffers;
1141 numdirtywakeup();
1142 }
1143
1144 runningbufspace -= bp->b_bufsize;
1145
1146 /*
1147 * Fixup numfreebuffers count. The bp is on an appropriate queue
1148 * unless locked. We then bump numfreebuffers if it is not B_DELWRI.
1149 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1150 * if B_INVAL is set ).
1151 */
1152
1153 if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI))
1154 bufcountwakeup();
1155
1156 /*
1157 * Something we can maybe free.
1158 */
1159
1160 if (bp->b_bufsize || bp->b_kvasize)
1161 bufspacewakeup();
1162
1163 /* unlock */
1164 BUF_UNLOCK(bp);
1165 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1166 bp->b_ioflags &= ~BIO_ORDERED;
1167 splx(s);
1168}
1169
1170/*
1171 * Release a buffer back to the appropriate queue but do not try to free
1172 * it.
1173 *
1174 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1175 * biodone() to requeue an async I/O on completion. It is also used when
1176 * known good buffers need to be requeued but we think we may need the data
1177 * again soon.
1178 */
1179void
1180bqrelse(struct buf * bp)
1181{
1182 int s;
1183
1184 s = splbio();
1185
1186 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1187
1188 if (bp->b_qindex != QUEUE_NONE)
1189 panic("bqrelse: free buffer onto another queue???");
1190 if (BUF_REFCNT(bp) > 1) {
1191 /* do not release to free list */
1192 panic("bqrelse: multiple refs");
1193 BUF_UNLOCK(bp);
1194 splx(s);
1195 return;
1196 }
1197 if (bp->b_flags & B_LOCKED) {
1198 bp->b_ioflags &= ~BIO_ERROR;
1199 bp->b_qindex = QUEUE_LOCKED;
1200 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1201 /* buffers with stale but valid contents */
1202 } else if (bp->b_flags & B_DELWRI) {
1203 bp->b_qindex = QUEUE_DIRTY;
1204 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1205 } else {
1206 bp->b_qindex = QUEUE_CLEAN;
1207 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1208 }
1209
1210 runningbufspace -= bp->b_bufsize;
1211
1212 if ((bp->b_flags & B_LOCKED) == 0 &&
1213 ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))) {
1214 bufcountwakeup();
1215 }
1216
1217 /*
1218 * Something we can maybe wakeup
1219 */
1220 if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1221 bufspacewakeup();
1222
1223 /* unlock */
1224 BUF_UNLOCK(bp);
1225 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1226 bp->b_ioflags &= ~BIO_ORDERED;
1227 splx(s);
1228}
1229
1230static void
1231vfs_vmio_release(bp)
1232 struct buf *bp;
1233{
1234 int i, s;
1235 vm_page_t m;
1236
1237 s = splvm();
1238 for (i = 0; i < bp->b_npages; i++) {
1239 m = bp->b_pages[i];
1240 bp->b_pages[i] = NULL;
1241 /*
1242 * In order to keep page LRU ordering consistent, put
1243 * everything on the inactive queue.
1244 */
1245 vm_page_unwire(m, 0);
1246 /*
1247 * We don't mess with busy pages, it is
1248 * the responsibility of the process that
1249 * busied the pages to deal with them.
1250 */
1251 if ((m->flags & PG_BUSY) || (m->busy != 0))
1252 continue;
1253
1254 if (m->wire_count == 0) {
1255 vm_page_flag_clear(m, PG_ZERO);
1256 /*
1257 * Might as well free the page if we can and it has
1258 * no valid data.
1259 */
1260 if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) {
1261 vm_page_busy(m);
1262 vm_page_protect(m, VM_PROT_NONE);
1263 vm_page_free(m);
1264 }
1265 }
1266 }
1267 runningbufspace -= bp->b_bufsize;
1268 splx(s);
1269 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
1270 if (bp->b_bufsize)
1271 bufspacewakeup();
1272 bp->b_npages = 0;
1273 bp->b_bufsize = 0;
1274 bp->b_flags &= ~B_VMIO;
1275 if (bp->b_vp)
1276 brelvp(bp);
1277}
1278
1279/*
1280 * Check to see if a block is currently memory resident.
1281 */
1282struct buf *
1283gbincore(struct vnode * vp, daddr_t blkno)
1284{
1285 struct buf *bp;
1286 struct bufhashhdr *bh;
1287
1288 bh = bufhash(vp, blkno);
1289
1290 /* Search hash chain */
1291 LIST_FOREACH(bp, bh, b_hash) {
1292 /* hit */
1293 if (bp->b_vp == vp && bp->b_lblkno == blkno &&
1294 (bp->b_flags & B_INVAL) == 0) {
1295 break;
1296 }
1297 }
1298 return (bp);
1299}
1300
1301/*
1302 * vfs_bio_awrite:
1303 *
1304 * Implement clustered async writes for clearing out B_DELWRI buffers.
1305 * This is much better then the old way of writing only one buffer at
1306 * a time. Note that we may not be presented with the buffers in the
1307 * correct order, so we search for the cluster in both directions.
1308 */
1309int
1310vfs_bio_awrite(struct buf * bp)
1311{
1312 int i;
1313 int j;
1314 daddr_t lblkno = bp->b_lblkno;
1315 struct vnode *vp = bp->b_vp;
1316 int s;
1317 int ncl;
1318 struct buf *bpa;
1319 int nwritten;
1320 int size;
1321 int maxcl;
1322
1323 s = splbio();
1324 /*
1325 * right now we support clustered writing only to regular files. If
1326 * we find a clusterable block we could be in the middle of a cluster
1327 * rather then at the beginning.
1328 */
1329 if ((vp->v_type == VREG) &&
1330 (vp->v_mount != 0) && /* Only on nodes that have the size info */
1331 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
1332
1333 size = vp->v_mount->mnt_stat.f_iosize;
1334 maxcl = MAXPHYS / size;
1335
1336 for (i = 1; i < maxcl; i++) {
1337 if ((bpa = gbincore(vp, lblkno + i)) &&
1338 BUF_REFCNT(bpa) == 0 &&
1339 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1340 (B_DELWRI | B_CLUSTEROK)) &&
1341 (bpa->b_bufsize == size)) {
1342 if ((bpa->b_blkno == bpa->b_lblkno) ||
1343 (bpa->b_blkno !=
1344 bp->b_blkno + ((i * size) >> DEV_BSHIFT)))
1345 break;
1346 } else {
1347 break;
1348 }
1349 }
1350 for (j = 1; i + j <= maxcl && j <= lblkno; j++) {
1351 if ((bpa = gbincore(vp, lblkno - j)) &&
1352 BUF_REFCNT(bpa) == 0 &&
1353 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1354 (B_DELWRI | B_CLUSTEROK)) &&
1355 (bpa->b_bufsize == size)) {
1356 if ((bpa->b_blkno == bpa->b_lblkno) ||
1357 (bpa->b_blkno !=
1358 bp->b_blkno - ((j * size) >> DEV_BSHIFT)))
1359 break;
1360 } else {
1361 break;
1362 }
1363 }
1364 --j;
1365 ncl = i + j;
1366 /*
1367 * this is a possible cluster write
1368 */
1369 if (ncl != 1) {
1370 nwritten = cluster_wbuild(vp, size, lblkno - j, ncl);
1371 splx(s);
1372 return nwritten;
1373 }
1374 }
1375
1376 BUF_LOCK(bp, LK_EXCLUSIVE);
1377 bremfree(bp);
1378 bp->b_flags |= B_ASYNC;
1379
1380 splx(s);
1381 /*
1382 * default (old) behavior, writing out only one block
1383 *
1384 * XXX returns b_bufsize instead of b_bcount for nwritten?
1385 */
1386 nwritten = bp->b_bufsize;
1387 (void) BUF_WRITE(bp);
1388
1389 return nwritten;
1390}
1391
1392/*
1393 * getnewbuf:
1394 *
1395 * Find and initialize a new buffer header, freeing up existing buffers
1396 * in the bufqueues as necessary. The new buffer is returned locked.
1397 *
1398 * Important: B_INVAL is not set. If the caller wishes to throw the
1399 * buffer away, the caller must set B_INVAL prior to calling brelse().
1400 *
1401 * We block if:
1402 * We have insufficient buffer headers
1403 * We have insufficient buffer space
1404 * buffer_map is too fragmented ( space reservation fails )
1405 * If we have to flush dirty buffers ( but we try to avoid this )
1406 *
1407 * To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1408 * Instead we ask the buf daemon to do it for us. We attempt to
1409 * avoid piecemeal wakeups of the pageout daemon.
1410 */
1411
1412static struct buf *
1413getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
1414{
1415 struct buf *bp;
1416 struct buf *nbp;
1417 int defrag = 0;
1418 int nqindex;
1419 int isspecial;
1420 static int flushingbufs;
1421
1422 if (curproc && (curproc->p_flag & P_BUFEXHAUST) == 0)
1423 isspecial = 0;
1424 else
1425 isspecial = 1;
1426
1427 ++getnewbufcalls;
1428 --getnewbufrestarts;
1429restart:
1430 ++getnewbufrestarts;
1431
1432 /*
1433 * Setup for scan. If we do not have enough free buffers,
1434 * we setup a degenerate case that immediately fails. Note
1435 * that if we are specially marked process, we are allowed to
1436 * dip into our reserves.
1437 *
1438 * The scanning sequence is nominally: EMPTY->EMPTYKVA->CLEAN
1439 *
1440 * We start with EMPTYKVA. If the list is empty we backup to EMPTY.
1441 * However, there are a number of cases (defragging, reusing, ...)
1442 * where we cannot backup.
1443 */
1444
1445 if (isspecial == 0 && numfreebuffers < lofreebuffers) {
1446 /*
1447 * This will cause an immediate failure
1448 */
1449 nqindex = QUEUE_CLEAN;
1450 nbp = NULL;
1451 } else {
1452 /*
1453 * Locate a buffer which already has KVA assigned. First
1454 * try EMPTYKVA buffers.
1455 */
1456 nqindex = QUEUE_EMPTYKVA;
1457 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA]);
1458
1459 if (nbp == NULL) {
1460 /*
1461 * If no EMPTYKVA buffers and we are either
1462 * defragging or reusing, locate a CLEAN buffer
1463 * to free or reuse. If bufspace useage is low
1464 * skip this step so we can allocate a new buffer.
1465 */
1466 if (defrag || bufspace >= lobufspace) {
1467 nqindex = QUEUE_CLEAN;
1468 nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
1469 }
1470
1471 /*
1472 * Nada. If we are allowed to allocate an EMPTY
1473 * buffer, go get one.
1474 */
1475 if (nbp == NULL && defrag == 0 &&
1476 (isspecial || bufspace < hibufspace)) {
1477 nqindex = QUEUE_EMPTY;
1478 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1479 }
1480 }
1481 }
1482
1483 /*
1484 * Run scan, possibly freeing data and/or kva mappings on the fly
1485 * depending.
1486 */
1487
1488 while ((bp = nbp) != NULL) {
1489 int qindex = nqindex;
1490
1491 /*
1492 * Calculate next bp ( we can only use it if we do not block
1493 * or do other fancy things ).
1494 */
1495 if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
1496 switch(qindex) {
1497 case QUEUE_EMPTY:
1498 nqindex = QUEUE_EMPTYKVA;
1499 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA])))
1500 break;
1501 /* fall through */
1502 case QUEUE_EMPTYKVA:
1503 nqindex = QUEUE_CLEAN;
1504 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN])))
1505 break;
1506 /* fall through */
1507 case QUEUE_CLEAN:
1508 /*
1509 * nbp is NULL.
1510 */
1511 break;
1512 }
1513 }
1514
1515 /*
1516 * Sanity Checks
1517 */
1518 KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp));
1519
1520 /*
1521 * Note: we no longer distinguish between VMIO and non-VMIO
1522 * buffers.
1523 */
1524
1525 KASSERT((bp->b_flags & B_DELWRI) == 0, ("delwri buffer %p found in queue %d", bp, qindex));
1526
1527 /*
1528 * If we are defragging then we need a buffer with
1529 * b_kvasize != 0. XXX this situation should no longer
1530 * occur, if defrag is non-zero the buffer's b_kvasize
1531 * should also be non-zero at this point. XXX
1532 */
1533 if (defrag && bp->b_kvasize == 0) {
1534 printf("Warning: defrag empty buffer %p\n", bp);
1535 continue;
1536 }
1537
1538 /*
1539 * Start freeing the bp. This is somewhat involved. nbp
1540 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1541 */
1542
1543 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1544 panic("getnewbuf: locked buf");
1545 bremfree(bp);
1546
1547 if (qindex == QUEUE_CLEAN) {
1548 if (bp->b_flags & B_VMIO) {
1549 bp->b_flags &= ~B_ASYNC;
1550 vfs_vmio_release(bp);
1551 }
1552 if (bp->b_vp)
1553 brelvp(bp);
1554 }
1555
1556 /*
1557 * NOTE: nbp is now entirely invalid. We can only restart
1558 * the scan from this point on.
1559 *
1560 * Get the rest of the buffer freed up. b_kva* is still
1561 * valid after this operation.
1562 */
1563
1564 if (bp->b_rcred != NOCRED) {
1565 crfree(bp->b_rcred);
1566 bp->b_rcred = NOCRED;
1567 }
1568 if (bp->b_wcred != NOCRED) {
1569 crfree(bp->b_wcred);
1570 bp->b_wcred = NOCRED;
1571 }
1572 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
1573 (*bioops.io_deallocate)(bp);
1574 if (bp->b_xflags & BX_BKGRDINPROG)
1575 panic("losing buffer 3");
1576 LIST_REMOVE(bp, b_hash);
1577 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1578
1579 if (bp->b_bufsize)
1580 allocbuf(bp, 0);
1581
1582 bp->b_flags = 0;
1583 bp->b_ioflags = 0;
1584 bp->b_xflags = 0;
1585 bp->b_dev = NODEV;
1586 bp->b_vp = NULL;
1587 bp->b_blkno = bp->b_lblkno = 0;
1588 bp->b_offset = NOOFFSET;
1589 bp->b_iodone = 0;
1590 bp->b_error = 0;
1591 bp->b_resid = 0;
1592 bp->b_bcount = 0;
1593 bp->b_npages = 0;
1594 bp->b_dirtyoff = bp->b_dirtyend = 0;
1595
1596 LIST_INIT(&bp->b_dep);
1597
1598 /*
1599 * If we are defragging then free the buffer.
1600 */
1601 if (defrag) {
1602 bp->b_flags |= B_INVAL;
1603 bfreekva(bp);
1604 brelse(bp);
1605 defrag = 0;
1606 goto restart;
1607 }
1608
1609 /*
1610 * If we are a normal process then deal with bufspace
1611 * hysteresis. A normal process tries to keep bufspace
1612 * between lobufspace and hibufspace. Note: if we encounter
1613 * a buffer with b_kvasize == 0 then it means we started
1614 * our scan on the EMPTY list and should allocate a new
1615 * buffer.
1616 */
1617 if (isspecial == 0) {
1618 if (bufspace > hibufspace)
1619 flushingbufs = 1;
1620 if (flushingbufs && bp->b_kvasize != 0) {
1621 bp->b_flags |= B_INVAL;
1622 bfreekva(bp);
1623 brelse(bp);
1624 goto restart;
1625 }
1626 if (bufspace < lobufspace)
1627 flushingbufs = 0;
1628 }
1629 break;
1630 }
1631
1632 /*
1633 * If we exhausted our list, sleep as appropriate. We may have to
1634 * wakeup various daemons and write out some dirty buffers.
1635 *
1636 * Generally we are sleeping due to insufficient buffer space.
1637 */
1638
1639 if (bp == NULL) {
1640 int flags;
1641 char *waitmsg;
1642
1643 if (defrag) {
1644 flags = VFS_BIO_NEED_BUFSPACE;
1645 waitmsg = "nbufkv";
1646 } else if (bufspace >= hibufspace) {
1647 waitmsg = "nbufbs";
1648 flags = VFS_BIO_NEED_BUFSPACE;
1649 } else {
1650 waitmsg = "newbuf";
1651 flags = VFS_BIO_NEED_ANY;
1652 }
1653
1654 bd_speedup(); /* heeeelp */
1655
1656 needsbuffer |= flags;
1657 while (needsbuffer & flags) {
1658 if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag,
1659 waitmsg, slptimeo))
1660 return (NULL);
1661 }
1662 } else {
1663 /*
1664 * We finally have a valid bp. We aren't quite out of the
1665 * woods, we still have to reserve kva space. In order
1666 * to keep fragmentation sane we only allocate kva in
1667 * BKVASIZE chunks.
1668 */
1669 maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
1670
1671 if (maxsize != bp->b_kvasize) {
1672 vm_offset_t addr = 0;
1673
1674 bfreekva(bp);
1675
1676 if (vm_map_findspace(buffer_map,
1677 vm_map_min(buffer_map), maxsize, &addr)) {
1678 /*
1679 * Uh oh. Buffer map is to fragmented. We
1680 * must defragment the map.
1681 */
1682 ++bufdefragcnt;
1683 defrag = 1;
1684 bp->b_flags |= B_INVAL;
1685 brelse(bp);
1686 goto restart;
1687 }
1688 if (addr) {
1689 vm_map_insert(buffer_map, NULL, 0,
1690 addr, addr + maxsize,
1691 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
1692
1693 bp->b_kvabase = (caddr_t) addr;
1694 bp->b_kvasize = maxsize;
1695 bufspace += bp->b_kvasize;
1696 ++bufreusecnt;
1697 }
1698 }
1699 bp->b_data = bp->b_kvabase;
1700 }
1701 return(bp);
1702}
1703
1704/*
1705 * waitfreebuffers:
1706 *
1707 * Wait for sufficient free buffers. Only called from normal processes.
1708 */
1709
1710static void
1711waitfreebuffers(int slpflag, int slptimeo)
1712{
1713 while (numfreebuffers < hifreebuffers) {
1714 if (numfreebuffers >= hifreebuffers)
1715 break;
1716 needsbuffer |= VFS_BIO_NEED_FREE;
1717 if (tsleep(&needsbuffer, (PRIBIO + 4)|slpflag, "biofre", slptimeo))
1718 break;
1719 }
1720}
1721
1722/*
1723 * buf_daemon:
1724 *
1725 * buffer flushing daemon. Buffers are normally flushed by the
1726 * update daemon but if it cannot keep up this process starts to
1727 * take the load in an attempt to prevent getnewbuf() from blocking.
1728 */
1729
1730static struct proc *bufdaemonproc;
1731static int bd_interval;
1732static int bd_flushto;
1733static int bd_flushinc;
1734
1735static struct kproc_desc buf_kp = {
1736 "bufdaemon",
1737 buf_daemon,
1738 &bufdaemonproc
1739};
1740SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp)
1741
1742static void
1743buf_daemon()
1744{
1745 int s;
1746
1747 /*
1748 * This process needs to be suspended prior to shutdown sync.
1749 */
1750 EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, bufdaemonproc,
1751 SHUTDOWN_PRI_LAST);
1752
1753 /*
1754 * This process is allowed to take the buffer cache to the limit
1755 */
1756 curproc->p_flag |= P_BUFEXHAUST;
1757 s = splbio();
1758
1759 bd_interval = 5 * hz; /* dynamically adjusted */
1760 bd_flushto = hidirtybuffers; /* dynamically adjusted */
1761 bd_flushinc = 1;
1762
1763 for (;;) {
1764 kproc_suspend_loop(bufdaemonproc);
1765
1766 bd_request = 0;
1767
1768 /*
1769 * Do the flush. Limit the number of buffers we flush in one
1770 * go. The failure condition occurs when processes are writing
1771 * buffers faster then we can dispose of them. In this case
1772 * we may be flushing so often that the previous set of flushes
1773 * have not had time to complete, causing us to run out of
1774 * physical buffers and block.
1775 */
1776 {
1777 int runcount = maxbdrun;
1778
1779 while (numdirtybuffers > bd_flushto && runcount) {
1780 --runcount;
1781 if (flushbufqueues() == 0)
1782 break;
1783 }
1784 }
1785
1786 if (bd_request ||
1787 tsleep(&bd_request, PVM, "psleep", bd_interval) == 0) {
1788 /*
1789 * Another request is pending or we were woken up
1790 * without timing out. Flush more.
1791 */
1792 --bd_flushto;
1793 if (bd_flushto >= numdirtybuffers - 5) {
1794 bd_flushto = numdirtybuffers - 10;
1795 bd_flushinc = 1;
1796 }
1797 if (bd_flushto < 2)
1798 bd_flushto = 2;
1799 } else {
1800 /*
1801 * We slept and timed out, we can slow down.
1802 */
1803 bd_flushto += bd_flushinc;
1804 if (bd_flushto > hidirtybuffers)
1805 bd_flushto = hidirtybuffers;
1806 ++bd_flushinc;
1807 if (bd_flushinc > hidirtybuffers / 20 + 1)
1808 bd_flushinc = hidirtybuffers / 20 + 1;
1809 }
1810
1811 /*
1812 * Set the interval on a linear scale based on hidirtybuffers
1813 * with a maximum frequency of 1/10 second.
1814 */
1815 bd_interval = bd_flushto * 5 * hz / hidirtybuffers;
1816 if (bd_interval < hz / 10)
1817 bd_interval = hz / 10;
1818 }
1819}
1820
1821/*
1822 * flushbufqueues:
1823 *
1824 * Try to flush a buffer in the dirty queue. We must be careful to
1825 * free up B_INVAL buffers instead of write them, which NFS is
1826 * particularly sensitive to.
1827 */
1828
1829static int
1830flushbufqueues(void)
1831{
1832 struct buf *bp;
1833 int r = 0;
1834
1835 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1836
1837 while (bp) {
1838 KASSERT((bp->b_flags & B_DELWRI), ("unexpected clean buffer %p", bp));
1839 if ((bp->b_flags & B_DELWRI) != 0 &&
1840 (bp->b_xflags & BX_BKGRDINPROG) == 0) {
1841 if (bp->b_flags & B_INVAL) {
1842 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1843 panic("flushbufqueues: locked buf");
1844 bremfree(bp);
1845 brelse(bp);
1846 ++r;
1847 break;
1848 }
1849 if (LIST_FIRST(&bp->b_dep) != NULL &&
1850 bioops.io_countdeps &&
1851 (bp->b_flags & B_DEFERRED) == 0 &&
1852 (*bioops.io_countdeps)(bp, 0)) {
1853 TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY],
1854 bp, b_freelist);
1855 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY],
1856 bp, b_freelist);
1857 bp->b_flags |= B_DEFERRED;
1858 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1859 continue;
1860 }
1861 vfs_bio_awrite(bp);
1862 ++r;
1863 break;
1864 }
1865 bp = TAILQ_NEXT(bp, b_freelist);
1866 }
1867 return (r);
1868}
1869
1870/*
1871 * Check to see if a block is currently memory resident.
1872 */
1873struct buf *
1874incore(struct vnode * vp, daddr_t blkno)
1875{
1876 struct buf *bp;
1877
1878 int s = splbio();
1879 bp = gbincore(vp, blkno);
1880 splx(s);
1881 return (bp);
1882}
1883
1884/*
1885 * Returns true if no I/O is needed to access the
1886 * associated VM object. This is like incore except
1887 * it also hunts around in the VM system for the data.
1888 */
1889
1890int
1891inmem(struct vnode * vp, daddr_t blkno)
1892{
1893 vm_object_t obj;
1894 vm_offset_t toff, tinc, size;
1895 vm_page_t m;
1896 vm_ooffset_t off;
1897
1898 if (incore(vp, blkno))
1899 return 1;
1900 if (vp->v_mount == NULL)
1901 return 0;
1902 if ((vp->v_object == NULL) || (vp->v_flag & VOBJBUF) == 0)
1903 return 0;
1904
1905 obj = vp->v_object;
1906 size = PAGE_SIZE;
1907 if (size > vp->v_mount->mnt_stat.f_iosize)
1908 size = vp->v_mount->mnt_stat.f_iosize;
1909 off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
1910
1911 for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
1912 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
1913 if (!m)
1914 return 0;
1915 tinc = size;
1916 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
1917 tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
1918 if (vm_page_is_valid(m,
1919 (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
1920 return 0;
1921 }
1922 return 1;
1923}
1924
1925/*
1926 * vfs_setdirty:
1927 *
1928 * Sets the dirty range for a buffer based on the status of the dirty
1929 * bits in the pages comprising the buffer.
1930 *
1931 * The range is limited to the size of the buffer.
1932 *
1933 * This routine is primarily used by NFS, but is generalized for the
1934 * B_VMIO case.
1935 */
1936static void
1937vfs_setdirty(struct buf *bp)
1938{
1939 int i;
1940 vm_object_t object;
1941
1942 /*
1943 * Degenerate case - empty buffer
1944 */
1945
1946 if (bp->b_bufsize == 0)
1947 return;
1948
1949 /*
1950 * We qualify the scan for modified pages on whether the
1951 * object has been flushed yet. The OBJ_WRITEABLE flag
1952 * is not cleared simply by protecting pages off.
1953 */
1954
1955 if ((bp->b_flags & B_VMIO) == 0)
1956 return;
1957
1958 object = bp->b_pages[0]->object;
1959
1960 if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY))
1961 printf("Warning: object %p writeable but not mightbedirty\n", object);
1962 if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY))
1963 printf("Warning: object %p mightbedirty but not writeable\n", object);
1964
1965 if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) {
1966 vm_offset_t boffset;
1967 vm_offset_t eoffset;
1968
1969 /*
1970 * test the pages to see if they have been modified directly
1971 * by users through the VM system.
1972 */
1973 for (i = 0; i < bp->b_npages; i++) {
1974 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
1975 vm_page_test_dirty(bp->b_pages[i]);
1976 }
1977
1978 /*
1979 * Calculate the encompassing dirty range, boffset and eoffset,
1980 * (eoffset - boffset) bytes.
1981 */
1982
1983 for (i = 0; i < bp->b_npages; i++) {
1984 if (bp->b_pages[i]->dirty)
1985 break;
1986 }
1987 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1988
1989 for (i = bp->b_npages - 1; i >= 0; --i) {
1990 if (bp->b_pages[i]->dirty) {
1991 break;
1992 }
1993 }
1994 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1995
1996 /*
1997 * Fit it to the buffer.
1998 */
1999
2000 if (eoffset > bp->b_bcount)
2001 eoffset = bp->b_bcount;
2002
2003 /*
2004 * If we have a good dirty range, merge with the existing
2005 * dirty range.
2006 */
2007
2008 if (boffset < eoffset) {
2009 if (bp->b_dirtyoff > boffset)
2010 bp->b_dirtyoff = boffset;
2011 if (bp->b_dirtyend < eoffset)
2012 bp->b_dirtyend = eoffset;
2013 }
2014 }
2015}
2016
2017/*
2018 * getblk:
2019 *
2020 * Get a block given a specified block and offset into a file/device.
2021 * The buffers B_DONE bit will be cleared on return, making it almost
2022 * ready for an I/O initiation. B_INVAL may or may not be set on
2023 * return. The caller should clear B_INVAL prior to initiating a
2024 * READ.
2025 *
2026 * For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2027 * an existing buffer.
2028 *
2029 * For a VMIO buffer, B_CACHE is modified according to the backing VM.
2030 * If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2031 * and then cleared based on the backing VM. If the previous buffer is
2032 * non-0-sized but invalid, B_CACHE will be cleared.
2033 *
2034 * If getblk() must create a new buffer, the new buffer is returned with
2035 * both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2036 * case it is returned with B_INVAL clear and B_CACHE set based on the
2037 * backing VM.
2038 *
2039 * getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2040 * B_CACHE bit is clear.
2041 *
2042 * What this means, basically, is that the caller should use B_CACHE to
2043 * determine whether the buffer is fully valid or not and should clear
2044 * B_INVAL prior to issuing a read. If the caller intends to validate
2045 * the buffer by loading its data area with something, the caller needs
2046 * to clear B_INVAL. If the caller does this without issuing an I/O,
2047 * the caller should set B_CACHE ( as an optimization ), else the caller
2048 * should issue the I/O and biodone() will set B_CACHE if the I/O was
2049 * a write attempt or if it was a successfull read. If the caller
2050 * intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
2051 * prior to issuing the READ. biodone() will *not* clear B_INVAL.
2052 */
2053struct buf *
2054getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo)
2055{
2056 struct buf *bp;
2057 int s;
2058 struct bufhashhdr *bh;
2059
2060 if (size > MAXBSIZE)
2061 panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE);
2062
2063 s = splbio();
2064loop:
2065 /*
2066 * Block if we are low on buffers. Certain processes are allowed
2067 * to completely exhaust the buffer cache.
2068 *
2069 * If this check ever becomes a bottleneck it may be better to
2070 * move it into the else, when gbincore() fails. At the moment
2071 * it isn't a problem.
2072 */
2073 if (!curproc || (curproc->p_flag & P_BUFEXHAUST)) {
2074 if (numfreebuffers == 0) {
2075 if (!curproc)
2076 return NULL;
2077 needsbuffer |= VFS_BIO_NEED_ANY;
2078 tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, "newbuf",
2079 slptimeo);
2080 }
2081 } else if (numfreebuffers < lofreebuffers) {
2082 waitfreebuffers(slpflag, slptimeo);
2083 }
2084
2085 if ((bp = gbincore(vp, blkno))) {
2086 /*
2087 * Buffer is in-core. If the buffer is not busy, it must
2088 * be on a queue.
2089 */
2090
2091 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
2092 if (BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL,
2093 "getblk", slpflag, slptimeo) == ENOLCK)
2094 goto loop;
2095 splx(s);
2096 return (struct buf *) NULL;
2097 }
2098
2099 /*
2100 * The buffer is locked. B_CACHE is cleared if the buffer is
2101 * invalid. Ohterwise, for a non-VMIO buffer, B_CACHE is set
2102 * and for a VMIO buffer B_CACHE is adjusted according to the
2103 * backing VM cache.
2104 */
2105 if (bp->b_flags & B_INVAL)
2106 bp->b_flags &= ~B_CACHE;
2107 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
2108 bp->b_flags |= B_CACHE;
2109 bremfree(bp);
2110
2111 /*
2112 * check for size inconsistancies for non-VMIO case.
2113 */
2114
2115 if (bp->b_bcount != size) {
2116 if ((bp->b_flags & B_VMIO) == 0 ||
2117 (size > bp->b_kvasize)) {
2118 if (bp->b_flags & B_DELWRI) {
2119 bp->b_flags |= B_NOCACHE;
2120 BUF_WRITE(bp);
2121 } else {
2122 if ((bp->b_flags & B_VMIO) &&
2123 (LIST_FIRST(&bp->b_dep) == NULL)) {
2124 bp->b_flags |= B_RELBUF;
2125 brelse(bp);
2126 } else {
2127 bp->b_flags |= B_NOCACHE;
2128 BUF_WRITE(bp);
2129 }
2130 }
2131 goto loop;
2132 }
2133 }
2134
2135 /*
2136 * If the size is inconsistant in the VMIO case, we can resize
2137 * the buffer. This might lead to B_CACHE getting set or
2138 * cleared. If the size has not changed, B_CACHE remains
2139 * unchanged from its previous state.
2140 */
2141
2142 if (bp->b_bcount != size)
2143 allocbuf(bp, size);
2144
2145 KASSERT(bp->b_offset != NOOFFSET,
2146 ("getblk: no buffer offset"));
2147
2148 /*
2149 * A buffer with B_DELWRI set and B_CACHE clear must
2150 * be committed before we can return the buffer in
2151 * order to prevent the caller from issuing a read
2152 * ( due to B_CACHE not being set ) and overwriting
2153 * it.
2154 *
2155 * Most callers, including NFS and FFS, need this to
2156 * operate properly either because they assume they
2157 * can issue a read if B_CACHE is not set, or because
2158 * ( for example ) an uncached B_DELWRI might loop due
2159 * to softupdates re-dirtying the buffer. In the latter
2160 * case, B_CACHE is set after the first write completes,
2161 * preventing further loops.
2162 */
2163
2164 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
2165 BUF_WRITE(bp);
2166 goto loop;
2167 }
2168
2169 splx(s);
2170 bp->b_flags &= ~B_DONE;
2171 } else {
2172 /*
2173 * Buffer is not in-core, create new buffer. The buffer
2174 * returned by getnewbuf() is locked. Note that the returned
2175 * buffer is also considered valid (not marked B_INVAL).
2176 */
2177 int bsize, maxsize, vmio;
2178 off_t offset;
2179
2180 if (vn_isdisk(vp, NULL))
2181 bsize = DEV_BSIZE;
2182 else if (vp->v_mountedhere)
2183 bsize = vp->v_mountedhere->mnt_stat.f_iosize;
2184 else if (vp->v_mount)
2185 bsize = vp->v_mount->mnt_stat.f_iosize;
2186 else
2187 bsize = size;
2188
2189 offset = (off_t)blkno * bsize;
2190 vmio = (vp->v_object != 0) && (vp->v_flag & VOBJBUF);
2191 maxsize = vmio ? size + (offset & PAGE_MASK) : size;
2192 maxsize = imax(maxsize, bsize);
2193
2194 if ((bp = getnewbuf(slpflag, slptimeo, size, maxsize)) == NULL) {
2195 if (slpflag || slptimeo) {
2196 splx(s);
2197 return NULL;
2198 }
2199 goto loop;
2200 }
2201
2202 /*
2203 * This code is used to make sure that a buffer is not
2204 * created while the getnewbuf routine is blocked.
2205 * This can be a problem whether the vnode is locked or not.
2206 * If the buffer is created out from under us, we have to
2207 * throw away the one we just created. There is now window
2208 * race because we are safely running at splbio() from the
2209 * point of the duplicate buffer creation through to here,
2210 * and we've locked the buffer.
2211 */
2212 if (gbincore(vp, blkno)) {
2213 bp->b_flags |= B_INVAL;
2214 brelse(bp);
2215 goto loop;
2216 }
2217
2218 /*
2219 * Insert the buffer into the hash, so that it can
2220 * be found by incore.
2221 */
2222 bp->b_blkno = bp->b_lblkno = blkno;
2223 bp->b_offset = offset;
2224
2225 bgetvp(vp, bp);
2226 LIST_REMOVE(bp, b_hash);
2227 bh = bufhash(vp, blkno);
2228 LIST_INSERT_HEAD(bh, bp, b_hash);
2229
2230 /*
2231 * set B_VMIO bit. allocbuf() the buffer bigger. Since the
2232 * buffer size starts out as 0, B_CACHE will be set by
2233 * allocbuf() for the VMIO case prior to it testing the
2234 * backing store for validity.
2235 */
2236
2237 if (vmio) {
2238 bp->b_flags |= B_VMIO;
2239#if defined(VFS_BIO_DEBUG)
2240 if (vp->v_type != VREG && vp->v_type != VBLK)
2241 printf("getblk: vmioing file type %d???\n", vp->v_type);
2242#endif
2243 } else {
2244 bp->b_flags &= ~B_VMIO;
2245 }
2246
2247 allocbuf(bp, size);
2248
2249 splx(s);
2250 bp->b_flags &= ~B_DONE;
2251 }
2252 return (bp);
2253}
2254
2255/*
2256 * Get an empty, disassociated buffer of given size. The buffer is initially
2257 * set to B_INVAL.
2258 */
2259struct buf *
2260geteblk(int size)
2261{
2262 struct buf *bp;
2263 int s;
2264 int maxsize;
2265
2266 maxsize = (size + BKVAMASK) & ~BKVAMASK;
2267
2268 s = splbio();
2269 while ((bp = getnewbuf(0, 0, size, maxsize)) == 0);
2270 splx(s);
2271 allocbuf(bp, size);
2272 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
2273 return (bp);
2274}
2275
2276
2277/*
2278 * This code constitutes the buffer memory from either anonymous system
2279 * memory (in the case of non-VMIO operations) or from an associated
2280 * VM object (in the case of VMIO operations). This code is able to
2281 * resize a buffer up or down.
2282 *
2283 * Note that this code is tricky, and has many complications to resolve
2284 * deadlock or inconsistant data situations. Tread lightly!!!
2285 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2286 * the caller. Calling this code willy nilly can result in the loss of data.
2287 *
2288 * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
2289 * B_CACHE for the non-VMIO case.
2290 */
2291
2292int
2293allocbuf(struct buf *bp, int size)
2294{
2295 int newbsize, mbsize;
2296 int i;
2297
2298 if (BUF_REFCNT(bp) == 0)
2299 panic("allocbuf: buffer not busy");
2300
2301 if (bp->b_kvasize < size)
2302 panic("allocbuf: buffer too small");
2303
2304 if ((bp->b_flags & B_VMIO) == 0) {
2305 caddr_t origbuf;
2306 int origbufsize;
2307 /*
2308 * Just get anonymous memory from the kernel. Don't
2309 * mess with B_CACHE.
2310 */
2311 mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2312#if !defined(NO_B_MALLOC)
2313 if (bp->b_flags & B_MALLOC)
2314 newbsize = mbsize;
2315 else
2316#endif
2317 newbsize = round_page(size);
2318
2319 if (newbsize < bp->b_bufsize) {
2320#if !defined(NO_B_MALLOC)
2321 /*
2322 * malloced buffers are not shrunk
2323 */
2324 if (bp->b_flags & B_MALLOC) {
2325 if (newbsize) {
2326 bp->b_bcount = size;
2327 } else {
2328 free(bp->b_data, M_BIOBUF);
2329 bufmallocspace -= bp->b_bufsize;
2330 runningbufspace -= bp->b_bufsize;
2331 if (bp->b_bufsize)
2332 bufspacewakeup();
2333 bp->b_data = bp->b_kvabase;
2334 bp->b_bufsize = 0;
2335 bp->b_bcount = 0;
2336 bp->b_flags &= ~B_MALLOC;
2337 }
2338 return 1;
2339 }
2340#endif
2341 vm_hold_free_pages(
2342 bp,
2343 (vm_offset_t) bp->b_data + newbsize,
2344 (vm_offset_t) bp->b_data + bp->b_bufsize);
2345 } else if (newbsize > bp->b_bufsize) {
2346#if !defined(NO_B_MALLOC)
2347 /*
2348 * We only use malloced memory on the first allocation.
2349 * and revert to page-allocated memory when the buffer
2350 * grows.
2351 */
2352 if ( (bufmallocspace < maxbufmallocspace) &&
2353 (bp->b_bufsize == 0) &&
2354 (mbsize <= PAGE_SIZE/2)) {
2355
2356 bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
2357 bp->b_bufsize = mbsize;
2358 bp->b_bcount = size;
2359 bp->b_flags |= B_MALLOC;
2360 bufmallocspace += mbsize;
2361 runningbufspace += bp->b_bufsize;
2362 return 1;
2363 }
2364#endif
2365 origbuf = NULL;
2366 origbufsize = 0;
2367#if !defined(NO_B_MALLOC)
2368 /*
2369 * If the buffer is growing on its other-than-first allocation,
2370 * then we revert to the page-allocation scheme.
2371 */
2372 if (bp->b_flags & B_MALLOC) {
2373 origbuf = bp->b_data;
2374 origbufsize = bp->b_bufsize;
2375 bp->b_data = bp->b_kvabase;
2376 bufmallocspace -= bp->b_bufsize;
2377 runningbufspace -= bp->b_bufsize;
2378 if (bp->b_bufsize)
2379 bufspacewakeup();
2380 bp->b_bufsize = 0;
2381 bp->b_flags &= ~B_MALLOC;
2382 newbsize = round_page(newbsize);
2383 }
2384#endif
2385 vm_hold_load_pages(
2386 bp,
2387 (vm_offset_t) bp->b_data + bp->b_bufsize,
2388 (vm_offset_t) bp->b_data + newbsize);
2389#if !defined(NO_B_MALLOC)
2390 if (origbuf) {
2391 bcopy(origbuf, bp->b_data, origbufsize);
2392 free(origbuf, M_BIOBUF);
2393 }
2394#endif
2395 }
2396 } else {
2397 vm_page_t m;
2398 int desiredpages;
2399
2400 newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2401 desiredpages = (size == 0) ? 0 :
2402 num_pages((bp->b_offset & PAGE_MASK) + newbsize);
2403
2404#if !defined(NO_B_MALLOC)
2405 if (bp->b_flags & B_MALLOC)
2406 panic("allocbuf: VMIO buffer can't be malloced");
2407#endif
2408 /*
2409 * Set B_CACHE initially if buffer is 0 length or will become
2410 * 0-length.
2411 */
2412 if (size == 0 || bp->b_bufsize == 0)
2413 bp->b_flags |= B_CACHE;
2414
2415 if (newbsize < bp->b_bufsize) {
2416 /*
2417 * DEV_BSIZE aligned new buffer size is less then the
2418 * DEV_BSIZE aligned existing buffer size. Figure out
2419 * if we have to remove any pages.
2420 */
2421 if (desiredpages < bp->b_npages) {
2422 for (i = desiredpages; i < bp->b_npages; i++) {
2423 /*
2424 * the page is not freed here -- it
2425 * is the responsibility of
2426 * vnode_pager_setsize
2427 */
2428 m = bp->b_pages[i];
2429 KASSERT(m != bogus_page,
2430 ("allocbuf: bogus page found"));
2431 while (vm_page_sleep_busy(m, TRUE, "biodep"))
2432 ;
2433
2434 bp->b_pages[i] = NULL;
2435 vm_page_unwire(m, 0);
2436 }
2437 pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) +
2438 (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages));
2439 bp->b_npages = desiredpages;
2440 }
2441 } else if (size > bp->b_bcount) {
2442 /*
2443 * We are growing the buffer, possibly in a
2444 * byte-granular fashion.
2445 */
2446 struct vnode *vp;
2447 vm_object_t obj;
2448 vm_offset_t toff;
2449 vm_offset_t tinc;
2450
2451 /*
2452 * Step 1, bring in the VM pages from the object,
2453 * allocating them if necessary. We must clear
2454 * B_CACHE if these pages are not valid for the
2455 * range covered by the buffer.
2456 */
2457
2458 vp = bp->b_vp;
2459 obj = vp->v_object;
2460
2461 while (bp->b_npages < desiredpages) {
2462 vm_page_t m;
2463 vm_pindex_t pi;
2464
2465 pi = OFF_TO_IDX(bp->b_offset) + bp->b_npages;
2466 if ((m = vm_page_lookup(obj, pi)) == NULL) {
2467 m = vm_page_alloc(obj, pi, VM_ALLOC_NORMAL);
2468 if (m == NULL) {
2469 VM_WAIT;
2470 vm_pageout_deficit += desiredpages - bp->b_npages;
2471 } else {
2472 vm_page_wire(m);
2473 vm_page_wakeup(m);
2474 bp->b_flags &= ~B_CACHE;
2475 bp->b_pages[bp->b_npages] = m;
2476 ++bp->b_npages;
2477 }
2478 continue;
2479 }
2480
2481 /*
2482 * We found a page. If we have to sleep on it,
2483 * retry because it might have gotten freed out
2484 * from under us.
2485 *
2486 * We can only test PG_BUSY here. Blocking on
2487 * m->busy might lead to a deadlock:
2488 *
2489 * vm_fault->getpages->cluster_read->allocbuf
2490 *
2491 */
2492
2493 if (vm_page_sleep_busy(m, FALSE, "pgtblk"))
2494 continue;
2495
2496 /*
2497 * We have a good page. Should we wakeup the
2498 * page daemon?
2499 */
2500 if ((curproc != pageproc) &&
2501 ((m->queue - m->pc) == PQ_CACHE) &&
2502 ((cnt.v_free_count + cnt.v_cache_count) <
2503 (cnt.v_free_min + cnt.v_cache_min))) {
2504 pagedaemon_wakeup();
2505 }
2506 vm_page_flag_clear(m, PG_ZERO);
2507 vm_page_wire(m);
2508 bp->b_pages[bp->b_npages] = m;
2509 ++bp->b_npages;
2510 }
2511
2512 /*
2513 * Step 2. We've loaded the pages into the buffer,
2514 * we have to figure out if we can still have B_CACHE
2515 * set. Note that B_CACHE is set according to the
2516 * byte-granular range ( bcount and size ), new the
2517 * aligned range ( newbsize ).
2518 *
2519 * The VM test is against m->valid, which is DEV_BSIZE
2520 * aligned. Needless to say, the validity of the data
2521 * needs to also be DEV_BSIZE aligned. Note that this
2522 * fails with NFS if the server or some other client
2523 * extends the file's EOF. If our buffer is resized,
2524 * B_CACHE may remain set! XXX
2525 */
2526
2527 toff = bp->b_bcount;
2528 tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2529
2530 while ((bp->b_flags & B_CACHE) && toff < size) {
2531 vm_pindex_t pi;
2532
2533 if (tinc > (size - toff))
2534 tinc = size - toff;
2535
2536 pi = ((bp->b_offset & PAGE_MASK) + toff) >>
2537 PAGE_SHIFT;
2538
2539 vfs_buf_test_cache(
2540 bp,
2541 bp->b_offset,
2542 toff,
2543 tinc,
2544 bp->b_pages[pi]
2545 );
2546 toff += tinc;
2547 tinc = PAGE_SIZE;
2548 }
2549
2550 /*
2551 * Step 3, fixup the KVM pmap. Remember that
2552 * bp->b_data is relative to bp->b_offset, but
2553 * bp->b_offset may be offset into the first page.
2554 */
2555
2556 bp->b_data = (caddr_t)
2557 trunc_page((vm_offset_t)bp->b_data);
2558 pmap_qenter(
2559 (vm_offset_t)bp->b_data,
2560 bp->b_pages,
2561 bp->b_npages
2562 );
2563 bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
2564 (vm_offset_t)(bp->b_offset & PAGE_MASK));
2565 }
2566 }
2567 runningbufspace += (newbsize - bp->b_bufsize);
2568 if (newbsize < bp->b_bufsize)
2569 bufspacewakeup();
2570 bp->b_bufsize = newbsize; /* actual buffer allocation */
2571 bp->b_bcount = size; /* requested buffer size */
2572 return 1;
2573}
2574
2575/*
2576 * biowait:
2577 *
2578 * Wait for buffer I/O completion, returning error status. The buffer
2579 * is left locked and B_DONE on return. B_EINTR is converted into a EINTR
2580 * error and cleared.
2581 */
2582int
2583biowait(register struct buf * bp)
2584{
2585 int s;
2586
2587 s = splbio();
2588 while ((bp->b_flags & B_DONE) == 0) {
2589#if defined(NO_SCHEDULE_MODS)
2590 tsleep(bp, PRIBIO, "biowait", 0);
2591#else
2592 if (bp->b_iocmd == BIO_READ)
2593 tsleep(bp, PRIBIO, "biord", 0);
2594 else
2595 tsleep(bp, PRIBIO, "biowr", 0);
2596#endif
2597 }
2598 splx(s);
2599 if (bp->b_flags & B_EINTR) {
2600 bp->b_flags &= ~B_EINTR;
2601 return (EINTR);
2602 }
2603 if (bp->b_ioflags & BIO_ERROR) {
2604 return (bp->b_error ? bp->b_error : EIO);
2605 } else {
2606 return (0);
2607 }
2608}
2609
2610/*
2611 * biodone:
2612 *
2613 * Finish I/O on a buffer, optionally calling a completion function.
2614 * This is usually called from an interrupt so process blocking is
2615 * not allowed.
2616 *
2617 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2618 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
2619 * assuming B_INVAL is clear.
2620 *
2621 * For the VMIO case, we set B_CACHE if the op was a read and no
2622 * read error occured, or if the op was a write. B_CACHE is never
2623 * set if the buffer is invalid or otherwise uncacheable.
2624 *
2625 * biodone does not mess with B_INVAL, allowing the I/O routine or the
2626 * initiator to leave B_INVAL set to brelse the buffer out of existance
2627 * in the biodone routine.
2628 */
2629void
| 707}
708
709/*
710 * Delayed write. (Buffer is marked dirty). Do not bother writing
711 * anything if the buffer is marked invalid.
712 *
713 * Note that since the buffer must be completely valid, we can safely
714 * set B_CACHE. In fact, we have to set B_CACHE here rather then in
715 * biodone() in order to prevent getblk from writing the buffer
716 * out synchronously.
717 */
718void
719bdwrite(struct buf * bp)
720{
721 if (BUF_REFCNT(bp) == 0)
722 panic("bdwrite: buffer is not busy");
723
724 if (bp->b_flags & B_INVAL) {
725 brelse(bp);
726 return;
727 }
728 bdirty(bp);
729
730 /*
731 * Set B_CACHE, indicating that the buffer is fully valid. This is
732 * true even of NFS now.
733 */
734 bp->b_flags |= B_CACHE;
735
736 /*
737 * This bmap keeps the system from needing to do the bmap later,
738 * perhaps when the system is attempting to do a sync. Since it
739 * is likely that the indirect block -- or whatever other datastructure
740 * that the filesystem needs is still in memory now, it is a good
741 * thing to do this. Note also, that if the pageout daemon is
742 * requesting a sync -- there might not be enough memory to do
743 * the bmap then... So, this is important to do.
744 */
745 if (bp->b_lblkno == bp->b_blkno) {
746 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
747 }
748
749 /*
750 * Set the *dirty* buffer range based upon the VM system dirty pages.
751 */
752 vfs_setdirty(bp);
753
754 /*
755 * We need to do this here to satisfy the vnode_pager and the
756 * pageout daemon, so that it thinks that the pages have been
757 * "cleaned". Note that since the pages are in a delayed write
758 * buffer -- the VFS layer "will" see that the pages get written
759 * out on the next sync, or perhaps the cluster will be completed.
760 */
761 vfs_clean_pages(bp);
762 bqrelse(bp);
763
764 /*
765 * Wakeup the buffer flushing daemon if we have saturated the
766 * buffer cache.
767 */
768
769 bd_wakeup(hidirtybuffers);
770
771 /*
772 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
773 * due to the softdep code.
774 */
775}
776
777/*
778 * bdirty:
779 *
780 * Turn buffer into delayed write request. We must clear BIO_READ and
781 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
782 * itself to properly update it in the dirty/clean lists. We mark it
783 * B_DONE to ensure that any asynchronization of the buffer properly
784 * clears B_DONE ( else a panic will occur later ).
785 *
786 * bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
787 * might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
788 * should only be called if the buffer is known-good.
789 *
790 * Since the buffer is not on a queue, we do not update the numfreebuffers
791 * count.
792 *
793 * Must be called at splbio().
794 * The buffer must be on QUEUE_NONE.
795 */
796void
797bdirty(bp)
798 struct buf *bp;
799{
800 KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
801 bp->b_flags &= ~(B_RELBUF);
802 bp->b_iocmd = BIO_WRITE;
803
804 if ((bp->b_flags & B_DELWRI) == 0) {
805 bp->b_flags |= B_DONE | B_DELWRI;
806 reassignbuf(bp, bp->b_vp);
807 ++numdirtybuffers;
808 bd_wakeup(hidirtybuffers);
809 }
810}
811
812/*
813 * bundirty:
814 *
815 * Clear B_DELWRI for buffer.
816 *
817 * Since the buffer is not on a queue, we do not update the numfreebuffers
818 * count.
819 *
820 * Must be called at splbio().
821 * The buffer must be on QUEUE_NONE.
822 */
823
824void
825bundirty(bp)
826 struct buf *bp;
827{
828 KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
829
830 if (bp->b_flags & B_DELWRI) {
831 bp->b_flags &= ~B_DELWRI;
832 reassignbuf(bp, bp->b_vp);
833 --numdirtybuffers;
834 numdirtywakeup();
835 }
836 /*
837 * Since it is now being written, we can clear its deferred write flag.
838 */
839 bp->b_flags &= ~B_DEFERRED;
840}
841
842/*
843 * bawrite:
844 *
845 * Asynchronous write. Start output on a buffer, but do not wait for
846 * it to complete. The buffer is released when the output completes.
847 *
848 * bwrite() ( or the VOP routine anyway ) is responsible for handling
849 * B_INVAL buffers. Not us.
850 */
851void
852bawrite(struct buf * bp)
853{
854 bp->b_flags |= B_ASYNC;
855 (void) BUF_WRITE(bp);
856}
857
858/*
859 * bowrite:
860 *
861 * Ordered write. Start output on a buffer, and flag it so that the
862 * device will write it in the order it was queued. The buffer is
863 * released when the output completes. bwrite() ( or the VOP routine
864 * anyway ) is responsible for handling B_INVAL buffers.
865 */
866int
867bowrite(struct buf * bp)
868{
869 bp->b_ioflags |= BIO_ORDERED;
870 bp->b_flags |= B_ASYNC;
871 return (BUF_WRITE(bp));
872}
873
874/*
875 * bwillwrite:
876 *
877 * Called prior to the locking of any vnodes when we are expecting to
878 * write. We do not want to starve the buffer cache with too many
879 * dirty buffers so we block here. By blocking prior to the locking
880 * of any vnodes we attempt to avoid the situation where a locked vnode
881 * prevents the various system daemons from flushing related buffers.
882 */
883
884void
885bwillwrite(void)
886{
887 int slop = hidirtybuffers / 10;
888
889 if (numdirtybuffers > hidirtybuffers + slop) {
890 int s;
891
892 s = splbio();
893 while (numdirtybuffers > hidirtybuffers) {
894 bd_wakeup(hidirtybuffers);
895 needsbuffer |= VFS_BIO_NEED_DIRTYFLUSH;
896 tsleep(&needsbuffer, (PRIBIO + 4), "flswai", 0);
897 }
898 splx(s);
899 }
900}
901
902/*
903 * brelse:
904 *
905 * Release a busy buffer and, if requested, free its resources. The
906 * buffer will be stashed in the appropriate bufqueue[] allowing it
907 * to be accessed later as a cache entity or reused for other purposes.
908 */
909void
910brelse(struct buf * bp)
911{
912 int s;
913
914 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
915
916 s = splbio();
917
918 if (bp->b_flags & B_LOCKED)
919 bp->b_ioflags &= ~BIO_ERROR;
920
921 if (bp->b_iocmd == BIO_WRITE &&
922 (bp->b_ioflags & BIO_ERROR) &&
923 !(bp->b_flags & B_INVAL)) {
924 /*
925 * Failed write, redirty. Must clear BIO_ERROR to prevent
926 * pages from being scrapped. If B_INVAL is set then
927 * this case is not run and the next case is run to
928 * destroy the buffer. B_INVAL can occur if the buffer
929 * is outside the range supported by the underlying device.
930 */
931 bp->b_ioflags &= ~BIO_ERROR;
932 bdirty(bp);
933 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
934 (bp->b_ioflags & BIO_ERROR) ||
935 bp->b_iocmd == BIO_DELETE || (bp->b_bufsize <= 0)) {
936 /*
937 * Either a failed I/O or we were asked to free or not
938 * cache the buffer.
939 */
940 bp->b_flags |= B_INVAL;
941 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
942 (*bioops.io_deallocate)(bp);
943 if (bp->b_flags & B_DELWRI) {
944 --numdirtybuffers;
945 numdirtywakeup();
946 }
947 bp->b_flags &= ~(B_DELWRI | B_CACHE);
948 if ((bp->b_flags & B_VMIO) == 0) {
949 if (bp->b_bufsize)
950 allocbuf(bp, 0);
951 if (bp->b_vp)
952 brelvp(bp);
953 }
954 }
955
956 /*
957 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release()
958 * is called with B_DELWRI set, the underlying pages may wind up
959 * getting freed causing a previous write (bdwrite()) to get 'lost'
960 * because pages associated with a B_DELWRI bp are marked clean.
961 *
962 * We still allow the B_INVAL case to call vfs_vmio_release(), even
963 * if B_DELWRI is set.
964 */
965
966 if (bp->b_flags & B_DELWRI)
967 bp->b_flags &= ~B_RELBUF;
968
969 /*
970 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
971 * constituted, not even NFS buffers now. Two flags effect this. If
972 * B_INVAL, the struct buf is invalidated but the VM object is kept
973 * around ( i.e. so it is trivial to reconstitute the buffer later ).
974 *
975 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
976 * invalidated. BIO_ERROR cannot be set for a failed write unless the
977 * buffer is also B_INVAL because it hits the re-dirtying code above.
978 *
979 * Normally we can do this whether a buffer is B_DELWRI or not. If
980 * the buffer is an NFS buffer, it is tracking piecemeal writes or
981 * the commit state and we cannot afford to lose the buffer. If the
982 * buffer has a background write in progress, we need to keep it
983 * around to prevent it from being reconstituted and starting a second
984 * background write.
985 */
986 if ((bp->b_flags & B_VMIO)
987 && !(bp->b_vp->v_tag == VT_NFS &&
988 !vn_isdisk(bp->b_vp, NULL) &&
989 (bp->b_flags & B_DELWRI) &&
990 (bp->b_xflags & BX_BKGRDINPROG))
991 ) {
992
993 int i, j, resid;
994 vm_page_t m;
995 off_t foff;
996 vm_pindex_t poff;
997 vm_object_t obj;
998 struct vnode *vp;
999
1000 vp = bp->b_vp;
1001
1002 /*
1003 * Get the base offset and length of the buffer. Note that
1004 * for block sizes that are less then PAGE_SIZE, the b_data
1005 * base of the buffer does not represent exactly b_offset and
1006 * neither b_offset nor b_size are necessarily page aligned.
1007 * Instead, the starting position of b_offset is:
1008 *
1009 * b_data + (b_offset & PAGE_MASK)
1010 *
1011 * block sizes less then DEV_BSIZE (usually 512) are not
1012 * supported due to the page granularity bits (m->valid,
1013 * m->dirty, etc...).
1014 *
1015 * See man buf(9) for more information
1016 */
1017
1018 resid = bp->b_bufsize;
1019 foff = bp->b_offset;
1020
1021 for (i = 0; i < bp->b_npages; i++) {
1022 m = bp->b_pages[i];
1023 vm_page_flag_clear(m, PG_ZERO);
1024 if (m == bogus_page) {
1025
1026 obj = (vm_object_t) vp->v_object;
1027 poff = OFF_TO_IDX(bp->b_offset);
1028
1029 for (j = i; j < bp->b_npages; j++) {
1030 m = bp->b_pages[j];
1031 if (m == bogus_page) {
1032 m = vm_page_lookup(obj, poff + j);
1033 if (!m) {
1034 panic("brelse: page missing\n");
1035 }
1036 bp->b_pages[j] = m;
1037 }
1038 }
1039
1040 if ((bp->b_flags & B_INVAL) == 0) {
1041 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
1042 }
1043 }
1044 if ((bp->b_flags & B_NOCACHE) || (bp->b_ioflags & BIO_ERROR)) {
1045 int poffset = foff & PAGE_MASK;
1046 int presid = resid > (PAGE_SIZE - poffset) ?
1047 (PAGE_SIZE - poffset) : resid;
1048
1049 KASSERT(presid >= 0, ("brelse: extra page"));
1050 vm_page_set_invalid(m, poffset, presid);
1051 }
1052 resid -= PAGE_SIZE - (foff & PAGE_MASK);
1053 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
1054 }
1055
1056 if (bp->b_flags & (B_INVAL | B_RELBUF))
1057 vfs_vmio_release(bp);
1058
1059 } else if (bp->b_flags & B_VMIO) {
1060
1061 if (bp->b_flags & (B_INVAL | B_RELBUF))
1062 vfs_vmio_release(bp);
1063
1064 }
1065
1066 if (bp->b_qindex != QUEUE_NONE)
1067 panic("brelse: free buffer onto another queue???");
1068 if (BUF_REFCNT(bp) > 1) {
1069 /* Temporary panic to verify exclusive locking */
1070 /* This panic goes away when we allow shared refs */
1071 panic("brelse: multiple refs");
1072 /* do not release to free list */
1073 BUF_UNLOCK(bp);
1074 splx(s);
1075 return;
1076 }
1077
1078 /* enqueue */
1079
1080 /* buffers with no memory */
1081 if (bp->b_bufsize == 0) {
1082 bp->b_flags |= B_INVAL;
1083 bp->b_xflags &= ~BX_BKGRDWRITE;
1084 if (bp->b_xflags & BX_BKGRDINPROG)
1085 panic("losing buffer 1");
1086 if (bp->b_kvasize) {
1087 bp->b_qindex = QUEUE_EMPTYKVA;
1088 } else {
1089 bp->b_qindex = QUEUE_EMPTY;
1090 }
1091 TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1092 LIST_REMOVE(bp, b_hash);
1093 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1094 bp->b_dev = NODEV;
1095 /* buffers with junk contents */
1096 } else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) || (bp->b_ioflags & BIO_ERROR)) {
1097 bp->b_flags |= B_INVAL;
1098 bp->b_xflags &= ~BX_BKGRDWRITE;
1099 if (bp->b_xflags & BX_BKGRDINPROG)
1100 panic("losing buffer 2");
1101 bp->b_qindex = QUEUE_CLEAN;
1102 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1103 LIST_REMOVE(bp, b_hash);
1104 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1105 bp->b_dev = NODEV;
1106
1107 /* buffers that are locked */
1108 } else if (bp->b_flags & B_LOCKED) {
1109 bp->b_qindex = QUEUE_LOCKED;
1110 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1111
1112 /* remaining buffers */
1113 } else {
1114 switch(bp->b_flags & (B_DELWRI|B_AGE)) {
1115 case B_DELWRI | B_AGE:
1116 bp->b_qindex = QUEUE_DIRTY;
1117 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1118 break;
1119 case B_DELWRI:
1120 bp->b_qindex = QUEUE_DIRTY;
1121 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1122 break;
1123 case B_AGE:
1124 bp->b_qindex = QUEUE_CLEAN;
1125 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1126 break;
1127 default:
1128 bp->b_qindex = QUEUE_CLEAN;
1129 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1130 break;
1131 }
1132 }
1133
1134 /*
1135 * If B_INVAL, clear B_DELWRI. We've already placed the buffer
1136 * on the correct queue.
1137 */
1138 if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) {
1139 bp->b_flags &= ~B_DELWRI;
1140 --numdirtybuffers;
1141 numdirtywakeup();
1142 }
1143
1144 runningbufspace -= bp->b_bufsize;
1145
1146 /*
1147 * Fixup numfreebuffers count. The bp is on an appropriate queue
1148 * unless locked. We then bump numfreebuffers if it is not B_DELWRI.
1149 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1150 * if B_INVAL is set ).
1151 */
1152
1153 if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI))
1154 bufcountwakeup();
1155
1156 /*
1157 * Something we can maybe free.
1158 */
1159
1160 if (bp->b_bufsize || bp->b_kvasize)
1161 bufspacewakeup();
1162
1163 /* unlock */
1164 BUF_UNLOCK(bp);
1165 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1166 bp->b_ioflags &= ~BIO_ORDERED;
1167 splx(s);
1168}
1169
1170/*
1171 * Release a buffer back to the appropriate queue but do not try to free
1172 * it.
1173 *
1174 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1175 * biodone() to requeue an async I/O on completion. It is also used when
1176 * known good buffers need to be requeued but we think we may need the data
1177 * again soon.
1178 */
1179void
1180bqrelse(struct buf * bp)
1181{
1182 int s;
1183
1184 s = splbio();
1185
1186 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1187
1188 if (bp->b_qindex != QUEUE_NONE)
1189 panic("bqrelse: free buffer onto another queue???");
1190 if (BUF_REFCNT(bp) > 1) {
1191 /* do not release to free list */
1192 panic("bqrelse: multiple refs");
1193 BUF_UNLOCK(bp);
1194 splx(s);
1195 return;
1196 }
1197 if (bp->b_flags & B_LOCKED) {
1198 bp->b_ioflags &= ~BIO_ERROR;
1199 bp->b_qindex = QUEUE_LOCKED;
1200 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist);
1201 /* buffers with stale but valid contents */
1202 } else if (bp->b_flags & B_DELWRI) {
1203 bp->b_qindex = QUEUE_DIRTY;
1204 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
1205 } else {
1206 bp->b_qindex = QUEUE_CLEAN;
1207 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_CLEAN], bp, b_freelist);
1208 }
1209
1210 runningbufspace -= bp->b_bufsize;
1211
1212 if ((bp->b_flags & B_LOCKED) == 0 &&
1213 ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))) {
1214 bufcountwakeup();
1215 }
1216
1217 /*
1218 * Something we can maybe wakeup
1219 */
1220 if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1221 bufspacewakeup();
1222
1223 /* unlock */
1224 BUF_UNLOCK(bp);
1225 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1226 bp->b_ioflags &= ~BIO_ORDERED;
1227 splx(s);
1228}
1229
1230static void
1231vfs_vmio_release(bp)
1232 struct buf *bp;
1233{
1234 int i, s;
1235 vm_page_t m;
1236
1237 s = splvm();
1238 for (i = 0; i < bp->b_npages; i++) {
1239 m = bp->b_pages[i];
1240 bp->b_pages[i] = NULL;
1241 /*
1242 * In order to keep page LRU ordering consistent, put
1243 * everything on the inactive queue.
1244 */
1245 vm_page_unwire(m, 0);
1246 /*
1247 * We don't mess with busy pages, it is
1248 * the responsibility of the process that
1249 * busied the pages to deal with them.
1250 */
1251 if ((m->flags & PG_BUSY) || (m->busy != 0))
1252 continue;
1253
1254 if (m->wire_count == 0) {
1255 vm_page_flag_clear(m, PG_ZERO);
1256 /*
1257 * Might as well free the page if we can and it has
1258 * no valid data.
1259 */
1260 if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) {
1261 vm_page_busy(m);
1262 vm_page_protect(m, VM_PROT_NONE);
1263 vm_page_free(m);
1264 }
1265 }
1266 }
1267 runningbufspace -= bp->b_bufsize;
1268 splx(s);
1269 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
1270 if (bp->b_bufsize)
1271 bufspacewakeup();
1272 bp->b_npages = 0;
1273 bp->b_bufsize = 0;
1274 bp->b_flags &= ~B_VMIO;
1275 if (bp->b_vp)
1276 brelvp(bp);
1277}
1278
1279/*
1280 * Check to see if a block is currently memory resident.
1281 */
1282struct buf *
1283gbincore(struct vnode * vp, daddr_t blkno)
1284{
1285 struct buf *bp;
1286 struct bufhashhdr *bh;
1287
1288 bh = bufhash(vp, blkno);
1289
1290 /* Search hash chain */
1291 LIST_FOREACH(bp, bh, b_hash) {
1292 /* hit */
1293 if (bp->b_vp == vp && bp->b_lblkno == blkno &&
1294 (bp->b_flags & B_INVAL) == 0) {
1295 break;
1296 }
1297 }
1298 return (bp);
1299}
1300
1301/*
1302 * vfs_bio_awrite:
1303 *
1304 * Implement clustered async writes for clearing out B_DELWRI buffers.
1305 * This is much better then the old way of writing only one buffer at
1306 * a time. Note that we may not be presented with the buffers in the
1307 * correct order, so we search for the cluster in both directions.
1308 */
1309int
1310vfs_bio_awrite(struct buf * bp)
1311{
1312 int i;
1313 int j;
1314 daddr_t lblkno = bp->b_lblkno;
1315 struct vnode *vp = bp->b_vp;
1316 int s;
1317 int ncl;
1318 struct buf *bpa;
1319 int nwritten;
1320 int size;
1321 int maxcl;
1322
1323 s = splbio();
1324 /*
1325 * right now we support clustered writing only to regular files. If
1326 * we find a clusterable block we could be in the middle of a cluster
1327 * rather then at the beginning.
1328 */
1329 if ((vp->v_type == VREG) &&
1330 (vp->v_mount != 0) && /* Only on nodes that have the size info */
1331 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
1332
1333 size = vp->v_mount->mnt_stat.f_iosize;
1334 maxcl = MAXPHYS / size;
1335
1336 for (i = 1; i < maxcl; i++) {
1337 if ((bpa = gbincore(vp, lblkno + i)) &&
1338 BUF_REFCNT(bpa) == 0 &&
1339 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1340 (B_DELWRI | B_CLUSTEROK)) &&
1341 (bpa->b_bufsize == size)) {
1342 if ((bpa->b_blkno == bpa->b_lblkno) ||
1343 (bpa->b_blkno !=
1344 bp->b_blkno + ((i * size) >> DEV_BSHIFT)))
1345 break;
1346 } else {
1347 break;
1348 }
1349 }
1350 for (j = 1; i + j <= maxcl && j <= lblkno; j++) {
1351 if ((bpa = gbincore(vp, lblkno - j)) &&
1352 BUF_REFCNT(bpa) == 0 &&
1353 ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) ==
1354 (B_DELWRI | B_CLUSTEROK)) &&
1355 (bpa->b_bufsize == size)) {
1356 if ((bpa->b_blkno == bpa->b_lblkno) ||
1357 (bpa->b_blkno !=
1358 bp->b_blkno - ((j * size) >> DEV_BSHIFT)))
1359 break;
1360 } else {
1361 break;
1362 }
1363 }
1364 --j;
1365 ncl = i + j;
1366 /*
1367 * this is a possible cluster write
1368 */
1369 if (ncl != 1) {
1370 nwritten = cluster_wbuild(vp, size, lblkno - j, ncl);
1371 splx(s);
1372 return nwritten;
1373 }
1374 }
1375
1376 BUF_LOCK(bp, LK_EXCLUSIVE);
1377 bremfree(bp);
1378 bp->b_flags |= B_ASYNC;
1379
1380 splx(s);
1381 /*
1382 * default (old) behavior, writing out only one block
1383 *
1384 * XXX returns b_bufsize instead of b_bcount for nwritten?
1385 */
1386 nwritten = bp->b_bufsize;
1387 (void) BUF_WRITE(bp);
1388
1389 return nwritten;
1390}
1391
1392/*
1393 * getnewbuf:
1394 *
1395 * Find and initialize a new buffer header, freeing up existing buffers
1396 * in the bufqueues as necessary. The new buffer is returned locked.
1397 *
1398 * Important: B_INVAL is not set. If the caller wishes to throw the
1399 * buffer away, the caller must set B_INVAL prior to calling brelse().
1400 *
1401 * We block if:
1402 * We have insufficient buffer headers
1403 * We have insufficient buffer space
1404 * buffer_map is too fragmented ( space reservation fails )
1405 * If we have to flush dirty buffers ( but we try to avoid this )
1406 *
1407 * To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1408 * Instead we ask the buf daemon to do it for us. We attempt to
1409 * avoid piecemeal wakeups of the pageout daemon.
1410 */
1411
1412static struct buf *
1413getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
1414{
1415 struct buf *bp;
1416 struct buf *nbp;
1417 int defrag = 0;
1418 int nqindex;
1419 int isspecial;
1420 static int flushingbufs;
1421
1422 if (curproc && (curproc->p_flag & P_BUFEXHAUST) == 0)
1423 isspecial = 0;
1424 else
1425 isspecial = 1;
1426
1427 ++getnewbufcalls;
1428 --getnewbufrestarts;
1429restart:
1430 ++getnewbufrestarts;
1431
1432 /*
1433 * Setup for scan. If we do not have enough free buffers,
1434 * we setup a degenerate case that immediately fails. Note
1435 * that if we are specially marked process, we are allowed to
1436 * dip into our reserves.
1437 *
1438 * The scanning sequence is nominally: EMPTY->EMPTYKVA->CLEAN
1439 *
1440 * We start with EMPTYKVA. If the list is empty we backup to EMPTY.
1441 * However, there are a number of cases (defragging, reusing, ...)
1442 * where we cannot backup.
1443 */
1444
1445 if (isspecial == 0 && numfreebuffers < lofreebuffers) {
1446 /*
1447 * This will cause an immediate failure
1448 */
1449 nqindex = QUEUE_CLEAN;
1450 nbp = NULL;
1451 } else {
1452 /*
1453 * Locate a buffer which already has KVA assigned. First
1454 * try EMPTYKVA buffers.
1455 */
1456 nqindex = QUEUE_EMPTYKVA;
1457 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA]);
1458
1459 if (nbp == NULL) {
1460 /*
1461 * If no EMPTYKVA buffers and we are either
1462 * defragging or reusing, locate a CLEAN buffer
1463 * to free or reuse. If bufspace useage is low
1464 * skip this step so we can allocate a new buffer.
1465 */
1466 if (defrag || bufspace >= lobufspace) {
1467 nqindex = QUEUE_CLEAN;
1468 nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
1469 }
1470
1471 /*
1472 * Nada. If we are allowed to allocate an EMPTY
1473 * buffer, go get one.
1474 */
1475 if (nbp == NULL && defrag == 0 &&
1476 (isspecial || bufspace < hibufspace)) {
1477 nqindex = QUEUE_EMPTY;
1478 nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1479 }
1480 }
1481 }
1482
1483 /*
1484 * Run scan, possibly freeing data and/or kva mappings on the fly
1485 * depending.
1486 */
1487
1488 while ((bp = nbp) != NULL) {
1489 int qindex = nqindex;
1490
1491 /*
1492 * Calculate next bp ( we can only use it if we do not block
1493 * or do other fancy things ).
1494 */
1495 if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
1496 switch(qindex) {
1497 case QUEUE_EMPTY:
1498 nqindex = QUEUE_EMPTYKVA;
1499 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTYKVA])))
1500 break;
1501 /* fall through */
1502 case QUEUE_EMPTYKVA:
1503 nqindex = QUEUE_CLEAN;
1504 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN])))
1505 break;
1506 /* fall through */
1507 case QUEUE_CLEAN:
1508 /*
1509 * nbp is NULL.
1510 */
1511 break;
1512 }
1513 }
1514
1515 /*
1516 * Sanity Checks
1517 */
1518 KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp));
1519
1520 /*
1521 * Note: we no longer distinguish between VMIO and non-VMIO
1522 * buffers.
1523 */
1524
1525 KASSERT((bp->b_flags & B_DELWRI) == 0, ("delwri buffer %p found in queue %d", bp, qindex));
1526
1527 /*
1528 * If we are defragging then we need a buffer with
1529 * b_kvasize != 0. XXX this situation should no longer
1530 * occur, if defrag is non-zero the buffer's b_kvasize
1531 * should also be non-zero at this point. XXX
1532 */
1533 if (defrag && bp->b_kvasize == 0) {
1534 printf("Warning: defrag empty buffer %p\n", bp);
1535 continue;
1536 }
1537
1538 /*
1539 * Start freeing the bp. This is somewhat involved. nbp
1540 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1541 */
1542
1543 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1544 panic("getnewbuf: locked buf");
1545 bremfree(bp);
1546
1547 if (qindex == QUEUE_CLEAN) {
1548 if (bp->b_flags & B_VMIO) {
1549 bp->b_flags &= ~B_ASYNC;
1550 vfs_vmio_release(bp);
1551 }
1552 if (bp->b_vp)
1553 brelvp(bp);
1554 }
1555
1556 /*
1557 * NOTE: nbp is now entirely invalid. We can only restart
1558 * the scan from this point on.
1559 *
1560 * Get the rest of the buffer freed up. b_kva* is still
1561 * valid after this operation.
1562 */
1563
1564 if (bp->b_rcred != NOCRED) {
1565 crfree(bp->b_rcred);
1566 bp->b_rcred = NOCRED;
1567 }
1568 if (bp->b_wcred != NOCRED) {
1569 crfree(bp->b_wcred);
1570 bp->b_wcred = NOCRED;
1571 }
1572 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
1573 (*bioops.io_deallocate)(bp);
1574 if (bp->b_xflags & BX_BKGRDINPROG)
1575 panic("losing buffer 3");
1576 LIST_REMOVE(bp, b_hash);
1577 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1578
1579 if (bp->b_bufsize)
1580 allocbuf(bp, 0);
1581
1582 bp->b_flags = 0;
1583 bp->b_ioflags = 0;
1584 bp->b_xflags = 0;
1585 bp->b_dev = NODEV;
1586 bp->b_vp = NULL;
1587 bp->b_blkno = bp->b_lblkno = 0;
1588 bp->b_offset = NOOFFSET;
1589 bp->b_iodone = 0;
1590 bp->b_error = 0;
1591 bp->b_resid = 0;
1592 bp->b_bcount = 0;
1593 bp->b_npages = 0;
1594 bp->b_dirtyoff = bp->b_dirtyend = 0;
1595
1596 LIST_INIT(&bp->b_dep);
1597
1598 /*
1599 * If we are defragging then free the buffer.
1600 */
1601 if (defrag) {
1602 bp->b_flags |= B_INVAL;
1603 bfreekva(bp);
1604 brelse(bp);
1605 defrag = 0;
1606 goto restart;
1607 }
1608
1609 /*
1610 * If we are a normal process then deal with bufspace
1611 * hysteresis. A normal process tries to keep bufspace
1612 * between lobufspace and hibufspace. Note: if we encounter
1613 * a buffer with b_kvasize == 0 then it means we started
1614 * our scan on the EMPTY list and should allocate a new
1615 * buffer.
1616 */
1617 if (isspecial == 0) {
1618 if (bufspace > hibufspace)
1619 flushingbufs = 1;
1620 if (flushingbufs && bp->b_kvasize != 0) {
1621 bp->b_flags |= B_INVAL;
1622 bfreekva(bp);
1623 brelse(bp);
1624 goto restart;
1625 }
1626 if (bufspace < lobufspace)
1627 flushingbufs = 0;
1628 }
1629 break;
1630 }
1631
1632 /*
1633 * If we exhausted our list, sleep as appropriate. We may have to
1634 * wakeup various daemons and write out some dirty buffers.
1635 *
1636 * Generally we are sleeping due to insufficient buffer space.
1637 */
1638
1639 if (bp == NULL) {
1640 int flags;
1641 char *waitmsg;
1642
1643 if (defrag) {
1644 flags = VFS_BIO_NEED_BUFSPACE;
1645 waitmsg = "nbufkv";
1646 } else if (bufspace >= hibufspace) {
1647 waitmsg = "nbufbs";
1648 flags = VFS_BIO_NEED_BUFSPACE;
1649 } else {
1650 waitmsg = "newbuf";
1651 flags = VFS_BIO_NEED_ANY;
1652 }
1653
1654 bd_speedup(); /* heeeelp */
1655
1656 needsbuffer |= flags;
1657 while (needsbuffer & flags) {
1658 if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag,
1659 waitmsg, slptimeo))
1660 return (NULL);
1661 }
1662 } else {
1663 /*
1664 * We finally have a valid bp. We aren't quite out of the
1665 * woods, we still have to reserve kva space. In order
1666 * to keep fragmentation sane we only allocate kva in
1667 * BKVASIZE chunks.
1668 */
1669 maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
1670
1671 if (maxsize != bp->b_kvasize) {
1672 vm_offset_t addr = 0;
1673
1674 bfreekva(bp);
1675
1676 if (vm_map_findspace(buffer_map,
1677 vm_map_min(buffer_map), maxsize, &addr)) {
1678 /*
1679 * Uh oh. Buffer map is to fragmented. We
1680 * must defragment the map.
1681 */
1682 ++bufdefragcnt;
1683 defrag = 1;
1684 bp->b_flags |= B_INVAL;
1685 brelse(bp);
1686 goto restart;
1687 }
1688 if (addr) {
1689 vm_map_insert(buffer_map, NULL, 0,
1690 addr, addr + maxsize,
1691 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
1692
1693 bp->b_kvabase = (caddr_t) addr;
1694 bp->b_kvasize = maxsize;
1695 bufspace += bp->b_kvasize;
1696 ++bufreusecnt;
1697 }
1698 }
1699 bp->b_data = bp->b_kvabase;
1700 }
1701 return(bp);
1702}
1703
1704/*
1705 * waitfreebuffers:
1706 *
1707 * Wait for sufficient free buffers. Only called from normal processes.
1708 */
1709
1710static void
1711waitfreebuffers(int slpflag, int slptimeo)
1712{
1713 while (numfreebuffers < hifreebuffers) {
1714 if (numfreebuffers >= hifreebuffers)
1715 break;
1716 needsbuffer |= VFS_BIO_NEED_FREE;
1717 if (tsleep(&needsbuffer, (PRIBIO + 4)|slpflag, "biofre", slptimeo))
1718 break;
1719 }
1720}
1721
1722/*
1723 * buf_daemon:
1724 *
1725 * buffer flushing daemon. Buffers are normally flushed by the
1726 * update daemon but if it cannot keep up this process starts to
1727 * take the load in an attempt to prevent getnewbuf() from blocking.
1728 */
1729
1730static struct proc *bufdaemonproc;
1731static int bd_interval;
1732static int bd_flushto;
1733static int bd_flushinc;
1734
1735static struct kproc_desc buf_kp = {
1736 "bufdaemon",
1737 buf_daemon,
1738 &bufdaemonproc
1739};
1740SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp)
1741
1742static void
1743buf_daemon()
1744{
1745 int s;
1746
1747 /*
1748 * This process needs to be suspended prior to shutdown sync.
1749 */
1750 EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc, bufdaemonproc,
1751 SHUTDOWN_PRI_LAST);
1752
1753 /*
1754 * This process is allowed to take the buffer cache to the limit
1755 */
1756 curproc->p_flag |= P_BUFEXHAUST;
1757 s = splbio();
1758
1759 bd_interval = 5 * hz; /* dynamically adjusted */
1760 bd_flushto = hidirtybuffers; /* dynamically adjusted */
1761 bd_flushinc = 1;
1762
1763 for (;;) {
1764 kproc_suspend_loop(bufdaemonproc);
1765
1766 bd_request = 0;
1767
1768 /*
1769 * Do the flush. Limit the number of buffers we flush in one
1770 * go. The failure condition occurs when processes are writing
1771 * buffers faster then we can dispose of them. In this case
1772 * we may be flushing so often that the previous set of flushes
1773 * have not had time to complete, causing us to run out of
1774 * physical buffers and block.
1775 */
1776 {
1777 int runcount = maxbdrun;
1778
1779 while (numdirtybuffers > bd_flushto && runcount) {
1780 --runcount;
1781 if (flushbufqueues() == 0)
1782 break;
1783 }
1784 }
1785
1786 if (bd_request ||
1787 tsleep(&bd_request, PVM, "psleep", bd_interval) == 0) {
1788 /*
1789 * Another request is pending or we were woken up
1790 * without timing out. Flush more.
1791 */
1792 --bd_flushto;
1793 if (bd_flushto >= numdirtybuffers - 5) {
1794 bd_flushto = numdirtybuffers - 10;
1795 bd_flushinc = 1;
1796 }
1797 if (bd_flushto < 2)
1798 bd_flushto = 2;
1799 } else {
1800 /*
1801 * We slept and timed out, we can slow down.
1802 */
1803 bd_flushto += bd_flushinc;
1804 if (bd_flushto > hidirtybuffers)
1805 bd_flushto = hidirtybuffers;
1806 ++bd_flushinc;
1807 if (bd_flushinc > hidirtybuffers / 20 + 1)
1808 bd_flushinc = hidirtybuffers / 20 + 1;
1809 }
1810
1811 /*
1812 * Set the interval on a linear scale based on hidirtybuffers
1813 * with a maximum frequency of 1/10 second.
1814 */
1815 bd_interval = bd_flushto * 5 * hz / hidirtybuffers;
1816 if (bd_interval < hz / 10)
1817 bd_interval = hz / 10;
1818 }
1819}
1820
1821/*
1822 * flushbufqueues:
1823 *
1824 * Try to flush a buffer in the dirty queue. We must be careful to
1825 * free up B_INVAL buffers instead of write them, which NFS is
1826 * particularly sensitive to.
1827 */
1828
1829static int
1830flushbufqueues(void)
1831{
1832 struct buf *bp;
1833 int r = 0;
1834
1835 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1836
1837 while (bp) {
1838 KASSERT((bp->b_flags & B_DELWRI), ("unexpected clean buffer %p", bp));
1839 if ((bp->b_flags & B_DELWRI) != 0 &&
1840 (bp->b_xflags & BX_BKGRDINPROG) == 0) {
1841 if (bp->b_flags & B_INVAL) {
1842 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT) != 0)
1843 panic("flushbufqueues: locked buf");
1844 bremfree(bp);
1845 brelse(bp);
1846 ++r;
1847 break;
1848 }
1849 if (LIST_FIRST(&bp->b_dep) != NULL &&
1850 bioops.io_countdeps &&
1851 (bp->b_flags & B_DEFERRED) == 0 &&
1852 (*bioops.io_countdeps)(bp, 0)) {
1853 TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY],
1854 bp, b_freelist);
1855 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY],
1856 bp, b_freelist);
1857 bp->b_flags |= B_DEFERRED;
1858 bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
1859 continue;
1860 }
1861 vfs_bio_awrite(bp);
1862 ++r;
1863 break;
1864 }
1865 bp = TAILQ_NEXT(bp, b_freelist);
1866 }
1867 return (r);
1868}
1869
1870/*
1871 * Check to see if a block is currently memory resident.
1872 */
1873struct buf *
1874incore(struct vnode * vp, daddr_t blkno)
1875{
1876 struct buf *bp;
1877
1878 int s = splbio();
1879 bp = gbincore(vp, blkno);
1880 splx(s);
1881 return (bp);
1882}
1883
1884/*
1885 * Returns true if no I/O is needed to access the
1886 * associated VM object. This is like incore except
1887 * it also hunts around in the VM system for the data.
1888 */
1889
1890int
1891inmem(struct vnode * vp, daddr_t blkno)
1892{
1893 vm_object_t obj;
1894 vm_offset_t toff, tinc, size;
1895 vm_page_t m;
1896 vm_ooffset_t off;
1897
1898 if (incore(vp, blkno))
1899 return 1;
1900 if (vp->v_mount == NULL)
1901 return 0;
1902 if ((vp->v_object == NULL) || (vp->v_flag & VOBJBUF) == 0)
1903 return 0;
1904
1905 obj = vp->v_object;
1906 size = PAGE_SIZE;
1907 if (size > vp->v_mount->mnt_stat.f_iosize)
1908 size = vp->v_mount->mnt_stat.f_iosize;
1909 off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
1910
1911 for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
1912 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
1913 if (!m)
1914 return 0;
1915 tinc = size;
1916 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
1917 tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
1918 if (vm_page_is_valid(m,
1919 (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
1920 return 0;
1921 }
1922 return 1;
1923}
1924
1925/*
1926 * vfs_setdirty:
1927 *
1928 * Sets the dirty range for a buffer based on the status of the dirty
1929 * bits in the pages comprising the buffer.
1930 *
1931 * The range is limited to the size of the buffer.
1932 *
1933 * This routine is primarily used by NFS, but is generalized for the
1934 * B_VMIO case.
1935 */
1936static void
1937vfs_setdirty(struct buf *bp)
1938{
1939 int i;
1940 vm_object_t object;
1941
1942 /*
1943 * Degenerate case - empty buffer
1944 */
1945
1946 if (bp->b_bufsize == 0)
1947 return;
1948
1949 /*
1950 * We qualify the scan for modified pages on whether the
1951 * object has been flushed yet. The OBJ_WRITEABLE flag
1952 * is not cleared simply by protecting pages off.
1953 */
1954
1955 if ((bp->b_flags & B_VMIO) == 0)
1956 return;
1957
1958 object = bp->b_pages[0]->object;
1959
1960 if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY))
1961 printf("Warning: object %p writeable but not mightbedirty\n", object);
1962 if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY))
1963 printf("Warning: object %p mightbedirty but not writeable\n", object);
1964
1965 if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) {
1966 vm_offset_t boffset;
1967 vm_offset_t eoffset;
1968
1969 /*
1970 * test the pages to see if they have been modified directly
1971 * by users through the VM system.
1972 */
1973 for (i = 0; i < bp->b_npages; i++) {
1974 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
1975 vm_page_test_dirty(bp->b_pages[i]);
1976 }
1977
1978 /*
1979 * Calculate the encompassing dirty range, boffset and eoffset,
1980 * (eoffset - boffset) bytes.
1981 */
1982
1983 for (i = 0; i < bp->b_npages; i++) {
1984 if (bp->b_pages[i]->dirty)
1985 break;
1986 }
1987 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1988
1989 for (i = bp->b_npages - 1; i >= 0; --i) {
1990 if (bp->b_pages[i]->dirty) {
1991 break;
1992 }
1993 }
1994 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
1995
1996 /*
1997 * Fit it to the buffer.
1998 */
1999
2000 if (eoffset > bp->b_bcount)
2001 eoffset = bp->b_bcount;
2002
2003 /*
2004 * If we have a good dirty range, merge with the existing
2005 * dirty range.
2006 */
2007
2008 if (boffset < eoffset) {
2009 if (bp->b_dirtyoff > boffset)
2010 bp->b_dirtyoff = boffset;
2011 if (bp->b_dirtyend < eoffset)
2012 bp->b_dirtyend = eoffset;
2013 }
2014 }
2015}
2016
2017/*
2018 * getblk:
2019 *
2020 * Get a block given a specified block and offset into a file/device.
2021 * The buffers B_DONE bit will be cleared on return, making it almost
2022 * ready for an I/O initiation. B_INVAL may or may not be set on
2023 * return. The caller should clear B_INVAL prior to initiating a
2024 * READ.
2025 *
2026 * For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2027 * an existing buffer.
2028 *
2029 * For a VMIO buffer, B_CACHE is modified according to the backing VM.
2030 * If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2031 * and then cleared based on the backing VM. If the previous buffer is
2032 * non-0-sized but invalid, B_CACHE will be cleared.
2033 *
2034 * If getblk() must create a new buffer, the new buffer is returned with
2035 * both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2036 * case it is returned with B_INVAL clear and B_CACHE set based on the
2037 * backing VM.
2038 *
2039 * getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2040 * B_CACHE bit is clear.
2041 *
2042 * What this means, basically, is that the caller should use B_CACHE to
2043 * determine whether the buffer is fully valid or not and should clear
2044 * B_INVAL prior to issuing a read. If the caller intends to validate
2045 * the buffer by loading its data area with something, the caller needs
2046 * to clear B_INVAL. If the caller does this without issuing an I/O,
2047 * the caller should set B_CACHE ( as an optimization ), else the caller
2048 * should issue the I/O and biodone() will set B_CACHE if the I/O was
2049 * a write attempt or if it was a successfull read. If the caller
2050 * intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
2051 * prior to issuing the READ. biodone() will *not* clear B_INVAL.
2052 */
2053struct buf *
2054getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo)
2055{
2056 struct buf *bp;
2057 int s;
2058 struct bufhashhdr *bh;
2059
2060 if (size > MAXBSIZE)
2061 panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE);
2062
2063 s = splbio();
2064loop:
2065 /*
2066 * Block if we are low on buffers. Certain processes are allowed
2067 * to completely exhaust the buffer cache.
2068 *
2069 * If this check ever becomes a bottleneck it may be better to
2070 * move it into the else, when gbincore() fails. At the moment
2071 * it isn't a problem.
2072 */
2073 if (!curproc || (curproc->p_flag & P_BUFEXHAUST)) {
2074 if (numfreebuffers == 0) {
2075 if (!curproc)
2076 return NULL;
2077 needsbuffer |= VFS_BIO_NEED_ANY;
2078 tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, "newbuf",
2079 slptimeo);
2080 }
2081 } else if (numfreebuffers < lofreebuffers) {
2082 waitfreebuffers(slpflag, slptimeo);
2083 }
2084
2085 if ((bp = gbincore(vp, blkno))) {
2086 /*
2087 * Buffer is in-core. If the buffer is not busy, it must
2088 * be on a queue.
2089 */
2090
2091 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
2092 if (BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL,
2093 "getblk", slpflag, slptimeo) == ENOLCK)
2094 goto loop;
2095 splx(s);
2096 return (struct buf *) NULL;
2097 }
2098
2099 /*
2100 * The buffer is locked. B_CACHE is cleared if the buffer is
2101 * invalid. Ohterwise, for a non-VMIO buffer, B_CACHE is set
2102 * and for a VMIO buffer B_CACHE is adjusted according to the
2103 * backing VM cache.
2104 */
2105 if (bp->b_flags & B_INVAL)
2106 bp->b_flags &= ~B_CACHE;
2107 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
2108 bp->b_flags |= B_CACHE;
2109 bremfree(bp);
2110
2111 /*
2112 * check for size inconsistancies for non-VMIO case.
2113 */
2114
2115 if (bp->b_bcount != size) {
2116 if ((bp->b_flags & B_VMIO) == 0 ||
2117 (size > bp->b_kvasize)) {
2118 if (bp->b_flags & B_DELWRI) {
2119 bp->b_flags |= B_NOCACHE;
2120 BUF_WRITE(bp);
2121 } else {
2122 if ((bp->b_flags & B_VMIO) &&
2123 (LIST_FIRST(&bp->b_dep) == NULL)) {
2124 bp->b_flags |= B_RELBUF;
2125 brelse(bp);
2126 } else {
2127 bp->b_flags |= B_NOCACHE;
2128 BUF_WRITE(bp);
2129 }
2130 }
2131 goto loop;
2132 }
2133 }
2134
2135 /*
2136 * If the size is inconsistant in the VMIO case, we can resize
2137 * the buffer. This might lead to B_CACHE getting set or
2138 * cleared. If the size has not changed, B_CACHE remains
2139 * unchanged from its previous state.
2140 */
2141
2142 if (bp->b_bcount != size)
2143 allocbuf(bp, size);
2144
2145 KASSERT(bp->b_offset != NOOFFSET,
2146 ("getblk: no buffer offset"));
2147
2148 /*
2149 * A buffer with B_DELWRI set and B_CACHE clear must
2150 * be committed before we can return the buffer in
2151 * order to prevent the caller from issuing a read
2152 * ( due to B_CACHE not being set ) and overwriting
2153 * it.
2154 *
2155 * Most callers, including NFS and FFS, need this to
2156 * operate properly either because they assume they
2157 * can issue a read if B_CACHE is not set, or because
2158 * ( for example ) an uncached B_DELWRI might loop due
2159 * to softupdates re-dirtying the buffer. In the latter
2160 * case, B_CACHE is set after the first write completes,
2161 * preventing further loops.
2162 */
2163
2164 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
2165 BUF_WRITE(bp);
2166 goto loop;
2167 }
2168
2169 splx(s);
2170 bp->b_flags &= ~B_DONE;
2171 } else {
2172 /*
2173 * Buffer is not in-core, create new buffer. The buffer
2174 * returned by getnewbuf() is locked. Note that the returned
2175 * buffer is also considered valid (not marked B_INVAL).
2176 */
2177 int bsize, maxsize, vmio;
2178 off_t offset;
2179
2180 if (vn_isdisk(vp, NULL))
2181 bsize = DEV_BSIZE;
2182 else if (vp->v_mountedhere)
2183 bsize = vp->v_mountedhere->mnt_stat.f_iosize;
2184 else if (vp->v_mount)
2185 bsize = vp->v_mount->mnt_stat.f_iosize;
2186 else
2187 bsize = size;
2188
2189 offset = (off_t)blkno * bsize;
2190 vmio = (vp->v_object != 0) && (vp->v_flag & VOBJBUF);
2191 maxsize = vmio ? size + (offset & PAGE_MASK) : size;
2192 maxsize = imax(maxsize, bsize);
2193
2194 if ((bp = getnewbuf(slpflag, slptimeo, size, maxsize)) == NULL) {
2195 if (slpflag || slptimeo) {
2196 splx(s);
2197 return NULL;
2198 }
2199 goto loop;
2200 }
2201
2202 /*
2203 * This code is used to make sure that a buffer is not
2204 * created while the getnewbuf routine is blocked.
2205 * This can be a problem whether the vnode is locked or not.
2206 * If the buffer is created out from under us, we have to
2207 * throw away the one we just created. There is now window
2208 * race because we are safely running at splbio() from the
2209 * point of the duplicate buffer creation through to here,
2210 * and we've locked the buffer.
2211 */
2212 if (gbincore(vp, blkno)) {
2213 bp->b_flags |= B_INVAL;
2214 brelse(bp);
2215 goto loop;
2216 }
2217
2218 /*
2219 * Insert the buffer into the hash, so that it can
2220 * be found by incore.
2221 */
2222 bp->b_blkno = bp->b_lblkno = blkno;
2223 bp->b_offset = offset;
2224
2225 bgetvp(vp, bp);
2226 LIST_REMOVE(bp, b_hash);
2227 bh = bufhash(vp, blkno);
2228 LIST_INSERT_HEAD(bh, bp, b_hash);
2229
2230 /*
2231 * set B_VMIO bit. allocbuf() the buffer bigger. Since the
2232 * buffer size starts out as 0, B_CACHE will be set by
2233 * allocbuf() for the VMIO case prior to it testing the
2234 * backing store for validity.
2235 */
2236
2237 if (vmio) {
2238 bp->b_flags |= B_VMIO;
2239#if defined(VFS_BIO_DEBUG)
2240 if (vp->v_type != VREG && vp->v_type != VBLK)
2241 printf("getblk: vmioing file type %d???\n", vp->v_type);
2242#endif
2243 } else {
2244 bp->b_flags &= ~B_VMIO;
2245 }
2246
2247 allocbuf(bp, size);
2248
2249 splx(s);
2250 bp->b_flags &= ~B_DONE;
2251 }
2252 return (bp);
2253}
2254
2255/*
2256 * Get an empty, disassociated buffer of given size. The buffer is initially
2257 * set to B_INVAL.
2258 */
2259struct buf *
2260geteblk(int size)
2261{
2262 struct buf *bp;
2263 int s;
2264 int maxsize;
2265
2266 maxsize = (size + BKVAMASK) & ~BKVAMASK;
2267
2268 s = splbio();
2269 while ((bp = getnewbuf(0, 0, size, maxsize)) == 0);
2270 splx(s);
2271 allocbuf(bp, size);
2272 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
2273 return (bp);
2274}
2275
2276
2277/*
2278 * This code constitutes the buffer memory from either anonymous system
2279 * memory (in the case of non-VMIO operations) or from an associated
2280 * VM object (in the case of VMIO operations). This code is able to
2281 * resize a buffer up or down.
2282 *
2283 * Note that this code is tricky, and has many complications to resolve
2284 * deadlock or inconsistant data situations. Tread lightly!!!
2285 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2286 * the caller. Calling this code willy nilly can result in the loss of data.
2287 *
2288 * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
2289 * B_CACHE for the non-VMIO case.
2290 */
2291
2292int
2293allocbuf(struct buf *bp, int size)
2294{
2295 int newbsize, mbsize;
2296 int i;
2297
2298 if (BUF_REFCNT(bp) == 0)
2299 panic("allocbuf: buffer not busy");
2300
2301 if (bp->b_kvasize < size)
2302 panic("allocbuf: buffer too small");
2303
2304 if ((bp->b_flags & B_VMIO) == 0) {
2305 caddr_t origbuf;
2306 int origbufsize;
2307 /*
2308 * Just get anonymous memory from the kernel. Don't
2309 * mess with B_CACHE.
2310 */
2311 mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2312#if !defined(NO_B_MALLOC)
2313 if (bp->b_flags & B_MALLOC)
2314 newbsize = mbsize;
2315 else
2316#endif
2317 newbsize = round_page(size);
2318
2319 if (newbsize < bp->b_bufsize) {
2320#if !defined(NO_B_MALLOC)
2321 /*
2322 * malloced buffers are not shrunk
2323 */
2324 if (bp->b_flags & B_MALLOC) {
2325 if (newbsize) {
2326 bp->b_bcount = size;
2327 } else {
2328 free(bp->b_data, M_BIOBUF);
2329 bufmallocspace -= bp->b_bufsize;
2330 runningbufspace -= bp->b_bufsize;
2331 if (bp->b_bufsize)
2332 bufspacewakeup();
2333 bp->b_data = bp->b_kvabase;
2334 bp->b_bufsize = 0;
2335 bp->b_bcount = 0;
2336 bp->b_flags &= ~B_MALLOC;
2337 }
2338 return 1;
2339 }
2340#endif
2341 vm_hold_free_pages(
2342 bp,
2343 (vm_offset_t) bp->b_data + newbsize,
2344 (vm_offset_t) bp->b_data + bp->b_bufsize);
2345 } else if (newbsize > bp->b_bufsize) {
2346#if !defined(NO_B_MALLOC)
2347 /*
2348 * We only use malloced memory on the first allocation.
2349 * and revert to page-allocated memory when the buffer
2350 * grows.
2351 */
2352 if ( (bufmallocspace < maxbufmallocspace) &&
2353 (bp->b_bufsize == 0) &&
2354 (mbsize <= PAGE_SIZE/2)) {
2355
2356 bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
2357 bp->b_bufsize = mbsize;
2358 bp->b_bcount = size;
2359 bp->b_flags |= B_MALLOC;
2360 bufmallocspace += mbsize;
2361 runningbufspace += bp->b_bufsize;
2362 return 1;
2363 }
2364#endif
2365 origbuf = NULL;
2366 origbufsize = 0;
2367#if !defined(NO_B_MALLOC)
2368 /*
2369 * If the buffer is growing on its other-than-first allocation,
2370 * then we revert to the page-allocation scheme.
2371 */
2372 if (bp->b_flags & B_MALLOC) {
2373 origbuf = bp->b_data;
2374 origbufsize = bp->b_bufsize;
2375 bp->b_data = bp->b_kvabase;
2376 bufmallocspace -= bp->b_bufsize;
2377 runningbufspace -= bp->b_bufsize;
2378 if (bp->b_bufsize)
2379 bufspacewakeup();
2380 bp->b_bufsize = 0;
2381 bp->b_flags &= ~B_MALLOC;
2382 newbsize = round_page(newbsize);
2383 }
2384#endif
2385 vm_hold_load_pages(
2386 bp,
2387 (vm_offset_t) bp->b_data + bp->b_bufsize,
2388 (vm_offset_t) bp->b_data + newbsize);
2389#if !defined(NO_B_MALLOC)
2390 if (origbuf) {
2391 bcopy(origbuf, bp->b_data, origbufsize);
2392 free(origbuf, M_BIOBUF);
2393 }
2394#endif
2395 }
2396 } else {
2397 vm_page_t m;
2398 int desiredpages;
2399
2400 newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
2401 desiredpages = (size == 0) ? 0 :
2402 num_pages((bp->b_offset & PAGE_MASK) + newbsize);
2403
2404#if !defined(NO_B_MALLOC)
2405 if (bp->b_flags & B_MALLOC)
2406 panic("allocbuf: VMIO buffer can't be malloced");
2407#endif
2408 /*
2409 * Set B_CACHE initially if buffer is 0 length or will become
2410 * 0-length.
2411 */
2412 if (size == 0 || bp->b_bufsize == 0)
2413 bp->b_flags |= B_CACHE;
2414
2415 if (newbsize < bp->b_bufsize) {
2416 /*
2417 * DEV_BSIZE aligned new buffer size is less then the
2418 * DEV_BSIZE aligned existing buffer size. Figure out
2419 * if we have to remove any pages.
2420 */
2421 if (desiredpages < bp->b_npages) {
2422 for (i = desiredpages; i < bp->b_npages; i++) {
2423 /*
2424 * the page is not freed here -- it
2425 * is the responsibility of
2426 * vnode_pager_setsize
2427 */
2428 m = bp->b_pages[i];
2429 KASSERT(m != bogus_page,
2430 ("allocbuf: bogus page found"));
2431 while (vm_page_sleep_busy(m, TRUE, "biodep"))
2432 ;
2433
2434 bp->b_pages[i] = NULL;
2435 vm_page_unwire(m, 0);
2436 }
2437 pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) +
2438 (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages));
2439 bp->b_npages = desiredpages;
2440 }
2441 } else if (size > bp->b_bcount) {
2442 /*
2443 * We are growing the buffer, possibly in a
2444 * byte-granular fashion.
2445 */
2446 struct vnode *vp;
2447 vm_object_t obj;
2448 vm_offset_t toff;
2449 vm_offset_t tinc;
2450
2451 /*
2452 * Step 1, bring in the VM pages from the object,
2453 * allocating them if necessary. We must clear
2454 * B_CACHE if these pages are not valid for the
2455 * range covered by the buffer.
2456 */
2457
2458 vp = bp->b_vp;
2459 obj = vp->v_object;
2460
2461 while (bp->b_npages < desiredpages) {
2462 vm_page_t m;
2463 vm_pindex_t pi;
2464
2465 pi = OFF_TO_IDX(bp->b_offset) + bp->b_npages;
2466 if ((m = vm_page_lookup(obj, pi)) == NULL) {
2467 m = vm_page_alloc(obj, pi, VM_ALLOC_NORMAL);
2468 if (m == NULL) {
2469 VM_WAIT;
2470 vm_pageout_deficit += desiredpages - bp->b_npages;
2471 } else {
2472 vm_page_wire(m);
2473 vm_page_wakeup(m);
2474 bp->b_flags &= ~B_CACHE;
2475 bp->b_pages[bp->b_npages] = m;
2476 ++bp->b_npages;
2477 }
2478 continue;
2479 }
2480
2481 /*
2482 * We found a page. If we have to sleep on it,
2483 * retry because it might have gotten freed out
2484 * from under us.
2485 *
2486 * We can only test PG_BUSY here. Blocking on
2487 * m->busy might lead to a deadlock:
2488 *
2489 * vm_fault->getpages->cluster_read->allocbuf
2490 *
2491 */
2492
2493 if (vm_page_sleep_busy(m, FALSE, "pgtblk"))
2494 continue;
2495
2496 /*
2497 * We have a good page. Should we wakeup the
2498 * page daemon?
2499 */
2500 if ((curproc != pageproc) &&
2501 ((m->queue - m->pc) == PQ_CACHE) &&
2502 ((cnt.v_free_count + cnt.v_cache_count) <
2503 (cnt.v_free_min + cnt.v_cache_min))) {
2504 pagedaemon_wakeup();
2505 }
2506 vm_page_flag_clear(m, PG_ZERO);
2507 vm_page_wire(m);
2508 bp->b_pages[bp->b_npages] = m;
2509 ++bp->b_npages;
2510 }
2511
2512 /*
2513 * Step 2. We've loaded the pages into the buffer,
2514 * we have to figure out if we can still have B_CACHE
2515 * set. Note that B_CACHE is set according to the
2516 * byte-granular range ( bcount and size ), new the
2517 * aligned range ( newbsize ).
2518 *
2519 * The VM test is against m->valid, which is DEV_BSIZE
2520 * aligned. Needless to say, the validity of the data
2521 * needs to also be DEV_BSIZE aligned. Note that this
2522 * fails with NFS if the server or some other client
2523 * extends the file's EOF. If our buffer is resized,
2524 * B_CACHE may remain set! XXX
2525 */
2526
2527 toff = bp->b_bcount;
2528 tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2529
2530 while ((bp->b_flags & B_CACHE) && toff < size) {
2531 vm_pindex_t pi;
2532
2533 if (tinc > (size - toff))
2534 tinc = size - toff;
2535
2536 pi = ((bp->b_offset & PAGE_MASK) + toff) >>
2537 PAGE_SHIFT;
2538
2539 vfs_buf_test_cache(
2540 bp,
2541 bp->b_offset,
2542 toff,
2543 tinc,
2544 bp->b_pages[pi]
2545 );
2546 toff += tinc;
2547 tinc = PAGE_SIZE;
2548 }
2549
2550 /*
2551 * Step 3, fixup the KVM pmap. Remember that
2552 * bp->b_data is relative to bp->b_offset, but
2553 * bp->b_offset may be offset into the first page.
2554 */
2555
2556 bp->b_data = (caddr_t)
2557 trunc_page((vm_offset_t)bp->b_data);
2558 pmap_qenter(
2559 (vm_offset_t)bp->b_data,
2560 bp->b_pages,
2561 bp->b_npages
2562 );
2563 bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
2564 (vm_offset_t)(bp->b_offset & PAGE_MASK));
2565 }
2566 }
2567 runningbufspace += (newbsize - bp->b_bufsize);
2568 if (newbsize < bp->b_bufsize)
2569 bufspacewakeup();
2570 bp->b_bufsize = newbsize; /* actual buffer allocation */
2571 bp->b_bcount = size; /* requested buffer size */
2572 return 1;
2573}
2574
2575/*
2576 * biowait:
2577 *
2578 * Wait for buffer I/O completion, returning error status. The buffer
2579 * is left locked and B_DONE on return. B_EINTR is converted into a EINTR
2580 * error and cleared.
2581 */
2582int
2583biowait(register struct buf * bp)
2584{
2585 int s;
2586
2587 s = splbio();
2588 while ((bp->b_flags & B_DONE) == 0) {
2589#if defined(NO_SCHEDULE_MODS)
2590 tsleep(bp, PRIBIO, "biowait", 0);
2591#else
2592 if (bp->b_iocmd == BIO_READ)
2593 tsleep(bp, PRIBIO, "biord", 0);
2594 else
2595 tsleep(bp, PRIBIO, "biowr", 0);
2596#endif
2597 }
2598 splx(s);
2599 if (bp->b_flags & B_EINTR) {
2600 bp->b_flags &= ~B_EINTR;
2601 return (EINTR);
2602 }
2603 if (bp->b_ioflags & BIO_ERROR) {
2604 return (bp->b_error ? bp->b_error : EIO);
2605 } else {
2606 return (0);
2607 }
2608}
2609
2610/*
2611 * biodone:
2612 *
2613 * Finish I/O on a buffer, optionally calling a completion function.
2614 * This is usually called from an interrupt so process blocking is
2615 * not allowed.
2616 *
2617 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2618 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
2619 * assuming B_INVAL is clear.
2620 *
2621 * For the VMIO case, we set B_CACHE if the op was a read and no
2622 * read error occured, or if the op was a write. B_CACHE is never
2623 * set if the buffer is invalid or otherwise uncacheable.
2624 *
2625 * biodone does not mess with B_INVAL, allowing the I/O routine or the
2626 * initiator to leave B_INVAL set to brelse the buffer out of existance
2627 * in the biodone routine.
2628 */
2629void
|
2630biodone(register struct buf * bp)
| 2630biodone(struct bio * bip)
|
2631{
| 2631{
|
| 2632 bufdone((struct buf *)bip);
2633}
2634
2635void
2636bufdone(struct buf *bp)
2637{
|
2632 int s;
2633 void (*biodone) __P((struct buf *));
2634
2635 s = splbio();
2636
2637 KASSERT(BUF_REFCNT(bp) > 0, ("biodone: bp %p not busy %d", bp, BUF_REFCNT(bp)));
2638 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
2639
2640 bp->b_flags |= B_DONE;
2641
2642 if (bp->b_iocmd == BIO_DELETE) {
2643 brelse(bp);
2644 splx(s);
2645 return;
2646 }
2647
2648 if (bp->b_iocmd == BIO_WRITE) {
2649 vwakeup(bp);
2650 }
2651
2652 /* call optional completion function if requested */
2653 if (bp->b_iodone != NULL) {
2654 biodone = bp->b_iodone;
2655 bp->b_iodone = NULL;
2656 (*biodone) (bp);
2657 splx(s);
2658 return;
2659 }
2660 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
2661 (*bioops.io_complete)(bp);
2662
2663 if (bp->b_flags & B_VMIO) {
2664 int i, resid;
2665 vm_ooffset_t foff;
2666 vm_page_t m;
2667 vm_object_t obj;
2668 int iosize;
2669 struct vnode *vp = bp->b_vp;
2670
2671 obj = vp->v_object;
2672
2673#if defined(VFS_BIO_DEBUG)
2674 if (vp->v_usecount == 0) {
2675 panic("biodone: zero vnode ref count");
2676 }
2677
2678 if (vp->v_object == NULL) {
2679 panic("biodone: missing VM object");
2680 }
2681
2682 if ((vp->v_flag & VOBJBUF) == 0) {
2683 panic("biodone: vnode is not setup for merged cache");
2684 }
2685#endif
2686
2687 foff = bp->b_offset;
2688 KASSERT(bp->b_offset != NOOFFSET,
2689 ("biodone: no buffer offset"));
2690
2691 if (!obj) {
2692 panic("biodone: no object");
2693 }
2694#if defined(VFS_BIO_DEBUG)
2695 if (obj->paging_in_progress < bp->b_npages) {
2696 printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n",
2697 obj->paging_in_progress, bp->b_npages);
2698 }
2699#endif
2700
2701 /*
2702 * Set B_CACHE if the op was a normal read and no error
2703 * occured. B_CACHE is set for writes in the b*write()
2704 * routines.
2705 */
2706 iosize = bp->b_bcount - bp->b_resid;
2707 if (bp->b_iocmd == BIO_READ &&
2708 !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
2709 !(bp->b_ioflags & BIO_ERROR)) {
2710 bp->b_flags |= B_CACHE;
2711 }
2712
2713 for (i = 0; i < bp->b_npages; i++) {
2714 int bogusflag = 0;
2715 m = bp->b_pages[i];
2716 if (m == bogus_page) {
2717 bogusflag = 1;
2718 m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2719 if (!m) {
2720#if defined(VFS_BIO_DEBUG)
2721 printf("biodone: page disappeared\n");
2722#endif
2723 vm_object_pip_subtract(obj, 1);
2724 bp->b_flags &= ~B_CACHE;
2725 continue;
2726 }
2727 bp->b_pages[i] = m;
2728 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2729 }
2730#if defined(VFS_BIO_DEBUG)
2731 if (OFF_TO_IDX(foff) != m->pindex) {
2732 printf(
2733"biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2734 (unsigned long)foff, m->pindex);
2735 }
2736#endif
2737 resid = IDX_TO_OFF(m->pindex + 1) - foff;
2738 if (resid > iosize)
2739 resid = iosize;
2740
2741 /*
2742 * In the write case, the valid and clean bits are
2743 * already changed correctly ( see bdwrite() ), so we
2744 * only need to do this here in the read case.
2745 */
2746 if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
2747 vfs_page_set_valid(bp, foff, i, m);
2748 }
2749 vm_page_flag_clear(m, PG_ZERO);
2750
2751 /*
2752 * when debugging new filesystems or buffer I/O methods, this
2753 * is the most common error that pops up. if you see this, you
2754 * have not set the page busy flag correctly!!!
2755 */
2756 if (m->busy == 0) {
2757 printf("biodone: page busy < 0, "
2758 "pindex: %d, foff: 0x(%x,%x), "
2759 "resid: %d, index: %d\n",
2760 (int) m->pindex, (int)(foff >> 32),
2761 (int) foff & 0xffffffff, resid, i);
2762 if (!vn_isdisk(vp, NULL))
2763 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2764 bp->b_vp->v_mount->mnt_stat.f_iosize,
2765 (int) bp->b_lblkno,
2766 bp->b_flags, bp->b_npages);
2767 else
2768 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2769 (int) bp->b_lblkno,
2770 bp->b_flags, bp->b_npages);
2771 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2772 m->valid, m->dirty, m->wire_count);
2773 panic("biodone: page busy < 0\n");
2774 }
2775 vm_page_io_finish(m);
2776 vm_object_pip_subtract(obj, 1);
2777 foff += resid;
2778 iosize -= resid;
2779 }
2780 if (obj)
2781 vm_object_pip_wakeupn(obj, 0);
2782 }
2783 /*
2784 * For asynchronous completions, release the buffer now. The brelse
2785 * will do a wakeup there if necessary - so no need to do a wakeup
2786 * here in the async case. The sync case always needs to do a wakeup.
2787 */
2788
2789 if (bp->b_flags & B_ASYNC) {
2790 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
2791 brelse(bp);
2792 else
2793 bqrelse(bp);
2794 } else {
2795 wakeup(bp);
2796 }
2797 splx(s);
2798}
2799
2800/*
2801 * This routine is called in lieu of iodone in the case of
2802 * incomplete I/O. This keeps the busy status for pages
2803 * consistant.
2804 */
2805void
2806vfs_unbusy_pages(struct buf * bp)
2807{
2808 int i;
2809
2810 if (bp->b_flags & B_VMIO) {
2811 struct vnode *vp = bp->b_vp;
2812 vm_object_t obj = vp->v_object;
2813
2814 for (i = 0; i < bp->b_npages; i++) {
2815 vm_page_t m = bp->b_pages[i];
2816
2817 if (m == bogus_page) {
2818 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
2819 if (!m) {
2820 panic("vfs_unbusy_pages: page missing\n");
2821 }
2822 bp->b_pages[i] = m;
2823 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2824 }
2825 vm_object_pip_subtract(obj, 1);
2826 vm_page_flag_clear(m, PG_ZERO);
2827 vm_page_io_finish(m);
2828 }
2829 vm_object_pip_wakeupn(obj, 0);
2830 }
2831}
2832
2833/*
2834 * vfs_page_set_valid:
2835 *
2836 * Set the valid bits in a page based on the supplied offset. The
2837 * range is restricted to the buffer's size.
2838 *
2839 * This routine is typically called after a read completes.
2840 */
2841static void
2842vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m)
2843{
2844 vm_ooffset_t soff, eoff;
2845
2846 /*
2847 * Start and end offsets in buffer. eoff - soff may not cross a
2848 * page boundry or cross the end of the buffer. The end of the
2849 * buffer, in this case, is our file EOF, not the allocation size
2850 * of the buffer.
2851 */
2852 soff = off;
2853 eoff = (off + PAGE_SIZE) & ~PAGE_MASK;
2854 if (eoff > bp->b_offset + bp->b_bcount)
2855 eoff = bp->b_offset + bp->b_bcount;
2856
2857 /*
2858 * Set valid range. This is typically the entire buffer and thus the
2859 * entire page.
2860 */
2861 if (eoff > soff) {
2862 vm_page_set_validclean(
2863 m,
2864 (vm_offset_t) (soff & PAGE_MASK),
2865 (vm_offset_t) (eoff - soff)
2866 );
2867 }
2868}
2869
2870/*
2871 * This routine is called before a device strategy routine.
2872 * It is used to tell the VM system that paging I/O is in
2873 * progress, and treat the pages associated with the buffer
2874 * almost as being PG_BUSY. Also the object paging_in_progress
2875 * flag is handled to make sure that the object doesn't become
2876 * inconsistant.
2877 *
2878 * Since I/O has not been initiated yet, certain buffer flags
2879 * such as BIO_ERROR or B_INVAL may be in an inconsistant state
2880 * and should be ignored.
2881 */
2882void
2883vfs_busy_pages(struct buf * bp, int clear_modify)
2884{
2885 int i, bogus;
2886
2887 if (bp->b_flags & B_VMIO) {
2888 struct vnode *vp = bp->b_vp;
2889 vm_object_t obj = vp->v_object;
2890 vm_ooffset_t foff;
2891
2892 foff = bp->b_offset;
2893 KASSERT(bp->b_offset != NOOFFSET,
2894 ("vfs_busy_pages: no buffer offset"));
2895 vfs_setdirty(bp);
2896
2897retry:
2898 for (i = 0; i < bp->b_npages; i++) {
2899 vm_page_t m = bp->b_pages[i];
2900 if (vm_page_sleep_busy(m, FALSE, "vbpage"))
2901 goto retry;
2902 }
2903
2904 bogus = 0;
2905 for (i = 0; i < bp->b_npages; i++) {
2906 vm_page_t m = bp->b_pages[i];
2907
2908 vm_page_flag_clear(m, PG_ZERO);
2909 if ((bp->b_flags & B_CLUSTER) == 0) {
2910 vm_object_pip_add(obj, 1);
2911 vm_page_io_start(m);
2912 }
2913
2914 /*
2915 * When readying a buffer for a read ( i.e
2916 * clear_modify == 0 ), it is important to do
2917 * bogus_page replacement for valid pages in
2918 * partially instantiated buffers. Partially
2919 * instantiated buffers can, in turn, occur when
2920 * reconstituting a buffer from its VM backing store
2921 * base. We only have to do this if B_CACHE is
2922 * clear ( which causes the I/O to occur in the
2923 * first place ). The replacement prevents the read
2924 * I/O from overwriting potentially dirty VM-backed
2925 * pages. XXX bogus page replacement is, uh, bogus.
2926 * It may not work properly with small-block devices.
2927 * We need to find a better way.
2928 */
2929
2930 vm_page_protect(m, VM_PROT_NONE);
2931 if (clear_modify)
2932 vfs_page_set_valid(bp, foff, i, m);
2933 else if (m->valid == VM_PAGE_BITS_ALL &&
2934 (bp->b_flags & B_CACHE) == 0) {
2935 bp->b_pages[i] = bogus_page;
2936 bogus++;
2937 }
2938 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2939 }
2940 if (bogus)
2941 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2942 }
2943}
2944
2945/*
2946 * Tell the VM system that the pages associated with this buffer
2947 * are clean. This is used for delayed writes where the data is
2948 * going to go to disk eventually without additional VM intevention.
2949 *
2950 * Note that while we only really need to clean through to b_bcount, we
2951 * just go ahead and clean through to b_bufsize.
2952 */
2953static void
2954vfs_clean_pages(struct buf * bp)
2955{
2956 int i;
2957
2958 if (bp->b_flags & B_VMIO) {
2959 vm_ooffset_t foff;
2960
2961 foff = bp->b_offset;
2962 KASSERT(bp->b_offset != NOOFFSET,
2963 ("vfs_clean_pages: no buffer offset"));
2964 for (i = 0; i < bp->b_npages; i++) {
2965 vm_page_t m = bp->b_pages[i];
2966 vm_ooffset_t noff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2967 vm_ooffset_t eoff = noff;
2968
2969 if (eoff > bp->b_offset + bp->b_bufsize)
2970 eoff = bp->b_offset + bp->b_bufsize;
2971 vfs_page_set_valid(bp, foff, i, m);
2972 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
2973 foff = noff;
2974 }
2975 }
2976}
2977
2978/*
2979 * vfs_bio_set_validclean:
2980 *
2981 * Set the range within the buffer to valid and clean. The range is
2982 * relative to the beginning of the buffer, b_offset. Note that b_offset
2983 * itself may be offset from the beginning of the first page.
2984 */
2985
2986void
2987vfs_bio_set_validclean(struct buf *bp, int base, int size)
2988{
2989 if (bp->b_flags & B_VMIO) {
2990 int i;
2991 int n;
2992
2993 /*
2994 * Fixup base to be relative to beginning of first page.
2995 * Set initial n to be the maximum number of bytes in the
2996 * first page that can be validated.
2997 */
2998
2999 base += (bp->b_offset & PAGE_MASK);
3000 n = PAGE_SIZE - (base & PAGE_MASK);
3001
3002 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
3003 vm_page_t m = bp->b_pages[i];
3004
3005 if (n > size)
3006 n = size;
3007
3008 vm_page_set_validclean(m, base & PAGE_MASK, n);
3009 base += n;
3010 size -= n;
3011 n = PAGE_SIZE;
3012 }
3013 }
3014}
3015
3016/*
3017 * vfs_bio_clrbuf:
3018 *
3019 * clear a buffer. This routine essentially fakes an I/O, so we need
3020 * to clear BIO_ERROR and B_INVAL.
3021 *
3022 * Note that while we only theoretically need to clear through b_bcount,
3023 * we go ahead and clear through b_bufsize.
3024 */
3025
3026void
3027vfs_bio_clrbuf(struct buf *bp) {
3028 int i, mask = 0;
3029 caddr_t sa, ea;
3030 if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) {
3031 bp->b_flags &= ~B_INVAL;
3032 bp->b_ioflags &= ~BIO_ERROR;
3033 if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
3034 (bp->b_offset & PAGE_MASK) == 0) {
3035 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
3036 if (((bp->b_pages[0]->flags & PG_ZERO) == 0) &&
3037 ((bp->b_pages[0]->valid & mask) != mask)) {
3038 bzero(bp->b_data, bp->b_bufsize);
3039 }
3040 bp->b_pages[0]->valid |= mask;
3041 bp->b_resid = 0;
3042 return;
3043 }
3044 ea = sa = bp->b_data;
3045 for(i=0;i<bp->b_npages;i++,sa=ea) {
3046 int j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
3047 ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE);
3048 ea = (caddr_t)(vm_offset_t)ulmin(
3049 (u_long)(vm_offset_t)ea,
3050 (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize);
3051 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
3052 if ((bp->b_pages[i]->valid & mask) == mask)
3053 continue;
3054 if ((bp->b_pages[i]->valid & mask) == 0) {
3055 if ((bp->b_pages[i]->flags & PG_ZERO) == 0) {
3056 bzero(sa, ea - sa);
3057 }
3058 } else {
3059 for (; sa < ea; sa += DEV_BSIZE, j++) {
3060 if (((bp->b_pages[i]->flags & PG_ZERO) == 0) &&
3061 (bp->b_pages[i]->valid & (1<<j)) == 0)
3062 bzero(sa, DEV_BSIZE);
3063 }
3064 }
3065 bp->b_pages[i]->valid |= mask;
3066 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
3067 }
3068 bp->b_resid = 0;
3069 } else {
3070 clrbuf(bp);
3071 }
3072}
3073
3074/*
3075 * vm_hold_load_pages and vm_hold_unload pages get pages into
3076 * a buffers address space. The pages are anonymous and are
3077 * not associated with a file object.
3078 */
3079void
3080vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3081{
3082 vm_offset_t pg;
3083 vm_page_t p;
3084 int index;
3085
3086 to = round_page(to);
3087 from = round_page(from);
3088 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3089
3090 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3091
3092tryagain:
3093
3094 p = vm_page_alloc(kernel_object,
3095 ((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
3096 VM_ALLOC_NORMAL);
3097 if (!p) {
3098 vm_pageout_deficit += (to - from) >> PAGE_SHIFT;
3099 VM_WAIT;
3100 goto tryagain;
3101 }
3102 vm_page_wire(p);
3103 p->valid = VM_PAGE_BITS_ALL;
3104 vm_page_flag_clear(p, PG_ZERO);
3105 pmap_kenter(pg, VM_PAGE_TO_PHYS(p));
3106 bp->b_pages[index] = p;
3107 vm_page_wakeup(p);
3108 }
3109 bp->b_npages = index;
3110}
3111
3112void
3113vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3114{
3115 vm_offset_t pg;
3116 vm_page_t p;
3117 int index, newnpages;
3118
3119 from = round_page(from);
3120 to = round_page(to);
3121 newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3122
3123 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3124 p = bp->b_pages[index];
3125 if (p && (index < bp->b_npages)) {
3126 if (p->busy) {
3127 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3128 bp->b_blkno, bp->b_lblkno);
3129 }
3130 bp->b_pages[index] = NULL;
3131 pmap_kremove(pg);
3132 vm_page_busy(p);
3133 vm_page_unwire(p, 0);
3134 vm_page_free(p);
3135 }
3136 }
3137 bp->b_npages = newnpages;
3138}
3139
3140
3141#include "opt_ddb.h"
3142#ifdef DDB
3143#include <ddb/ddb.h>
3144
3145DB_SHOW_COMMAND(buffer, db_show_buffer)
3146{
3147 /* get args */
3148 struct buf *bp = (struct buf *)addr;
3149
3150 if (!have_addr) {
3151 db_printf("usage: show buffer <addr>\n");
3152 return;
3153 }
3154
3155 db_printf("b_flags = 0x%b\n", (u_int)bp->b_flags, PRINT_BUF_FLAGS);
3156 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3157 "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3158 "b_blkno = %d, b_pblkno = %d\n",
3159 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
3160 major(bp->b_dev), minor(bp->b_dev),
3161 bp->b_data, bp->b_blkno, bp->b_pblkno);
3162 if (bp->b_npages) {
3163 int i;
3164 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
3165 for (i = 0; i < bp->b_npages; i++) {
3166 vm_page_t m;
3167 m = bp->b_pages[i];
3168 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
3169 (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
3170 if ((i + 1) < bp->b_npages)
3171 db_printf(",");
3172 }
3173 db_printf("\n");
3174 }
3175}
3176#endif /* DDB */
| 2638 int s;
2639 void (*biodone) __P((struct buf *));
2640
2641 s = splbio();
2642
2643 KASSERT(BUF_REFCNT(bp) > 0, ("biodone: bp %p not busy %d", bp, BUF_REFCNT(bp)));
2644 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
2645
2646 bp->b_flags |= B_DONE;
2647
2648 if (bp->b_iocmd == BIO_DELETE) {
2649 brelse(bp);
2650 splx(s);
2651 return;
2652 }
2653
2654 if (bp->b_iocmd == BIO_WRITE) {
2655 vwakeup(bp);
2656 }
2657
2658 /* call optional completion function if requested */
2659 if (bp->b_iodone != NULL) {
2660 biodone = bp->b_iodone;
2661 bp->b_iodone = NULL;
2662 (*biodone) (bp);
2663 splx(s);
2664 return;
2665 }
2666 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
2667 (*bioops.io_complete)(bp);
2668
2669 if (bp->b_flags & B_VMIO) {
2670 int i, resid;
2671 vm_ooffset_t foff;
2672 vm_page_t m;
2673 vm_object_t obj;
2674 int iosize;
2675 struct vnode *vp = bp->b_vp;
2676
2677 obj = vp->v_object;
2678
2679#if defined(VFS_BIO_DEBUG)
2680 if (vp->v_usecount == 0) {
2681 panic("biodone: zero vnode ref count");
2682 }
2683
2684 if (vp->v_object == NULL) {
2685 panic("biodone: missing VM object");
2686 }
2687
2688 if ((vp->v_flag & VOBJBUF) == 0) {
2689 panic("biodone: vnode is not setup for merged cache");
2690 }
2691#endif
2692
2693 foff = bp->b_offset;
2694 KASSERT(bp->b_offset != NOOFFSET,
2695 ("biodone: no buffer offset"));
2696
2697 if (!obj) {
2698 panic("biodone: no object");
2699 }
2700#if defined(VFS_BIO_DEBUG)
2701 if (obj->paging_in_progress < bp->b_npages) {
2702 printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n",
2703 obj->paging_in_progress, bp->b_npages);
2704 }
2705#endif
2706
2707 /*
2708 * Set B_CACHE if the op was a normal read and no error
2709 * occured. B_CACHE is set for writes in the b*write()
2710 * routines.
2711 */
2712 iosize = bp->b_bcount - bp->b_resid;
2713 if (bp->b_iocmd == BIO_READ &&
2714 !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
2715 !(bp->b_ioflags & BIO_ERROR)) {
2716 bp->b_flags |= B_CACHE;
2717 }
2718
2719 for (i = 0; i < bp->b_npages; i++) {
2720 int bogusflag = 0;
2721 m = bp->b_pages[i];
2722 if (m == bogus_page) {
2723 bogusflag = 1;
2724 m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2725 if (!m) {
2726#if defined(VFS_BIO_DEBUG)
2727 printf("biodone: page disappeared\n");
2728#endif
2729 vm_object_pip_subtract(obj, 1);
2730 bp->b_flags &= ~B_CACHE;
2731 continue;
2732 }
2733 bp->b_pages[i] = m;
2734 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2735 }
2736#if defined(VFS_BIO_DEBUG)
2737 if (OFF_TO_IDX(foff) != m->pindex) {
2738 printf(
2739"biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2740 (unsigned long)foff, m->pindex);
2741 }
2742#endif
2743 resid = IDX_TO_OFF(m->pindex + 1) - foff;
2744 if (resid > iosize)
2745 resid = iosize;
2746
2747 /*
2748 * In the write case, the valid and clean bits are
2749 * already changed correctly ( see bdwrite() ), so we
2750 * only need to do this here in the read case.
2751 */
2752 if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
2753 vfs_page_set_valid(bp, foff, i, m);
2754 }
2755 vm_page_flag_clear(m, PG_ZERO);
2756
2757 /*
2758 * when debugging new filesystems or buffer I/O methods, this
2759 * is the most common error that pops up. if you see this, you
2760 * have not set the page busy flag correctly!!!
2761 */
2762 if (m->busy == 0) {
2763 printf("biodone: page busy < 0, "
2764 "pindex: %d, foff: 0x(%x,%x), "
2765 "resid: %d, index: %d\n",
2766 (int) m->pindex, (int)(foff >> 32),
2767 (int) foff & 0xffffffff, resid, i);
2768 if (!vn_isdisk(vp, NULL))
2769 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2770 bp->b_vp->v_mount->mnt_stat.f_iosize,
2771 (int) bp->b_lblkno,
2772 bp->b_flags, bp->b_npages);
2773 else
2774 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2775 (int) bp->b_lblkno,
2776 bp->b_flags, bp->b_npages);
2777 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2778 m->valid, m->dirty, m->wire_count);
2779 panic("biodone: page busy < 0\n");
2780 }
2781 vm_page_io_finish(m);
2782 vm_object_pip_subtract(obj, 1);
2783 foff += resid;
2784 iosize -= resid;
2785 }
2786 if (obj)
2787 vm_object_pip_wakeupn(obj, 0);
2788 }
2789 /*
2790 * For asynchronous completions, release the buffer now. The brelse
2791 * will do a wakeup there if necessary - so no need to do a wakeup
2792 * here in the async case. The sync case always needs to do a wakeup.
2793 */
2794
2795 if (bp->b_flags & B_ASYNC) {
2796 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
2797 brelse(bp);
2798 else
2799 bqrelse(bp);
2800 } else {
2801 wakeup(bp);
2802 }
2803 splx(s);
2804}
2805
2806/*
2807 * This routine is called in lieu of iodone in the case of
2808 * incomplete I/O. This keeps the busy status for pages
2809 * consistant.
2810 */
2811void
2812vfs_unbusy_pages(struct buf * bp)
2813{
2814 int i;
2815
2816 if (bp->b_flags & B_VMIO) {
2817 struct vnode *vp = bp->b_vp;
2818 vm_object_t obj = vp->v_object;
2819
2820 for (i = 0; i < bp->b_npages; i++) {
2821 vm_page_t m = bp->b_pages[i];
2822
2823 if (m == bogus_page) {
2824 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
2825 if (!m) {
2826 panic("vfs_unbusy_pages: page missing\n");
2827 }
2828 bp->b_pages[i] = m;
2829 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2830 }
2831 vm_object_pip_subtract(obj, 1);
2832 vm_page_flag_clear(m, PG_ZERO);
2833 vm_page_io_finish(m);
2834 }
2835 vm_object_pip_wakeupn(obj, 0);
2836 }
2837}
2838
2839/*
2840 * vfs_page_set_valid:
2841 *
2842 * Set the valid bits in a page based on the supplied offset. The
2843 * range is restricted to the buffer's size.
2844 *
2845 * This routine is typically called after a read completes.
2846 */
2847static void
2848vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m)
2849{
2850 vm_ooffset_t soff, eoff;
2851
2852 /*
2853 * Start and end offsets in buffer. eoff - soff may not cross a
2854 * page boundry or cross the end of the buffer. The end of the
2855 * buffer, in this case, is our file EOF, not the allocation size
2856 * of the buffer.
2857 */
2858 soff = off;
2859 eoff = (off + PAGE_SIZE) & ~PAGE_MASK;
2860 if (eoff > bp->b_offset + bp->b_bcount)
2861 eoff = bp->b_offset + bp->b_bcount;
2862
2863 /*
2864 * Set valid range. This is typically the entire buffer and thus the
2865 * entire page.
2866 */
2867 if (eoff > soff) {
2868 vm_page_set_validclean(
2869 m,
2870 (vm_offset_t) (soff & PAGE_MASK),
2871 (vm_offset_t) (eoff - soff)
2872 );
2873 }
2874}
2875
2876/*
2877 * This routine is called before a device strategy routine.
2878 * It is used to tell the VM system that paging I/O is in
2879 * progress, and treat the pages associated with the buffer
2880 * almost as being PG_BUSY. Also the object paging_in_progress
2881 * flag is handled to make sure that the object doesn't become
2882 * inconsistant.
2883 *
2884 * Since I/O has not been initiated yet, certain buffer flags
2885 * such as BIO_ERROR or B_INVAL may be in an inconsistant state
2886 * and should be ignored.
2887 */
2888void
2889vfs_busy_pages(struct buf * bp, int clear_modify)
2890{
2891 int i, bogus;
2892
2893 if (bp->b_flags & B_VMIO) {
2894 struct vnode *vp = bp->b_vp;
2895 vm_object_t obj = vp->v_object;
2896 vm_ooffset_t foff;
2897
2898 foff = bp->b_offset;
2899 KASSERT(bp->b_offset != NOOFFSET,
2900 ("vfs_busy_pages: no buffer offset"));
2901 vfs_setdirty(bp);
2902
2903retry:
2904 for (i = 0; i < bp->b_npages; i++) {
2905 vm_page_t m = bp->b_pages[i];
2906 if (vm_page_sleep_busy(m, FALSE, "vbpage"))
2907 goto retry;
2908 }
2909
2910 bogus = 0;
2911 for (i = 0; i < bp->b_npages; i++) {
2912 vm_page_t m = bp->b_pages[i];
2913
2914 vm_page_flag_clear(m, PG_ZERO);
2915 if ((bp->b_flags & B_CLUSTER) == 0) {
2916 vm_object_pip_add(obj, 1);
2917 vm_page_io_start(m);
2918 }
2919
2920 /*
2921 * When readying a buffer for a read ( i.e
2922 * clear_modify == 0 ), it is important to do
2923 * bogus_page replacement for valid pages in
2924 * partially instantiated buffers. Partially
2925 * instantiated buffers can, in turn, occur when
2926 * reconstituting a buffer from its VM backing store
2927 * base. We only have to do this if B_CACHE is
2928 * clear ( which causes the I/O to occur in the
2929 * first place ). The replacement prevents the read
2930 * I/O from overwriting potentially dirty VM-backed
2931 * pages. XXX bogus page replacement is, uh, bogus.
2932 * It may not work properly with small-block devices.
2933 * We need to find a better way.
2934 */
2935
2936 vm_page_protect(m, VM_PROT_NONE);
2937 if (clear_modify)
2938 vfs_page_set_valid(bp, foff, i, m);
2939 else if (m->valid == VM_PAGE_BITS_ALL &&
2940 (bp->b_flags & B_CACHE) == 0) {
2941 bp->b_pages[i] = bogus_page;
2942 bogus++;
2943 }
2944 foff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2945 }
2946 if (bogus)
2947 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages);
2948 }
2949}
2950
2951/*
2952 * Tell the VM system that the pages associated with this buffer
2953 * are clean. This is used for delayed writes where the data is
2954 * going to go to disk eventually without additional VM intevention.
2955 *
2956 * Note that while we only really need to clean through to b_bcount, we
2957 * just go ahead and clean through to b_bufsize.
2958 */
2959static void
2960vfs_clean_pages(struct buf * bp)
2961{
2962 int i;
2963
2964 if (bp->b_flags & B_VMIO) {
2965 vm_ooffset_t foff;
2966
2967 foff = bp->b_offset;
2968 KASSERT(bp->b_offset != NOOFFSET,
2969 ("vfs_clean_pages: no buffer offset"));
2970 for (i = 0; i < bp->b_npages; i++) {
2971 vm_page_t m = bp->b_pages[i];
2972 vm_ooffset_t noff = (foff + PAGE_SIZE) & ~PAGE_MASK;
2973 vm_ooffset_t eoff = noff;
2974
2975 if (eoff > bp->b_offset + bp->b_bufsize)
2976 eoff = bp->b_offset + bp->b_bufsize;
2977 vfs_page_set_valid(bp, foff, i, m);
2978 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
2979 foff = noff;
2980 }
2981 }
2982}
2983
2984/*
2985 * vfs_bio_set_validclean:
2986 *
2987 * Set the range within the buffer to valid and clean. The range is
2988 * relative to the beginning of the buffer, b_offset. Note that b_offset
2989 * itself may be offset from the beginning of the first page.
2990 */
2991
2992void
2993vfs_bio_set_validclean(struct buf *bp, int base, int size)
2994{
2995 if (bp->b_flags & B_VMIO) {
2996 int i;
2997 int n;
2998
2999 /*
3000 * Fixup base to be relative to beginning of first page.
3001 * Set initial n to be the maximum number of bytes in the
3002 * first page that can be validated.
3003 */
3004
3005 base += (bp->b_offset & PAGE_MASK);
3006 n = PAGE_SIZE - (base & PAGE_MASK);
3007
3008 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
3009 vm_page_t m = bp->b_pages[i];
3010
3011 if (n > size)
3012 n = size;
3013
3014 vm_page_set_validclean(m, base & PAGE_MASK, n);
3015 base += n;
3016 size -= n;
3017 n = PAGE_SIZE;
3018 }
3019 }
3020}
3021
3022/*
3023 * vfs_bio_clrbuf:
3024 *
3025 * clear a buffer. This routine essentially fakes an I/O, so we need
3026 * to clear BIO_ERROR and B_INVAL.
3027 *
3028 * Note that while we only theoretically need to clear through b_bcount,
3029 * we go ahead and clear through b_bufsize.
3030 */
3031
3032void
3033vfs_bio_clrbuf(struct buf *bp) {
3034 int i, mask = 0;
3035 caddr_t sa, ea;
3036 if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) {
3037 bp->b_flags &= ~B_INVAL;
3038 bp->b_ioflags &= ~BIO_ERROR;
3039 if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
3040 (bp->b_offset & PAGE_MASK) == 0) {
3041 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
3042 if (((bp->b_pages[0]->flags & PG_ZERO) == 0) &&
3043 ((bp->b_pages[0]->valid & mask) != mask)) {
3044 bzero(bp->b_data, bp->b_bufsize);
3045 }
3046 bp->b_pages[0]->valid |= mask;
3047 bp->b_resid = 0;
3048 return;
3049 }
3050 ea = sa = bp->b_data;
3051 for(i=0;i<bp->b_npages;i++,sa=ea) {
3052 int j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
3053 ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE);
3054 ea = (caddr_t)(vm_offset_t)ulmin(
3055 (u_long)(vm_offset_t)ea,
3056 (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize);
3057 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
3058 if ((bp->b_pages[i]->valid & mask) == mask)
3059 continue;
3060 if ((bp->b_pages[i]->valid & mask) == 0) {
3061 if ((bp->b_pages[i]->flags & PG_ZERO) == 0) {
3062 bzero(sa, ea - sa);
3063 }
3064 } else {
3065 for (; sa < ea; sa += DEV_BSIZE, j++) {
3066 if (((bp->b_pages[i]->flags & PG_ZERO) == 0) &&
3067 (bp->b_pages[i]->valid & (1<<j)) == 0)
3068 bzero(sa, DEV_BSIZE);
3069 }
3070 }
3071 bp->b_pages[i]->valid |= mask;
3072 vm_page_flag_clear(bp->b_pages[i], PG_ZERO);
3073 }
3074 bp->b_resid = 0;
3075 } else {
3076 clrbuf(bp);
3077 }
3078}
3079
3080/*
3081 * vm_hold_load_pages and vm_hold_unload pages get pages into
3082 * a buffers address space. The pages are anonymous and are
3083 * not associated with a file object.
3084 */
3085void
3086vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3087{
3088 vm_offset_t pg;
3089 vm_page_t p;
3090 int index;
3091
3092 to = round_page(to);
3093 from = round_page(from);
3094 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3095
3096 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3097
3098tryagain:
3099
3100 p = vm_page_alloc(kernel_object,
3101 ((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT),
3102 VM_ALLOC_NORMAL);
3103 if (!p) {
3104 vm_pageout_deficit += (to - from) >> PAGE_SHIFT;
3105 VM_WAIT;
3106 goto tryagain;
3107 }
3108 vm_page_wire(p);
3109 p->valid = VM_PAGE_BITS_ALL;
3110 vm_page_flag_clear(p, PG_ZERO);
3111 pmap_kenter(pg, VM_PAGE_TO_PHYS(p));
3112 bp->b_pages[index] = p;
3113 vm_page_wakeup(p);
3114 }
3115 bp->b_npages = index;
3116}
3117
3118void
3119vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to)
3120{
3121 vm_offset_t pg;
3122 vm_page_t p;
3123 int index, newnpages;
3124
3125 from = round_page(from);
3126 to = round_page(to);
3127 newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
3128
3129 for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
3130 p = bp->b_pages[index];
3131 if (p && (index < bp->b_npages)) {
3132 if (p->busy) {
3133 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3134 bp->b_blkno, bp->b_lblkno);
3135 }
3136 bp->b_pages[index] = NULL;
3137 pmap_kremove(pg);
3138 vm_page_busy(p);
3139 vm_page_unwire(p, 0);
3140 vm_page_free(p);
3141 }
3142 }
3143 bp->b_npages = newnpages;
3144}
3145
3146
3147#include "opt_ddb.h"
3148#ifdef DDB
3149#include <ddb/ddb.h>
3150
3151DB_SHOW_COMMAND(buffer, db_show_buffer)
3152{
3153 /* get args */
3154 struct buf *bp = (struct buf *)addr;
3155
3156 if (!have_addr) {
3157 db_printf("usage: show buffer <addr>\n");
3158 return;
3159 }
3160
3161 db_printf("b_flags = 0x%b\n", (u_int)bp->b_flags, PRINT_BUF_FLAGS);
3162 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3163 "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3164 "b_blkno = %d, b_pblkno = %d\n",
3165 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
3166 major(bp->b_dev), minor(bp->b_dev),
3167 bp->b_data, bp->b_blkno, bp->b_pblkno);
3168 if (bp->b_npages) {
3169 int i;
3170 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
3171 for (i = 0; i < bp->b_npages; i++) {
3172 vm_page_t m;
3173 m = bp->b_pages[i];
3174 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
3175 (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
3176 if ((i + 1) < bp->b_npages)
3177 db_printf(",");
3178 }
3179 db_printf("\n");
3180 }
3181}
3182#endif /* DDB */
|