37 */
38
39#include "opt_debug_cluster.h"
40
41#include <sys/param.h>
42#include <sys/systm.h>
43#include <sys/kernel.h>
44#include <sys/proc.h>
45#include <sys/buf.h>
46#include <sys/vnode.h>
47#include <sys/malloc.h>
48#include <sys/mount.h>
49#include <sys/resourcevar.h>
50#include <vm/vm.h>
51#include <vm/vm_object.h>
52#include <vm/vm_page.h>
53#include <sys/sysctl.h>
54
55#if defined(CLUSTERDEBUG)
56#include <sys/sysctl.h>
57static int rcluster= 0;
58SYSCTL_INT(_debug, OID_AUTO, rcluster, CTLFLAG_RW, &rcluster, 0, "");
59#endif
60
61static MALLOC_DEFINE(M_SEGMENT, "cluster_save buffer", "cluster_save buffer");
62
63static struct cluster_save *
64 cluster_collectbufs __P((struct vnode *vp, struct buf *last_bp));
65static struct buf *
66 cluster_rbuild __P((struct vnode *vp, u_quad_t filesize, daddr_t lbn,
67 daddr_t blkno, long size, int run, struct buf *fbp));
68
69static int write_behind = 1;
70SYSCTL_INT(_vfs, OID_AUTO, write_behind, CTLFLAG_RW, &write_behind, 0, "");
71
72extern vm_page_t bogus_page;
73
74extern int cluster_pbuf_freecnt;
75
76/*
77 * Maximum number of blocks for read-ahead.
78 */
79#define MAXRA 32
80
81/*
82 * This replaces bread.
83 */
84int
85cluster_read(vp, filesize, lblkno, size, cred, totread, seqcount, bpp)
86 struct vnode *vp;
87 u_quad_t filesize;
88 daddr_t lblkno;
89 long size;
90 struct ucred *cred;
91 long totread;
92 int seqcount;
93 struct buf **bpp;
94{
95 struct buf *bp, *rbp, *reqbp;
96 daddr_t blkno, origblkno;
97 int error, num_ra;
98 int i;
99 int maxra, racluster;
100 long origtotread;
101
102 error = 0;
103
104 /*
105 * Try to limit the amount of read-ahead by a few
106 * ad-hoc parameters. This needs work!!!
107 */
108 racluster = vp->v_mount->mnt_iosize_max / size;
109 maxra = 2 * racluster + (totread / size);
110 if (maxra > MAXRA)
111 maxra = MAXRA;
112 if (maxra > nbuf/8)
113 maxra = nbuf/8;
114
115 /*
116 * get the requested block
117 */
118 *bpp = reqbp = bp = getblk(vp, lblkno, size, 0, 0);
119 origblkno = lblkno;
120 origtotread = totread;
121
122 /*
123 * if it is in the cache, then check to see if the reads have been
124 * sequential. If they have, then try some read-ahead, otherwise
125 * back-off on prospective read-aheads.
126 */
127 if (bp->b_flags & B_CACHE) {
128 if (!seqcount) {
129 return 0;
130 } else if ((bp->b_flags & B_RAM) == 0) {
131 return 0;
132 } else {
133 int s;
134 struct buf *tbp;
135 bp->b_flags &= ~B_RAM;
136 /*
137 * We do the spl here so that there is no window
138 * between the incore and the b_usecount increment
139 * below. We opt to keep the spl out of the loop
140 * for efficiency.
141 */
142 s = splbio();
143 for (i = 1; i < maxra; i++) {
144
145 if (!(tbp = incore(vp, lblkno+i))) {
146 break;
147 }
148
149 /*
150 * Set another read-ahead mark so we know
151 * to check again.
152 */
153 if (((i % racluster) == (racluster - 1)) ||
154 (i == (maxra - 1)))
155 tbp->b_flags |= B_RAM;
156 }
157 splx(s);
158 if (i >= maxra) {
159 return 0;
160 }
161 lblkno += i;
162 }
163 reqbp = bp = NULL;
164 } else {
165 off_t firstread = bp->b_offset;
166
167 KASSERT(bp->b_offset != NOOFFSET,
168 ("cluster_read: no buffer offset"));
169 if (firstread + totread > filesize)
170 totread = filesize - firstread;
171 if (totread > size) {
172 int nblks = 0;
173 int ncontigafter;
174 while (totread > 0) {
175 nblks++;
176 totread -= size;
177 }
178 if (nblks == 1)
179 goto single_block_read;
180 if (nblks > racluster)
181 nblks = racluster;
182
183 error = VOP_BMAP(vp, lblkno, NULL,
184 &blkno, &ncontigafter, NULL);
185 if (error)
186 goto single_block_read;
187 if (blkno == -1)
188 goto single_block_read;
189 if (ncontigafter == 0)
190 goto single_block_read;
191 if (ncontigafter + 1 < nblks)
192 nblks = ncontigafter + 1;
193
194 bp = cluster_rbuild(vp, filesize, lblkno,
195 blkno, size, nblks, bp);
196 lblkno += (bp->b_bufsize / size);
197 } else {
198single_block_read:
199 /*
200 * if it isn't in the cache, then get a chunk from
201 * disk if sequential, otherwise just get the block.
202 */
203 bp->b_flags |= B_RAM;
204 bp->b_iocmd = BIO_READ;
205 lblkno += 1;
206 }
207 }
208
209 /*
210 * if we have been doing sequential I/O, then do some read-ahead
211 */
212 rbp = NULL;
213 if (seqcount && (lblkno < (origblkno + seqcount))) {
214 /*
215 * we now build the read-ahead buffer if it is desirable.
216 */
217 if (((u_quad_t)(lblkno + 1) * size) <= filesize &&
218 !(error = VOP_BMAP(vp, lblkno, NULL, &blkno, &num_ra, NULL)) &&
219 blkno != -1) {
220 int nblksread;
221 int ntoread = num_ra + 1;
222 nblksread = (origtotread + size - 1) / size;
223 if (seqcount < nblksread)
224 seqcount = nblksread;
225 if (seqcount < ntoread)
226 ntoread = seqcount;
227 if (num_ra) {
228 rbp = cluster_rbuild(vp, filesize, lblkno,
229 blkno, size, ntoread, NULL);
230 } else {
231 rbp = getblk(vp, lblkno, size, 0, 0);
232 rbp->b_flags |= B_ASYNC | B_RAM;
233 rbp->b_iocmd = BIO_READ;
234 rbp->b_blkno = blkno;
235 }
236 }
237 }
238
239 /*
240 * handle the synchronous read
241 */
242 if (bp) {
243#if defined(CLUSTERDEBUG)
244 if (rcluster)
245 printf("S(%ld,%ld,%d) ",
246 (long)bp->b_lblkno, bp->b_bcount, seqcount);
247#endif
248 if ((bp->b_flags & B_CLUSTER) == 0)
249 vfs_busy_pages(bp, 0);
250 bp->b_flags &= ~B_INVAL;
251 bp->b_ioflags &= ~BIO_ERROR;
252 if ((bp->b_flags & B_ASYNC) || bp->b_iodone != NULL)
253 BUF_KERNPROC(bp);
254 error = VOP_STRATEGY(vp, bp);
255 curproc->p_stats->p_ru.ru_inblock++;
256 }
257
258 /*
259 * and if we have read-aheads, do them too
260 */
261 if (rbp) {
262 if (error) {
263 rbp->b_flags &= ~B_ASYNC;
264 brelse(rbp);
265 } else if (rbp->b_flags & B_CACHE) {
266 rbp->b_flags &= ~B_ASYNC;
267 bqrelse(rbp);
268 } else {
269#if defined(CLUSTERDEBUG)
270 if (rcluster) {
271 if (bp)
272 printf("A+(%ld,%ld,%ld,%d) ",
273 (long)rbp->b_lblkno, rbp->b_bcount,
274 (long)(rbp->b_lblkno - origblkno),
275 seqcount);
276 else
277 printf("A(%ld,%ld,%ld,%d) ",
278 (long)rbp->b_lblkno, rbp->b_bcount,
279 (long)(rbp->b_lblkno - origblkno),
280 seqcount);
281 }
282#endif
283
284 if ((rbp->b_flags & B_CLUSTER) == 0)
285 vfs_busy_pages(rbp, 0);
286 rbp->b_flags &= ~B_INVAL;
287 rbp->b_ioflags &= ~BIO_ERROR;
288 if ((rbp->b_flags & B_ASYNC) || rbp->b_iodone != NULL)
289 BUF_KERNPROC(rbp);
290 (void) VOP_STRATEGY(vp, rbp);
291 curproc->p_stats->p_ru.ru_inblock++;
292 }
293 }
294 if (reqbp)
295 return (biowait(reqbp));
296 else
297 return (error);
298}
299
300/*
301 * If blocks are contiguous on disk, use this to provide clustered
302 * read ahead. We will read as many blocks as possible sequentially
303 * and then parcel them up into logical blocks in the buffer hash table.
304 */
305static struct buf *
306cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
307 struct vnode *vp;
308 u_quad_t filesize;
309 daddr_t lbn;
310 daddr_t blkno;
311 long size;
312 int run;
313 struct buf *fbp;
314{
315 struct buf *bp, *tbp;
316 daddr_t bn;
317 int i, inc, j;
318
319 KASSERT(size == vp->v_mount->mnt_stat.f_iosize,
320 ("cluster_rbuild: size %ld != filesize %ld\n",
321 size, vp->v_mount->mnt_stat.f_iosize));
322
323 /*
324 * avoid a division
325 */
326 while ((u_quad_t) size * (lbn + run) > filesize) {
327 --run;
328 }
329
330 if (fbp) {
331 tbp = fbp;
332 tbp->b_iocmd = BIO_READ;
333 } else {
334 tbp = getblk(vp, lbn, size, 0, 0);
335 if (tbp->b_flags & B_CACHE)
336 return tbp;
337 tbp->b_flags |= B_ASYNC | B_RAM;
338 tbp->b_iocmd = BIO_READ;
339 }
340
341 tbp->b_blkno = blkno;
342 if( (tbp->b_flags & B_MALLOC) ||
343 ((tbp->b_flags & B_VMIO) == 0) || (run <= 1) )
344 return tbp;
345
346 bp = trypbuf(&cluster_pbuf_freecnt);
347 if (bp == 0)
348 return tbp;
349
350 bp->b_data = (char *)((vm_offset_t)bp->b_data |
351 ((vm_offset_t)tbp->b_data & PAGE_MASK));
352 bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
353 bp->b_iocmd = BIO_READ;
354 bp->b_iodone = cluster_callback;
355 bp->b_blkno = blkno;
356 bp->b_lblkno = lbn;
357 bp->b_offset = tbp->b_offset;
358 KASSERT(bp->b_offset != NOOFFSET, ("cluster_rbuild: no buffer offset"));
359 pbgetvp(vp, bp);
360
361 TAILQ_INIT(&bp->b_cluster.cluster_head);
362
363 bp->b_bcount = 0;
364 bp->b_bufsize = 0;
365 bp->b_npages = 0;
366
367 inc = btodb(size);
368 for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
369 if (i != 0) {
370 if ((bp->b_npages * PAGE_SIZE) +
371 round_page(size) > vp->v_mount->mnt_iosize_max)
372 break;
373
374 if ((tbp = incore(vp, lbn + i)) != NULL) {
375 if (BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT))
376 break;
377 BUF_UNLOCK(tbp);
378
379 for (j = 0; j < tbp->b_npages; j++)
380 if (tbp->b_pages[j]->valid)
381 break;
382
383 if (j != tbp->b_npages)
384 break;
385
386 if (tbp->b_bcount != size)
387 break;
388 }
389
390 tbp = getblk(vp, lbn + i, size, 0, 0);
391
392 if ((tbp->b_flags & B_CACHE) ||
393 (tbp->b_flags & B_VMIO) == 0) {
394 bqrelse(tbp);
395 break;
396 }
397
398 for (j = 0;j < tbp->b_npages; j++)
399 if (tbp->b_pages[j]->valid)
400 break;
401
402 if (j != tbp->b_npages) {
403 bqrelse(tbp);
404 break;
405 }
406
407 if ((fbp && (i == 1)) || (i == (run - 1)))
408 tbp->b_flags |= B_RAM;
409 tbp->b_flags |= B_ASYNC;
410 tbp->b_iocmd = BIO_READ;
411 if (tbp->b_blkno == tbp->b_lblkno) {
412 tbp->b_blkno = bn;
413 } else if (tbp->b_blkno != bn) {
414 brelse(tbp);
415 break;
416 }
417 }
418 /*
419 * XXX fbp from caller may not be B_ASYNC, but we are going
420 * to biodone() it in cluster_callback() anyway
421 */
422 BUF_KERNPROC(tbp);
423 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
424 tbp, b_cluster.cluster_entry);
425 for (j = 0; j < tbp->b_npages; j += 1) {
426 vm_page_t m;
427 m = tbp->b_pages[j];
428 vm_page_io_start(m);
429 vm_object_pip_add(m->object, 1);
430 if ((bp->b_npages == 0) ||
431 (bp->b_pages[bp->b_npages-1] != m)) {
432 bp->b_pages[bp->b_npages] = m;
433 bp->b_npages++;
434 }
435 if ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL)
436 tbp->b_pages[j] = bogus_page;
437 }
438 bp->b_bcount += tbp->b_bcount;
439 bp->b_bufsize += tbp->b_bufsize;
440 }
441
442 for(j=0;j<bp->b_npages;j++) {
443 if ((bp->b_pages[j]->valid & VM_PAGE_BITS_ALL) ==
444 VM_PAGE_BITS_ALL)
445 bp->b_pages[j] = bogus_page;
446 }
447 if (bp->b_bufsize > bp->b_kvasize)
448 panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
449 bp->b_bufsize, bp->b_kvasize);
450 bp->b_kvasize = bp->b_bufsize;
451
452 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
453 (vm_page_t *)bp->b_pages, bp->b_npages);
454 return (bp);
455}
456
457/*
458 * Cleanup after a clustered read or write.
459 * This is complicated by the fact that any of the buffers might have
460 * extra memory (if there were no empty buffer headers at allocbuf time)
461 * that we will need to shift around.
462 */
463void
464cluster_callback(bp)
465 struct buf *bp;
466{
467 struct buf *nbp, *tbp;
468 int error = 0;
469
470 /*
471 * Must propogate errors to all the components.
472 */
473 if (bp->b_ioflags & BIO_ERROR)
474 error = bp->b_error;
475
476 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
477 /*
478 * Move memory from the large cluster buffer into the component
479 * buffers and mark IO as done on these.
480 */
481 for (tbp = TAILQ_FIRST(&bp->b_cluster.cluster_head);
482 tbp; tbp = nbp) {
483 nbp = TAILQ_NEXT(&tbp->b_cluster, cluster_entry);
484 if (error) {
485 tbp->b_ioflags |= BIO_ERROR;
486 tbp->b_error = error;
487 } else {
488 tbp->b_dirtyoff = tbp->b_dirtyend = 0;
489 tbp->b_flags &= ~B_INVAL;
490 tbp->b_ioflags &= ~BIO_ERROR;
491 }
| 37 */
38
39#include "opt_debug_cluster.h"
40
41#include <sys/param.h>
42#include <sys/systm.h>
43#include <sys/kernel.h>
44#include <sys/proc.h>
45#include <sys/buf.h>
46#include <sys/vnode.h>
47#include <sys/malloc.h>
48#include <sys/mount.h>
49#include <sys/resourcevar.h>
50#include <vm/vm.h>
51#include <vm/vm_object.h>
52#include <vm/vm_page.h>
53#include <sys/sysctl.h>
54
55#if defined(CLUSTERDEBUG)
56#include <sys/sysctl.h>
57static int rcluster= 0;
58SYSCTL_INT(_debug, OID_AUTO, rcluster, CTLFLAG_RW, &rcluster, 0, "");
59#endif
60
61static MALLOC_DEFINE(M_SEGMENT, "cluster_save buffer", "cluster_save buffer");
62
63static struct cluster_save *
64 cluster_collectbufs __P((struct vnode *vp, struct buf *last_bp));
65static struct buf *
66 cluster_rbuild __P((struct vnode *vp, u_quad_t filesize, daddr_t lbn,
67 daddr_t blkno, long size, int run, struct buf *fbp));
68
69static int write_behind = 1;
70SYSCTL_INT(_vfs, OID_AUTO, write_behind, CTLFLAG_RW, &write_behind, 0, "");
71
72extern vm_page_t bogus_page;
73
74extern int cluster_pbuf_freecnt;
75
76/*
77 * Maximum number of blocks for read-ahead.
78 */
79#define MAXRA 32
80
81/*
82 * This replaces bread.
83 */
84int
85cluster_read(vp, filesize, lblkno, size, cred, totread, seqcount, bpp)
86 struct vnode *vp;
87 u_quad_t filesize;
88 daddr_t lblkno;
89 long size;
90 struct ucred *cred;
91 long totread;
92 int seqcount;
93 struct buf **bpp;
94{
95 struct buf *bp, *rbp, *reqbp;
96 daddr_t blkno, origblkno;
97 int error, num_ra;
98 int i;
99 int maxra, racluster;
100 long origtotread;
101
102 error = 0;
103
104 /*
105 * Try to limit the amount of read-ahead by a few
106 * ad-hoc parameters. This needs work!!!
107 */
108 racluster = vp->v_mount->mnt_iosize_max / size;
109 maxra = 2 * racluster + (totread / size);
110 if (maxra > MAXRA)
111 maxra = MAXRA;
112 if (maxra > nbuf/8)
113 maxra = nbuf/8;
114
115 /*
116 * get the requested block
117 */
118 *bpp = reqbp = bp = getblk(vp, lblkno, size, 0, 0);
119 origblkno = lblkno;
120 origtotread = totread;
121
122 /*
123 * if it is in the cache, then check to see if the reads have been
124 * sequential. If they have, then try some read-ahead, otherwise
125 * back-off on prospective read-aheads.
126 */
127 if (bp->b_flags & B_CACHE) {
128 if (!seqcount) {
129 return 0;
130 } else if ((bp->b_flags & B_RAM) == 0) {
131 return 0;
132 } else {
133 int s;
134 struct buf *tbp;
135 bp->b_flags &= ~B_RAM;
136 /*
137 * We do the spl here so that there is no window
138 * between the incore and the b_usecount increment
139 * below. We opt to keep the spl out of the loop
140 * for efficiency.
141 */
142 s = splbio();
143 for (i = 1; i < maxra; i++) {
144
145 if (!(tbp = incore(vp, lblkno+i))) {
146 break;
147 }
148
149 /*
150 * Set another read-ahead mark so we know
151 * to check again.
152 */
153 if (((i % racluster) == (racluster - 1)) ||
154 (i == (maxra - 1)))
155 tbp->b_flags |= B_RAM;
156 }
157 splx(s);
158 if (i >= maxra) {
159 return 0;
160 }
161 lblkno += i;
162 }
163 reqbp = bp = NULL;
164 } else {
165 off_t firstread = bp->b_offset;
166
167 KASSERT(bp->b_offset != NOOFFSET,
168 ("cluster_read: no buffer offset"));
169 if (firstread + totread > filesize)
170 totread = filesize - firstread;
171 if (totread > size) {
172 int nblks = 0;
173 int ncontigafter;
174 while (totread > 0) {
175 nblks++;
176 totread -= size;
177 }
178 if (nblks == 1)
179 goto single_block_read;
180 if (nblks > racluster)
181 nblks = racluster;
182
183 error = VOP_BMAP(vp, lblkno, NULL,
184 &blkno, &ncontigafter, NULL);
185 if (error)
186 goto single_block_read;
187 if (blkno == -1)
188 goto single_block_read;
189 if (ncontigafter == 0)
190 goto single_block_read;
191 if (ncontigafter + 1 < nblks)
192 nblks = ncontigafter + 1;
193
194 bp = cluster_rbuild(vp, filesize, lblkno,
195 blkno, size, nblks, bp);
196 lblkno += (bp->b_bufsize / size);
197 } else {
198single_block_read:
199 /*
200 * if it isn't in the cache, then get a chunk from
201 * disk if sequential, otherwise just get the block.
202 */
203 bp->b_flags |= B_RAM;
204 bp->b_iocmd = BIO_READ;
205 lblkno += 1;
206 }
207 }
208
209 /*
210 * if we have been doing sequential I/O, then do some read-ahead
211 */
212 rbp = NULL;
213 if (seqcount && (lblkno < (origblkno + seqcount))) {
214 /*
215 * we now build the read-ahead buffer if it is desirable.
216 */
217 if (((u_quad_t)(lblkno + 1) * size) <= filesize &&
218 !(error = VOP_BMAP(vp, lblkno, NULL, &blkno, &num_ra, NULL)) &&
219 blkno != -1) {
220 int nblksread;
221 int ntoread = num_ra + 1;
222 nblksread = (origtotread + size - 1) / size;
223 if (seqcount < nblksread)
224 seqcount = nblksread;
225 if (seqcount < ntoread)
226 ntoread = seqcount;
227 if (num_ra) {
228 rbp = cluster_rbuild(vp, filesize, lblkno,
229 blkno, size, ntoread, NULL);
230 } else {
231 rbp = getblk(vp, lblkno, size, 0, 0);
232 rbp->b_flags |= B_ASYNC | B_RAM;
233 rbp->b_iocmd = BIO_READ;
234 rbp->b_blkno = blkno;
235 }
236 }
237 }
238
239 /*
240 * handle the synchronous read
241 */
242 if (bp) {
243#if defined(CLUSTERDEBUG)
244 if (rcluster)
245 printf("S(%ld,%ld,%d) ",
246 (long)bp->b_lblkno, bp->b_bcount, seqcount);
247#endif
248 if ((bp->b_flags & B_CLUSTER) == 0)
249 vfs_busy_pages(bp, 0);
250 bp->b_flags &= ~B_INVAL;
251 bp->b_ioflags &= ~BIO_ERROR;
252 if ((bp->b_flags & B_ASYNC) || bp->b_iodone != NULL)
253 BUF_KERNPROC(bp);
254 error = VOP_STRATEGY(vp, bp);
255 curproc->p_stats->p_ru.ru_inblock++;
256 }
257
258 /*
259 * and if we have read-aheads, do them too
260 */
261 if (rbp) {
262 if (error) {
263 rbp->b_flags &= ~B_ASYNC;
264 brelse(rbp);
265 } else if (rbp->b_flags & B_CACHE) {
266 rbp->b_flags &= ~B_ASYNC;
267 bqrelse(rbp);
268 } else {
269#if defined(CLUSTERDEBUG)
270 if (rcluster) {
271 if (bp)
272 printf("A+(%ld,%ld,%ld,%d) ",
273 (long)rbp->b_lblkno, rbp->b_bcount,
274 (long)(rbp->b_lblkno - origblkno),
275 seqcount);
276 else
277 printf("A(%ld,%ld,%ld,%d) ",
278 (long)rbp->b_lblkno, rbp->b_bcount,
279 (long)(rbp->b_lblkno - origblkno),
280 seqcount);
281 }
282#endif
283
284 if ((rbp->b_flags & B_CLUSTER) == 0)
285 vfs_busy_pages(rbp, 0);
286 rbp->b_flags &= ~B_INVAL;
287 rbp->b_ioflags &= ~BIO_ERROR;
288 if ((rbp->b_flags & B_ASYNC) || rbp->b_iodone != NULL)
289 BUF_KERNPROC(rbp);
290 (void) VOP_STRATEGY(vp, rbp);
291 curproc->p_stats->p_ru.ru_inblock++;
292 }
293 }
294 if (reqbp)
295 return (biowait(reqbp));
296 else
297 return (error);
298}
299
300/*
301 * If blocks are contiguous on disk, use this to provide clustered
302 * read ahead. We will read as many blocks as possible sequentially
303 * and then parcel them up into logical blocks in the buffer hash table.
304 */
305static struct buf *
306cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
307 struct vnode *vp;
308 u_quad_t filesize;
309 daddr_t lbn;
310 daddr_t blkno;
311 long size;
312 int run;
313 struct buf *fbp;
314{
315 struct buf *bp, *tbp;
316 daddr_t bn;
317 int i, inc, j;
318
319 KASSERT(size == vp->v_mount->mnt_stat.f_iosize,
320 ("cluster_rbuild: size %ld != filesize %ld\n",
321 size, vp->v_mount->mnt_stat.f_iosize));
322
323 /*
324 * avoid a division
325 */
326 while ((u_quad_t) size * (lbn + run) > filesize) {
327 --run;
328 }
329
330 if (fbp) {
331 tbp = fbp;
332 tbp->b_iocmd = BIO_READ;
333 } else {
334 tbp = getblk(vp, lbn, size, 0, 0);
335 if (tbp->b_flags & B_CACHE)
336 return tbp;
337 tbp->b_flags |= B_ASYNC | B_RAM;
338 tbp->b_iocmd = BIO_READ;
339 }
340
341 tbp->b_blkno = blkno;
342 if( (tbp->b_flags & B_MALLOC) ||
343 ((tbp->b_flags & B_VMIO) == 0) || (run <= 1) )
344 return tbp;
345
346 bp = trypbuf(&cluster_pbuf_freecnt);
347 if (bp == 0)
348 return tbp;
349
350 bp->b_data = (char *)((vm_offset_t)bp->b_data |
351 ((vm_offset_t)tbp->b_data & PAGE_MASK));
352 bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
353 bp->b_iocmd = BIO_READ;
354 bp->b_iodone = cluster_callback;
355 bp->b_blkno = blkno;
356 bp->b_lblkno = lbn;
357 bp->b_offset = tbp->b_offset;
358 KASSERT(bp->b_offset != NOOFFSET, ("cluster_rbuild: no buffer offset"));
359 pbgetvp(vp, bp);
360
361 TAILQ_INIT(&bp->b_cluster.cluster_head);
362
363 bp->b_bcount = 0;
364 bp->b_bufsize = 0;
365 bp->b_npages = 0;
366
367 inc = btodb(size);
368 for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
369 if (i != 0) {
370 if ((bp->b_npages * PAGE_SIZE) +
371 round_page(size) > vp->v_mount->mnt_iosize_max)
372 break;
373
374 if ((tbp = incore(vp, lbn + i)) != NULL) {
375 if (BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT))
376 break;
377 BUF_UNLOCK(tbp);
378
379 for (j = 0; j < tbp->b_npages; j++)
380 if (tbp->b_pages[j]->valid)
381 break;
382
383 if (j != tbp->b_npages)
384 break;
385
386 if (tbp->b_bcount != size)
387 break;
388 }
389
390 tbp = getblk(vp, lbn + i, size, 0, 0);
391
392 if ((tbp->b_flags & B_CACHE) ||
393 (tbp->b_flags & B_VMIO) == 0) {
394 bqrelse(tbp);
395 break;
396 }
397
398 for (j = 0;j < tbp->b_npages; j++)
399 if (tbp->b_pages[j]->valid)
400 break;
401
402 if (j != tbp->b_npages) {
403 bqrelse(tbp);
404 break;
405 }
406
407 if ((fbp && (i == 1)) || (i == (run - 1)))
408 tbp->b_flags |= B_RAM;
409 tbp->b_flags |= B_ASYNC;
410 tbp->b_iocmd = BIO_READ;
411 if (tbp->b_blkno == tbp->b_lblkno) {
412 tbp->b_blkno = bn;
413 } else if (tbp->b_blkno != bn) {
414 brelse(tbp);
415 break;
416 }
417 }
418 /*
419 * XXX fbp from caller may not be B_ASYNC, but we are going
420 * to biodone() it in cluster_callback() anyway
421 */
422 BUF_KERNPROC(tbp);
423 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
424 tbp, b_cluster.cluster_entry);
425 for (j = 0; j < tbp->b_npages; j += 1) {
426 vm_page_t m;
427 m = tbp->b_pages[j];
428 vm_page_io_start(m);
429 vm_object_pip_add(m->object, 1);
430 if ((bp->b_npages == 0) ||
431 (bp->b_pages[bp->b_npages-1] != m)) {
432 bp->b_pages[bp->b_npages] = m;
433 bp->b_npages++;
434 }
435 if ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL)
436 tbp->b_pages[j] = bogus_page;
437 }
438 bp->b_bcount += tbp->b_bcount;
439 bp->b_bufsize += tbp->b_bufsize;
440 }
441
442 for(j=0;j<bp->b_npages;j++) {
443 if ((bp->b_pages[j]->valid & VM_PAGE_BITS_ALL) ==
444 VM_PAGE_BITS_ALL)
445 bp->b_pages[j] = bogus_page;
446 }
447 if (bp->b_bufsize > bp->b_kvasize)
448 panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
449 bp->b_bufsize, bp->b_kvasize);
450 bp->b_kvasize = bp->b_bufsize;
451
452 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
453 (vm_page_t *)bp->b_pages, bp->b_npages);
454 return (bp);
455}
456
457/*
458 * Cleanup after a clustered read or write.
459 * This is complicated by the fact that any of the buffers might have
460 * extra memory (if there were no empty buffer headers at allocbuf time)
461 * that we will need to shift around.
462 */
463void
464cluster_callback(bp)
465 struct buf *bp;
466{
467 struct buf *nbp, *tbp;
468 int error = 0;
469
470 /*
471 * Must propogate errors to all the components.
472 */
473 if (bp->b_ioflags & BIO_ERROR)
474 error = bp->b_error;
475
476 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
477 /*
478 * Move memory from the large cluster buffer into the component
479 * buffers and mark IO as done on these.
480 */
481 for (tbp = TAILQ_FIRST(&bp->b_cluster.cluster_head);
482 tbp; tbp = nbp) {
483 nbp = TAILQ_NEXT(&tbp->b_cluster, cluster_entry);
484 if (error) {
485 tbp->b_ioflags |= BIO_ERROR;
486 tbp->b_error = error;
487 } else {
488 tbp->b_dirtyoff = tbp->b_dirtyend = 0;
489 tbp->b_flags &= ~B_INVAL;
490 tbp->b_ioflags &= ~BIO_ERROR;
491 }
|
493 }
494 relpbuf(bp, &cluster_pbuf_freecnt);
495}
496
497/*
498 * cluster_wbuild_wb:
499 *
500 * Implement modified write build for cluster.
501 *
502 * write_behind = 0 write behind disabled
503 * write_behind = 1 write behind normal (default)
504 * write_behind = 2 write behind backed-off
505 */
506
507static __inline int
508cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len)
509{
510 int r = 0;
511
512 switch(write_behind) {
513 case 2:
514 if (start_lbn < len)
515 break;
516 start_lbn -= len;
517 /* fall through */
518 case 1:
519 r = cluster_wbuild(vp, size, start_lbn, len);
520 /* fall through */
521 default:
522 /* fall through */
523 break;
524 }
525 return(r);
526}
527
528/*
529 * Do clustered write for FFS.
530 *
531 * Three cases:
532 * 1. Write is not sequential (write asynchronously)
533 * Write is sequential:
534 * 2. beginning of cluster - begin cluster
535 * 3. middle of a cluster - add to cluster
536 * 4. end of a cluster - asynchronously write cluster
537 */
538void
539cluster_write(bp, filesize, seqcount)
540 struct buf *bp;
541 u_quad_t filesize;
542 int seqcount;
543{
544 struct vnode *vp;
545 daddr_t lbn;
546 int maxclen, cursize;
547 int lblocksize;
548 int async;
549
550 vp = bp->b_vp;
551 if (vp->v_type == VREG) {
552 async = vp->v_mount->mnt_flag & MNT_ASYNC;
553 lblocksize = vp->v_mount->mnt_stat.f_iosize;
554 } else {
555 async = 0;
556 lblocksize = bp->b_bufsize;
557 }
558 lbn = bp->b_lblkno;
559 KASSERT(bp->b_offset != NOOFFSET, ("cluster_write: no buffer offset"));
560
561 /* Initialize vnode to beginning of file. */
562 if (lbn == 0)
563 vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
564
565 if (vp->v_clen == 0 || lbn != vp->v_lastw + 1 ||
566 (bp->b_blkno != vp->v_lasta + btodb(lblocksize))) {
567 maxclen = vp->v_mount->mnt_iosize_max / lblocksize - 1;
568 if (vp->v_clen != 0) {
569 /*
570 * Next block is not sequential.
571 *
572 * If we are not writing at end of file, the process
573 * seeked to another point in the file since its last
574 * write, or we have reached our maximum cluster size,
575 * then push the previous cluster. Otherwise try
576 * reallocating to make it sequential.
577 *
578 * Change to algorithm: only push previous cluster if
579 * it was sequential from the point of view of the
580 * seqcount heuristic, otherwise leave the buffer
581 * intact so we can potentially optimize the I/O
582 * later on in the buf_daemon or update daemon
583 * flush.
584 */
585 cursize = vp->v_lastw - vp->v_cstart + 1;
586 if (((u_quad_t) bp->b_offset + lblocksize) != filesize ||
587 lbn != vp->v_lastw + 1 || vp->v_clen <= cursize) {
588 if (!async && seqcount > 0) {
589 cluster_wbuild_wb(vp, lblocksize,
590 vp->v_cstart, cursize);
591 }
592 } else {
593 struct buf **bpp, **endbp;
594 struct cluster_save *buflist;
595
596 buflist = cluster_collectbufs(vp, bp);
597 endbp = &buflist->bs_children
598 [buflist->bs_nchildren - 1];
599 if (VOP_REALLOCBLKS(vp, buflist)) {
600 /*
601 * Failed, push the previous cluster
602 * if *really* writing sequentially
603 * in the logical file (seqcount > 1),
604 * otherwise delay it in the hopes that
605 * the low level disk driver can
606 * optimize the write ordering.
607 */
608 for (bpp = buflist->bs_children;
609 bpp < endbp; bpp++)
610 brelse(*bpp);
611 free(buflist, M_SEGMENT);
612 if (seqcount > 1) {
613 cluster_wbuild_wb(vp,
614 lblocksize, vp->v_cstart,
615 cursize);
616 }
617 } else {
618 /*
619 * Succeeded, keep building cluster.
620 */
621 for (bpp = buflist->bs_children;
622 bpp <= endbp; bpp++)
623 bdwrite(*bpp);
624 free(buflist, M_SEGMENT);
625 vp->v_lastw = lbn;
626 vp->v_lasta = bp->b_blkno;
627 return;
628 }
629 }
630 }
631 /*
632 * Consider beginning a cluster. If at end of file, make
633 * cluster as large as possible, otherwise find size of
634 * existing cluster.
635 */
636 if ((vp->v_type == VREG) &&
637 ((u_quad_t) bp->b_offset + lblocksize) != filesize &&
638 (bp->b_blkno == bp->b_lblkno) &&
639 (VOP_BMAP(vp, lbn, NULL, &bp->b_blkno, &maxclen, NULL) ||
640 bp->b_blkno == -1)) {
641 bawrite(bp);
642 vp->v_clen = 0;
643 vp->v_lasta = bp->b_blkno;
644 vp->v_cstart = lbn + 1;
645 vp->v_lastw = lbn;
646 return;
647 }
648 vp->v_clen = maxclen;
649 if (!async && maxclen == 0) { /* I/O not contiguous */
650 vp->v_cstart = lbn + 1;
651 bawrite(bp);
652 } else { /* Wait for rest of cluster */
653 vp->v_cstart = lbn;
654 bdwrite(bp);
655 }
656 } else if (lbn == vp->v_cstart + vp->v_clen) {
657 /*
658 * At end of cluster, write it out if seqcount tells us we
659 * are operating sequentially, otherwise let the buf or
660 * update daemon handle it.
661 */
662 bdwrite(bp);
663 if (seqcount > 1)
664 cluster_wbuild_wb(vp, lblocksize, vp->v_cstart, vp->v_clen + 1);
665 vp->v_clen = 0;
666 vp->v_cstart = lbn + 1;
667 } else {
668 /*
669 * In the middle of a cluster, so just delay the I/O for now.
670 */
671 bdwrite(bp);
672 }
673 vp->v_lastw = lbn;
674 vp->v_lasta = bp->b_blkno;
675}
676
677
678/*
679 * This is an awful lot like cluster_rbuild...wish they could be combined.
680 * The last lbn argument is the current block on which I/O is being
681 * performed. Check to see that it doesn't fall in the middle of
682 * the current block (if last_bp == NULL).
683 */
684int
685cluster_wbuild(vp, size, start_lbn, len)
686 struct vnode *vp;
687 long size;
688 daddr_t start_lbn;
689 int len;
690{
691 struct buf *bp, *tbp;
692 int i, j, s;
693 int totalwritten = 0;
694 int dbsize = btodb(size);
695
696 while (len > 0) {
697 s = splbio();
698 if (((tbp = gbincore(vp, start_lbn)) == NULL) ||
699 ((tbp->b_flags & (B_INVAL | B_DELWRI)) != B_DELWRI) ||
700 BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT)) {
701 ++start_lbn;
702 --len;
703 splx(s);
704 continue;
705 }
706 bremfree(tbp);
707 tbp->b_flags &= ~B_DONE;
708 splx(s);
709
710 /*
711 * Extra memory in the buffer, punt on this buffer.
712 * XXX we could handle this in most cases, but we would
713 * have to push the extra memory down to after our max
714 * possible cluster size and then potentially pull it back
715 * up if the cluster was terminated prematurely--too much
716 * hassle.
717 */
718 if (((tbp->b_flags & (B_CLUSTEROK|B_MALLOC)) != B_CLUSTEROK) ||
719 (tbp->b_bcount != tbp->b_bufsize) ||
720 (tbp->b_bcount != size) ||
721 (len == 1) ||
722 ((bp = getpbuf(&cluster_pbuf_freecnt)) == NULL)) {
723 totalwritten += tbp->b_bufsize;
724 bawrite(tbp);
725 ++start_lbn;
726 --len;
727 continue;
728 }
729
730 /*
731 * We got a pbuf to make the cluster in.
732 * so initialise it.
733 */
734 TAILQ_INIT(&bp->b_cluster.cluster_head);
735 bp->b_bcount = 0;
736 bp->b_bufsize = 0;
737 bp->b_npages = 0;
738 if (tbp->b_wcred != NOCRED) {
739 bp->b_wcred = tbp->b_wcred;
740 crhold(bp->b_wcred);
741 }
742
743 bp->b_blkno = tbp->b_blkno;
744 bp->b_lblkno = tbp->b_lblkno;
745 bp->b_offset = tbp->b_offset;
746 bp->b_data = (char *)((vm_offset_t)bp->b_data |
747 ((vm_offset_t)tbp->b_data & PAGE_MASK));
748 bp->b_flags |= B_CLUSTER |
749 (tbp->b_flags & (B_VMIO | B_NEEDCOMMIT));
750 bp->b_iodone = cluster_callback;
751 pbgetvp(vp, bp);
752 /*
753 * From this location in the file, scan forward to see
754 * if there are buffers with adjacent data that need to
755 * be written as well.
756 */
757 for (i = 0; i < len; ++i, ++start_lbn) {
758 if (i != 0) { /* If not the first buffer */
759 s = splbio();
760 /*
761 * If the adjacent data is not even in core it
762 * can't need to be written.
763 */
764 if ((tbp = gbincore(vp, start_lbn)) == NULL) {
765 splx(s);
766 break;
767 }
768
769 /*
770 * If it IS in core, but has different
771 * characteristics, don't cluster with it.
772 */
773 if ((tbp->b_flags & (B_VMIO | B_CLUSTEROK |
774 B_INVAL | B_DELWRI | B_NEEDCOMMIT))
775 != (B_DELWRI | B_CLUSTEROK |
776 (bp->b_flags & (B_VMIO | B_NEEDCOMMIT))) ||
777 tbp->b_wcred != bp->b_wcred ||
778 BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT)) {
779 splx(s);
780 break;
781 }
782
783 /*
784 * Check that the combined cluster
785 * would make sense with regard to pages
786 * and would not be too large
787 */
788 if ((tbp->b_bcount != size) ||
789 ((bp->b_blkno + (dbsize * i)) !=
790 tbp->b_blkno) ||
791 ((tbp->b_npages + bp->b_npages) >
792 (vp->v_mount->mnt_iosize_max / PAGE_SIZE))) {
793 BUF_UNLOCK(tbp);
794 splx(s);
795 break;
796 }
797 /*
798 * Ok, it's passed all the tests,
799 * so remove it from the free list
800 * and mark it busy. We will use it.
801 */
802 bremfree(tbp);
803 tbp->b_flags &= ~B_DONE;
804 splx(s);
805 } /* end of code for non-first buffers only */
806 /* check for latent dependencies to be handled */
807 if ((LIST_FIRST(&tbp->b_dep)) != NULL &&
808 bioops.io_start)
809 (*bioops.io_start)(tbp);
810 /*
811 * If the IO is via the VM then we do some
812 * special VM hackery. (yuck)
813 */
814 if (tbp->b_flags & B_VMIO) {
815 vm_page_t m;
816
817 if (i != 0) { /* if not first buffer */
818 for (j = 0; j < tbp->b_npages; j += 1) {
819 m = tbp->b_pages[j];
820 if (m->flags & PG_BUSY) {
821 bqrelse(tbp);
822 goto finishcluster;
823 }
824 }
825 }
826
827 for (j = 0; j < tbp->b_npages; j += 1) {
828 m = tbp->b_pages[j];
829 vm_page_io_start(m);
830 vm_object_pip_add(m->object, 1);
831 if ((bp->b_npages == 0) ||
832 (bp->b_pages[bp->b_npages - 1] != m)) {
833 bp->b_pages[bp->b_npages] = m;
834 bp->b_npages++;
835 }
836 }
837 }
838 bp->b_bcount += size;
839 bp->b_bufsize += size;
840
841 s = splbio();
842 bundirty(tbp);
843 tbp->b_flags &= ~B_DONE;
844 tbp->b_ioflags &= ~BIO_ERROR;
845 tbp->b_flags |= B_ASYNC;
846 tbp->b_iocmd = BIO_WRITE;
847 reassignbuf(tbp, tbp->b_vp); /* put on clean list */
848 ++tbp->b_vp->v_numoutput;
849 splx(s);
850 BUF_KERNPROC(tbp);
851 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
852 tbp, b_cluster.cluster_entry);
853 }
854 finishcluster:
855 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
856 (vm_page_t *) bp->b_pages, bp->b_npages);
857 if (bp->b_bufsize > bp->b_kvasize)
858 panic(
859 "cluster_wbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
860 bp->b_bufsize, bp->b_kvasize);
861 bp->b_kvasize = bp->b_bufsize;
862 totalwritten += bp->b_bufsize;
863 bp->b_dirtyoff = 0;
864 bp->b_dirtyend = bp->b_bufsize;
865 bawrite(bp);
866
867 len -= i;
868 }
869 return totalwritten;
870}
871
872/*
873 * Collect together all the buffers in a cluster.
874 * Plus add one additional buffer.
875 */
876static struct cluster_save *
877cluster_collectbufs(vp, last_bp)
878 struct vnode *vp;
879 struct buf *last_bp;
880{
881 struct cluster_save *buflist;
882 struct buf *bp;
883 daddr_t lbn;
884 int i, len;
885
886 len = vp->v_lastw - vp->v_cstart + 1;
887 buflist = malloc(sizeof(struct buf *) * (len + 1) + sizeof(*buflist),
888 M_SEGMENT, M_WAITOK);
889 buflist->bs_nchildren = 0;
890 buflist->bs_children = (struct buf **) (buflist + 1);
891 for (lbn = vp->v_cstart, i = 0; i < len; lbn++, i++) {
892 (void) bread(vp, lbn, last_bp->b_bcount, NOCRED, &bp);
893 buflist->bs_children[i] = bp;
894 if (bp->b_blkno == bp->b_lblkno)
895 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
896 NULL, NULL);
897 }
898 buflist->bs_children[i] = bp = last_bp;
899 if (bp->b_blkno == bp->b_lblkno)
900 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
901 NULL, NULL);
902 buflist->bs_nchildren = i + 1;
903 return (buflist);
904}
| 493 }
494 relpbuf(bp, &cluster_pbuf_freecnt);
495}
496
497/*
498 * cluster_wbuild_wb:
499 *
500 * Implement modified write build for cluster.
501 *
502 * write_behind = 0 write behind disabled
503 * write_behind = 1 write behind normal (default)
504 * write_behind = 2 write behind backed-off
505 */
506
507static __inline int
508cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len)
509{
510 int r = 0;
511
512 switch(write_behind) {
513 case 2:
514 if (start_lbn < len)
515 break;
516 start_lbn -= len;
517 /* fall through */
518 case 1:
519 r = cluster_wbuild(vp, size, start_lbn, len);
520 /* fall through */
521 default:
522 /* fall through */
523 break;
524 }
525 return(r);
526}
527
528/*
529 * Do clustered write for FFS.
530 *
531 * Three cases:
532 * 1. Write is not sequential (write asynchronously)
533 * Write is sequential:
534 * 2. beginning of cluster - begin cluster
535 * 3. middle of a cluster - add to cluster
536 * 4. end of a cluster - asynchronously write cluster
537 */
538void
539cluster_write(bp, filesize, seqcount)
540 struct buf *bp;
541 u_quad_t filesize;
542 int seqcount;
543{
544 struct vnode *vp;
545 daddr_t lbn;
546 int maxclen, cursize;
547 int lblocksize;
548 int async;
549
550 vp = bp->b_vp;
551 if (vp->v_type == VREG) {
552 async = vp->v_mount->mnt_flag & MNT_ASYNC;
553 lblocksize = vp->v_mount->mnt_stat.f_iosize;
554 } else {
555 async = 0;
556 lblocksize = bp->b_bufsize;
557 }
558 lbn = bp->b_lblkno;
559 KASSERT(bp->b_offset != NOOFFSET, ("cluster_write: no buffer offset"));
560
561 /* Initialize vnode to beginning of file. */
562 if (lbn == 0)
563 vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
564
565 if (vp->v_clen == 0 || lbn != vp->v_lastw + 1 ||
566 (bp->b_blkno != vp->v_lasta + btodb(lblocksize))) {
567 maxclen = vp->v_mount->mnt_iosize_max / lblocksize - 1;
568 if (vp->v_clen != 0) {
569 /*
570 * Next block is not sequential.
571 *
572 * If we are not writing at end of file, the process
573 * seeked to another point in the file since its last
574 * write, or we have reached our maximum cluster size,
575 * then push the previous cluster. Otherwise try
576 * reallocating to make it sequential.
577 *
578 * Change to algorithm: only push previous cluster if
579 * it was sequential from the point of view of the
580 * seqcount heuristic, otherwise leave the buffer
581 * intact so we can potentially optimize the I/O
582 * later on in the buf_daemon or update daemon
583 * flush.
584 */
585 cursize = vp->v_lastw - vp->v_cstart + 1;
586 if (((u_quad_t) bp->b_offset + lblocksize) != filesize ||
587 lbn != vp->v_lastw + 1 || vp->v_clen <= cursize) {
588 if (!async && seqcount > 0) {
589 cluster_wbuild_wb(vp, lblocksize,
590 vp->v_cstart, cursize);
591 }
592 } else {
593 struct buf **bpp, **endbp;
594 struct cluster_save *buflist;
595
596 buflist = cluster_collectbufs(vp, bp);
597 endbp = &buflist->bs_children
598 [buflist->bs_nchildren - 1];
599 if (VOP_REALLOCBLKS(vp, buflist)) {
600 /*
601 * Failed, push the previous cluster
602 * if *really* writing sequentially
603 * in the logical file (seqcount > 1),
604 * otherwise delay it in the hopes that
605 * the low level disk driver can
606 * optimize the write ordering.
607 */
608 for (bpp = buflist->bs_children;
609 bpp < endbp; bpp++)
610 brelse(*bpp);
611 free(buflist, M_SEGMENT);
612 if (seqcount > 1) {
613 cluster_wbuild_wb(vp,
614 lblocksize, vp->v_cstart,
615 cursize);
616 }
617 } else {
618 /*
619 * Succeeded, keep building cluster.
620 */
621 for (bpp = buflist->bs_children;
622 bpp <= endbp; bpp++)
623 bdwrite(*bpp);
624 free(buflist, M_SEGMENT);
625 vp->v_lastw = lbn;
626 vp->v_lasta = bp->b_blkno;
627 return;
628 }
629 }
630 }
631 /*
632 * Consider beginning a cluster. If at end of file, make
633 * cluster as large as possible, otherwise find size of
634 * existing cluster.
635 */
636 if ((vp->v_type == VREG) &&
637 ((u_quad_t) bp->b_offset + lblocksize) != filesize &&
638 (bp->b_blkno == bp->b_lblkno) &&
639 (VOP_BMAP(vp, lbn, NULL, &bp->b_blkno, &maxclen, NULL) ||
640 bp->b_blkno == -1)) {
641 bawrite(bp);
642 vp->v_clen = 0;
643 vp->v_lasta = bp->b_blkno;
644 vp->v_cstart = lbn + 1;
645 vp->v_lastw = lbn;
646 return;
647 }
648 vp->v_clen = maxclen;
649 if (!async && maxclen == 0) { /* I/O not contiguous */
650 vp->v_cstart = lbn + 1;
651 bawrite(bp);
652 } else { /* Wait for rest of cluster */
653 vp->v_cstart = lbn;
654 bdwrite(bp);
655 }
656 } else if (lbn == vp->v_cstart + vp->v_clen) {
657 /*
658 * At end of cluster, write it out if seqcount tells us we
659 * are operating sequentially, otherwise let the buf or
660 * update daemon handle it.
661 */
662 bdwrite(bp);
663 if (seqcount > 1)
664 cluster_wbuild_wb(vp, lblocksize, vp->v_cstart, vp->v_clen + 1);
665 vp->v_clen = 0;
666 vp->v_cstart = lbn + 1;
667 } else {
668 /*
669 * In the middle of a cluster, so just delay the I/O for now.
670 */
671 bdwrite(bp);
672 }
673 vp->v_lastw = lbn;
674 vp->v_lasta = bp->b_blkno;
675}
676
677
678/*
679 * This is an awful lot like cluster_rbuild...wish they could be combined.
680 * The last lbn argument is the current block on which I/O is being
681 * performed. Check to see that it doesn't fall in the middle of
682 * the current block (if last_bp == NULL).
683 */
684int
685cluster_wbuild(vp, size, start_lbn, len)
686 struct vnode *vp;
687 long size;
688 daddr_t start_lbn;
689 int len;
690{
691 struct buf *bp, *tbp;
692 int i, j, s;
693 int totalwritten = 0;
694 int dbsize = btodb(size);
695
696 while (len > 0) {
697 s = splbio();
698 if (((tbp = gbincore(vp, start_lbn)) == NULL) ||
699 ((tbp->b_flags & (B_INVAL | B_DELWRI)) != B_DELWRI) ||
700 BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT)) {
701 ++start_lbn;
702 --len;
703 splx(s);
704 continue;
705 }
706 bremfree(tbp);
707 tbp->b_flags &= ~B_DONE;
708 splx(s);
709
710 /*
711 * Extra memory in the buffer, punt on this buffer.
712 * XXX we could handle this in most cases, but we would
713 * have to push the extra memory down to after our max
714 * possible cluster size and then potentially pull it back
715 * up if the cluster was terminated prematurely--too much
716 * hassle.
717 */
718 if (((tbp->b_flags & (B_CLUSTEROK|B_MALLOC)) != B_CLUSTEROK) ||
719 (tbp->b_bcount != tbp->b_bufsize) ||
720 (tbp->b_bcount != size) ||
721 (len == 1) ||
722 ((bp = getpbuf(&cluster_pbuf_freecnt)) == NULL)) {
723 totalwritten += tbp->b_bufsize;
724 bawrite(tbp);
725 ++start_lbn;
726 --len;
727 continue;
728 }
729
730 /*
731 * We got a pbuf to make the cluster in.
732 * so initialise it.
733 */
734 TAILQ_INIT(&bp->b_cluster.cluster_head);
735 bp->b_bcount = 0;
736 bp->b_bufsize = 0;
737 bp->b_npages = 0;
738 if (tbp->b_wcred != NOCRED) {
739 bp->b_wcred = tbp->b_wcred;
740 crhold(bp->b_wcred);
741 }
742
743 bp->b_blkno = tbp->b_blkno;
744 bp->b_lblkno = tbp->b_lblkno;
745 bp->b_offset = tbp->b_offset;
746 bp->b_data = (char *)((vm_offset_t)bp->b_data |
747 ((vm_offset_t)tbp->b_data & PAGE_MASK));
748 bp->b_flags |= B_CLUSTER |
749 (tbp->b_flags & (B_VMIO | B_NEEDCOMMIT));
750 bp->b_iodone = cluster_callback;
751 pbgetvp(vp, bp);
752 /*
753 * From this location in the file, scan forward to see
754 * if there are buffers with adjacent data that need to
755 * be written as well.
756 */
757 for (i = 0; i < len; ++i, ++start_lbn) {
758 if (i != 0) { /* If not the first buffer */
759 s = splbio();
760 /*
761 * If the adjacent data is not even in core it
762 * can't need to be written.
763 */
764 if ((tbp = gbincore(vp, start_lbn)) == NULL) {
765 splx(s);
766 break;
767 }
768
769 /*
770 * If it IS in core, but has different
771 * characteristics, don't cluster with it.
772 */
773 if ((tbp->b_flags & (B_VMIO | B_CLUSTEROK |
774 B_INVAL | B_DELWRI | B_NEEDCOMMIT))
775 != (B_DELWRI | B_CLUSTEROK |
776 (bp->b_flags & (B_VMIO | B_NEEDCOMMIT))) ||
777 tbp->b_wcred != bp->b_wcred ||
778 BUF_LOCK(tbp, LK_EXCLUSIVE | LK_NOWAIT)) {
779 splx(s);
780 break;
781 }
782
783 /*
784 * Check that the combined cluster
785 * would make sense with regard to pages
786 * and would not be too large
787 */
788 if ((tbp->b_bcount != size) ||
789 ((bp->b_blkno + (dbsize * i)) !=
790 tbp->b_blkno) ||
791 ((tbp->b_npages + bp->b_npages) >
792 (vp->v_mount->mnt_iosize_max / PAGE_SIZE))) {
793 BUF_UNLOCK(tbp);
794 splx(s);
795 break;
796 }
797 /*
798 * Ok, it's passed all the tests,
799 * so remove it from the free list
800 * and mark it busy. We will use it.
801 */
802 bremfree(tbp);
803 tbp->b_flags &= ~B_DONE;
804 splx(s);
805 } /* end of code for non-first buffers only */
806 /* check for latent dependencies to be handled */
807 if ((LIST_FIRST(&tbp->b_dep)) != NULL &&
808 bioops.io_start)
809 (*bioops.io_start)(tbp);
810 /*
811 * If the IO is via the VM then we do some
812 * special VM hackery. (yuck)
813 */
814 if (tbp->b_flags & B_VMIO) {
815 vm_page_t m;
816
817 if (i != 0) { /* if not first buffer */
818 for (j = 0; j < tbp->b_npages; j += 1) {
819 m = tbp->b_pages[j];
820 if (m->flags & PG_BUSY) {
821 bqrelse(tbp);
822 goto finishcluster;
823 }
824 }
825 }
826
827 for (j = 0; j < tbp->b_npages; j += 1) {
828 m = tbp->b_pages[j];
829 vm_page_io_start(m);
830 vm_object_pip_add(m->object, 1);
831 if ((bp->b_npages == 0) ||
832 (bp->b_pages[bp->b_npages - 1] != m)) {
833 bp->b_pages[bp->b_npages] = m;
834 bp->b_npages++;
835 }
836 }
837 }
838 bp->b_bcount += size;
839 bp->b_bufsize += size;
840
841 s = splbio();
842 bundirty(tbp);
843 tbp->b_flags &= ~B_DONE;
844 tbp->b_ioflags &= ~BIO_ERROR;
845 tbp->b_flags |= B_ASYNC;
846 tbp->b_iocmd = BIO_WRITE;
847 reassignbuf(tbp, tbp->b_vp); /* put on clean list */
848 ++tbp->b_vp->v_numoutput;
849 splx(s);
850 BUF_KERNPROC(tbp);
851 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
852 tbp, b_cluster.cluster_entry);
853 }
854 finishcluster:
855 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
856 (vm_page_t *) bp->b_pages, bp->b_npages);
857 if (bp->b_bufsize > bp->b_kvasize)
858 panic(
859 "cluster_wbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
860 bp->b_bufsize, bp->b_kvasize);
861 bp->b_kvasize = bp->b_bufsize;
862 totalwritten += bp->b_bufsize;
863 bp->b_dirtyoff = 0;
864 bp->b_dirtyend = bp->b_bufsize;
865 bawrite(bp);
866
867 len -= i;
868 }
869 return totalwritten;
870}
871
872/*
873 * Collect together all the buffers in a cluster.
874 * Plus add one additional buffer.
875 */
876static struct cluster_save *
877cluster_collectbufs(vp, last_bp)
878 struct vnode *vp;
879 struct buf *last_bp;
880{
881 struct cluster_save *buflist;
882 struct buf *bp;
883 daddr_t lbn;
884 int i, len;
885
886 len = vp->v_lastw - vp->v_cstart + 1;
887 buflist = malloc(sizeof(struct buf *) * (len + 1) + sizeof(*buflist),
888 M_SEGMENT, M_WAITOK);
889 buflist->bs_nchildren = 0;
890 buflist->bs_children = (struct buf **) (buflist + 1);
891 for (lbn = vp->v_cstart, i = 0; i < len; lbn++, i++) {
892 (void) bread(vp, lbn, last_bp->b_bcount, NOCRED, &bp);
893 buflist->bs_children[i] = bp;
894 if (bp->b_blkno == bp->b_lblkno)
895 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
896 NULL, NULL);
897 }
898 buflist->bs_children[i] = bp = last_bp;
899 if (bp->b_blkno == bp->b_lblkno)
900 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
901 NULL, NULL);
902 buflist->bs_nchildren = i + 1;
903 return (buflist);
904}
|