ffs_softdep.c revision 149808
1/*-
2 * Copyright 1998, 2000 Marshall Kirk McKusick. All Rights Reserved.
3 *
4 * The soft updates code is derived from the appendix of a University
5 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt,
6 * "Soft Updates: A Solution to the Metadata Update Problem in File
7 * Systems", CSE-TR-254-95, August 1995).
8 *
9 * Further information about soft updates can be obtained from:
10 *
11 *	Marshall Kirk McKusick		http://www.mckusick.com/softdep/
12 *	1614 Oxford Street		mckusick@mckusick.com
13 *	Berkeley, CA 94709-1608		+1-510-843-9542
14 *	USA
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 *
20 * 1. Redistributions of source code must retain the above copyright
21 *    notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 *    notice, this list of conditions and the following disclaimer in the
24 *    documentation and/or other materials provided with the distribution.
25 *
26 * THIS SOFTWARE IS PROVIDED BY MARSHALL KIRK MCKUSICK ``AS IS'' AND ANY
27 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
28 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
29 * DISCLAIMED.  IN NO EVENT SHALL MARSHALL KIRK MCKUSICK BE LIABLE FOR
30 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 *	from: @(#)ffs_softdep.c	9.59 (McKusick) 6/21/00
39 */
40
41#include <sys/cdefs.h>
42__FBSDID("$FreeBSD: head/sys/ufs/ffs/ffs_softdep.c 149808 2005-09-05 22:14:33Z tegge $");
43
44/*
45 * For now we want the safety net that the DIAGNOSTIC and DEBUG flags provide.
46 */
47#ifndef DIAGNOSTIC
48#define DIAGNOSTIC
49#endif
50#ifndef DEBUG
51#define DEBUG
52#endif
53
54#include <sys/param.h>
55#include <sys/kernel.h>
56#include <sys/systm.h>
57#include <sys/bio.h>
58#include <sys/buf.h>
59#include <sys/kdb.h>
60#include <sys/lock.h>
61#include <sys/malloc.h>
62#include <sys/mount.h>
63#include <sys/mutex.h>
64#include <sys/proc.h>
65#include <sys/stat.h>
66#include <sys/sysctl.h>
67#include <sys/syslog.h>
68#include <sys/vnode.h>
69#include <sys/conf.h>
70#include <ufs/ufs/dir.h>
71#include <ufs/ufs/extattr.h>
72#include <ufs/ufs/quota.h>
73#include <ufs/ufs/inode.h>
74#include <ufs/ufs/ufsmount.h>
75#include <ufs/ffs/fs.h>
76#include <ufs/ffs/softdep.h>
77#include <ufs/ffs/ffs_extern.h>
78#include <ufs/ufs/ufs_extern.h>
79
80#include <vm/vm.h>
81
82#include "opt_ffs.h"
83
84#ifndef SOFTUPDATES
85
86int
87softdep_flushfiles(oldmnt, flags, td)
88	struct mount *oldmnt;
89	int flags;
90	struct thread *td;
91{
92
93	panic("softdep_flushfiles called");
94}
95
96int
97softdep_mount(devvp, mp, fs, cred)
98	struct vnode *devvp;
99	struct mount *mp;
100	struct fs *fs;
101	struct ucred *cred;
102{
103
104	return (0);
105}
106
107void
108softdep_initialize()
109{
110
111	return;
112}
113
114void
115softdep_uninitialize()
116{
117
118	return;
119}
120
121void
122softdep_setup_inomapdep(bp, ip, newinum)
123	struct buf *bp;
124	struct inode *ip;
125	ino_t newinum;
126{
127
128	panic("softdep_setup_inomapdep called");
129}
130
131void
132softdep_setup_blkmapdep(bp, fs, newblkno)
133	struct buf *bp;
134	struct fs *fs;
135	ufs2_daddr_t newblkno;
136{
137
138	panic("softdep_setup_blkmapdep called");
139}
140
141void
142softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
143	struct inode *ip;
144	ufs_lbn_t lbn;
145	ufs2_daddr_t newblkno;
146	ufs2_daddr_t oldblkno;
147	long newsize;
148	long oldsize;
149	struct buf *bp;
150{
151
152	panic("softdep_setup_allocdirect called");
153}
154
155void
156softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
157	struct inode *ip;
158	ufs_lbn_t lbn;
159	ufs2_daddr_t newblkno;
160	ufs2_daddr_t oldblkno;
161	long newsize;
162	long oldsize;
163	struct buf *bp;
164{
165
166	panic("softdep_setup_allocdirect called");
167}
168
169void
170softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
171	struct inode *ip;
172	ufs_lbn_t lbn;
173	struct buf *bp;
174	int ptrno;
175	ufs2_daddr_t newblkno;
176	ufs2_daddr_t oldblkno;
177	struct buf *nbp;
178{
179
180	panic("softdep_setup_allocindir_page called");
181}
182
183void
184softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
185	struct buf *nbp;
186	struct inode *ip;
187	struct buf *bp;
188	int ptrno;
189	ufs2_daddr_t newblkno;
190{
191
192	panic("softdep_setup_allocindir_meta called");
193}
194
195void
196softdep_setup_freeblocks(ip, length, flags)
197	struct inode *ip;
198	off_t length;
199	int flags;
200{
201
202	panic("softdep_setup_freeblocks called");
203}
204
205void
206softdep_freefile(pvp, ino, mode)
207		struct vnode *pvp;
208		ino_t ino;
209		int mode;
210{
211
212	panic("softdep_freefile called");
213}
214
215int
216softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
217	struct buf *bp;
218	struct inode *dp;
219	off_t diroffset;
220	ino_t newinum;
221	struct buf *newdirbp;
222	int isnewblk;
223{
224
225	panic("softdep_setup_directory_add called");
226}
227
228void
229softdep_change_directoryentry_offset(dp, base, oldloc, newloc, entrysize)
230	struct inode *dp;
231	caddr_t base;
232	caddr_t oldloc;
233	caddr_t newloc;
234	int entrysize;
235{
236
237	panic("softdep_change_directoryentry_offset called");
238}
239
240void
241softdep_setup_remove(bp, dp, ip, isrmdir)
242	struct buf *bp;
243	struct inode *dp;
244	struct inode *ip;
245	int isrmdir;
246{
247
248	panic("softdep_setup_remove called");
249}
250
251void
252softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
253	struct buf *bp;
254	struct inode *dp;
255	struct inode *ip;
256	ino_t newinum;
257	int isrmdir;
258{
259
260	panic("softdep_setup_directory_change called");
261}
262
263void
264softdep_change_linkcnt(ip)
265	struct inode *ip;
266{
267
268	panic("softdep_change_linkcnt called");
269}
270
271void
272softdep_load_inodeblock(ip)
273	struct inode *ip;
274{
275
276	panic("softdep_load_inodeblock called");
277}
278
279void
280softdep_update_inodeblock(ip, bp, waitfor)
281	struct inode *ip;
282	struct buf *bp;
283	int waitfor;
284{
285
286	panic("softdep_update_inodeblock called");
287}
288
289int
290softdep_fsync(vp)
291	struct vnode *vp;	/* the "in_core" copy of the inode */
292{
293
294	return (0);
295}
296
297void
298softdep_fsync_mountdev(vp)
299	struct vnode *vp;
300{
301
302	return;
303}
304
305int
306softdep_flushworklist(oldmnt, countp, td)
307	struct mount *oldmnt;
308	int *countp;
309	struct thread *td;
310{
311
312	*countp = 0;
313	return (0);
314}
315
316int
317softdep_sync_metadata(struct vnode *vp)
318{
319
320	return (0);
321}
322
323int
324softdep_slowdown(vp)
325	struct vnode *vp;
326{
327
328	panic("softdep_slowdown called");
329}
330
331void
332softdep_releasefile(ip)
333	struct inode *ip;	/* inode with the zero effective link count */
334{
335
336	panic("softdep_releasefile called");
337}
338
339int
340softdep_request_cleanup(fs, vp)
341	struct fs *fs;
342	struct vnode *vp;
343{
344
345	return (0);
346}
347
348#else
349/*
350 * These definitions need to be adapted to the system to which
351 * this file is being ported.
352 */
353/*
354 * malloc types defined for the softdep system.
355 */
356static MALLOC_DEFINE(M_PAGEDEP, "pagedep","File page dependencies");
357static MALLOC_DEFINE(M_INODEDEP, "inodedep","Inode dependencies");
358static MALLOC_DEFINE(M_NEWBLK, "newblk","New block allocation");
359static MALLOC_DEFINE(M_BMSAFEMAP, "bmsafemap","Block or frag allocated from cyl group map");
360static MALLOC_DEFINE(M_ALLOCDIRECT, "allocdirect","Block or frag dependency for an inode");
361static MALLOC_DEFINE(M_INDIRDEP, "indirdep","Indirect block dependencies");
362static MALLOC_DEFINE(M_ALLOCINDIR, "allocindir","Block dependency for an indirect block");
363static MALLOC_DEFINE(M_FREEFRAG, "freefrag","Previously used frag for an inode");
364static MALLOC_DEFINE(M_FREEBLKS, "freeblks","Blocks freed from an inode");
365static MALLOC_DEFINE(M_FREEFILE, "freefile","Inode deallocated");
366static MALLOC_DEFINE(M_DIRADD, "diradd","New directory entry");
367static MALLOC_DEFINE(M_MKDIR, "mkdir","New directory");
368static MALLOC_DEFINE(M_DIRREM, "dirrem","Directory entry deleted");
369static MALLOC_DEFINE(M_NEWDIRBLK, "newdirblk","Unclaimed new directory block");
370static MALLOC_DEFINE(M_SAVEDINO, "savedino","Saved inodes");
371
372#define M_SOFTDEP_FLAGS	(M_WAITOK | M_USE_RESERVE)
373
374#define	D_PAGEDEP	0
375#define	D_INODEDEP	1
376#define	D_NEWBLK	2
377#define	D_BMSAFEMAP	3
378#define	D_ALLOCDIRECT	4
379#define	D_INDIRDEP	5
380#define	D_ALLOCINDIR	6
381#define	D_FREEFRAG	7
382#define	D_FREEBLKS	8
383#define	D_FREEFILE	9
384#define	D_DIRADD	10
385#define	D_MKDIR		11
386#define	D_DIRREM	12
387#define	D_NEWDIRBLK	13
388#define	D_LAST		D_NEWDIRBLK
389
390/*
391 * translate from workitem type to memory type
392 * MUST match the defines above, such that memtype[D_XXX] == M_XXX
393 */
394static struct malloc_type *memtype[] = {
395	M_PAGEDEP,
396	M_INODEDEP,
397	M_NEWBLK,
398	M_BMSAFEMAP,
399	M_ALLOCDIRECT,
400	M_INDIRDEP,
401	M_ALLOCINDIR,
402	M_FREEFRAG,
403	M_FREEBLKS,
404	M_FREEFILE,
405	M_DIRADD,
406	M_MKDIR,
407	M_DIRREM,
408	M_NEWDIRBLK
409};
410
411#define DtoM(type) (memtype[type])
412
413/*
414 * Names of malloc types.
415 */
416#define TYPENAME(type)  \
417	((unsigned)(type) < D_LAST ? memtype[type]->ks_shortdesc : "???")
418/*
419 * End system adaptaion definitions.
420 */
421
422/*
423 * Forward declarations.
424 */
425struct inodedep_hashhead;
426struct newblk_hashhead;
427struct pagedep_hashhead;
428
429/*
430 * Internal function prototypes.
431 */
432static	void softdep_error(char *, int);
433static	void drain_output(struct vnode *);
434static	struct buf *getdirtybuf(struct buf *, struct mtx *, int);
435static	void clear_remove(struct thread *);
436static	void clear_inodedeps(struct thread *);
437static	int flush_pagedep_deps(struct vnode *, struct mount *,
438	    struct diraddhd *);
439static	int flush_inodedep_deps(struct fs *, ino_t);
440static	int flush_deplist(struct allocdirectlst *, int, int *);
441static	int handle_written_filepage(struct pagedep *, struct buf *);
442static  void diradd_inode_written(struct diradd *, struct inodedep *);
443static	int handle_written_inodeblock(struct inodedep *, struct buf *);
444static	void handle_allocdirect_partdone(struct allocdirect *);
445static	void handle_allocindir_partdone(struct allocindir *);
446static	void initiate_write_filepage(struct pagedep *, struct buf *);
447static	void handle_written_mkdir(struct mkdir *, int);
448static	void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *);
449static	void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *);
450static	void handle_workitem_freefile(struct freefile *);
451static	void handle_workitem_remove(struct dirrem *, struct vnode *);
452static	struct dirrem *newdirrem(struct buf *, struct inode *,
453	    struct inode *, int, struct dirrem **);
454static	void free_diradd(struct diradd *);
455static	void free_allocindir(struct allocindir *, struct inodedep *);
456static	void free_newdirblk(struct newdirblk *);
457static	int indir_trunc(struct freeblks *, ufs2_daddr_t, int, ufs_lbn_t,
458	    ufs2_daddr_t *);
459static	void deallocate_dependencies(struct buf *, struct inodedep *);
460static	void free_allocdirect(struct allocdirectlst *,
461	    struct allocdirect *, int);
462static	int check_inode_unwritten(struct inodedep *);
463static	int free_inodedep(struct inodedep *);
464static	void handle_workitem_freeblocks(struct freeblks *, int);
465static	void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *);
466static	void setup_allocindir_phase2(struct buf *, struct inode *,
467	    struct allocindir *);
468static	struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t,
469	    ufs2_daddr_t);
470static	void handle_workitem_freefrag(struct freefrag *);
471static	struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long);
472static	void allocdirect_merge(struct allocdirectlst *,
473	    struct allocdirect *, struct allocdirect *);
474static	struct bmsafemap *bmsafemap_lookup(struct buf *);
475static	int newblk_find(struct newblk_hashhead *, struct fs *, ufs2_daddr_t,
476	    struct newblk **);
477static	int newblk_lookup(struct fs *, ufs2_daddr_t, int, struct newblk **);
478static	int inodedep_find(struct inodedep_hashhead *, struct fs *, ino_t,
479	    struct inodedep **);
480static	int inodedep_lookup(struct fs *, ino_t, int, struct inodedep **);
481static	int pagedep_lookup(struct inode *, ufs_lbn_t, int, struct pagedep **);
482static	int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t,
483	    struct mount *mp, int, struct pagedep **);
484static	void pause_timer(void *);
485static	int request_cleanup(int);
486static	int process_worklist_item(struct mount *, int);
487static	void add_to_worklist(struct worklist *);
488
489/*
490 * Exported softdep operations.
491 */
492static	void softdep_disk_io_initiation(struct buf *);
493static	void softdep_disk_write_complete(struct buf *);
494static	void softdep_deallocate_dependencies(struct buf *);
495static	int softdep_count_dependencies(struct buf *bp, int);
496
497static struct mtx lk;
498MTX_SYSINIT(softdep_lock, &lk, "Softdep Lock", MTX_DEF);
499
500#define ACQUIRE_LOCK(lk)		mtx_lock(lk)
501#define FREE_LOCK(lk)			mtx_unlock(lk)
502
503/*
504 * Worklist queue management.
505 * These routines require that the lock be held.
506 */
507#ifndef /* NOT */ DEBUG
508#define WORKLIST_INSERT(head, item) do {	\
509	(item)->wk_state |= ONWORKLIST;		\
510	LIST_INSERT_HEAD(head, item, wk_list);	\
511} while (0)
512#define WORKLIST_REMOVE(item) do {		\
513	(item)->wk_state &= ~ONWORKLIST;	\
514	LIST_REMOVE(item, wk_list);		\
515} while (0)
516#define WORKITEM_FREE(item, type) FREE(item, DtoM(type))
517
518#else /* DEBUG */
519static	void worklist_insert(struct workhead *, struct worklist *);
520static	void worklist_remove(struct worklist *);
521static	void workitem_free(struct worklist *, int);
522
523#define WORKLIST_INSERT(head, item) worklist_insert(head, item)
524#define WORKLIST_REMOVE(item) worklist_remove(item)
525#define WORKITEM_FREE(item, type) workitem_free((struct worklist *)item, type)
526
527static void
528worklist_insert(head, item)
529	struct workhead *head;
530	struct worklist *item;
531{
532
533	mtx_assert(&lk, MA_OWNED);
534	if (item->wk_state & ONWORKLIST)
535		panic("worklist_insert: already on list");
536	item->wk_state |= ONWORKLIST;
537	LIST_INSERT_HEAD(head, item, wk_list);
538}
539
540static void
541worklist_remove(item)
542	struct worklist *item;
543{
544
545	mtx_assert(&lk, MA_OWNED);
546	if ((item->wk_state & ONWORKLIST) == 0)
547		panic("worklist_remove: not on list");
548	item->wk_state &= ~ONWORKLIST;
549	LIST_REMOVE(item, wk_list);
550}
551
552static void
553workitem_free(item, type)
554	struct worklist *item;
555	int type;
556{
557
558	if (item->wk_state & ONWORKLIST)
559		panic("workitem_free: still on list");
560	if (item->wk_type != type)
561		panic("workitem_free: type mismatch");
562	FREE(item, DtoM(type));
563}
564#endif /* DEBUG */
565
566/*
567 * Workitem queue management
568 */
569static struct workhead softdep_workitem_pending;
570static struct worklist *worklist_tail;
571static int num_on_worklist;	/* number of worklist items to be processed */
572static int softdep_worklist_busy; /* 1 => trying to do unmount */
573static int softdep_worklist_req; /* serialized waiters */
574static int max_softdeps;	/* maximum number of structs before slowdown */
575static int maxindirdeps = 50;	/* max number of indirdeps before slowdown */
576static int tickdelay = 2;	/* number of ticks to pause during slowdown */
577static int proc_waiting;	/* tracks whether we have a timeout posted */
578static int *stat_countp;	/* statistic to count in proc_waiting timeout */
579static struct callout_handle handle; /* handle on posted proc_waiting timeout */
580static struct thread *filesys_syncer; /* proc of filesystem syncer process */
581static int req_clear_inodedeps;	/* syncer process flush some inodedeps */
582#define FLUSH_INODES		1
583static int req_clear_remove;	/* syncer process flush some freeblks */
584#define FLUSH_REMOVE		2
585#define FLUSH_REMOVE_WAIT	3
586/*
587 * runtime statistics
588 */
589static int stat_worklist_push;	/* number of worklist cleanups */
590static int stat_blk_limit_push;	/* number of times block limit neared */
591static int stat_ino_limit_push;	/* number of times inode limit neared */
592static int stat_blk_limit_hit;	/* number of times block slowdown imposed */
593static int stat_ino_limit_hit;	/* number of times inode slowdown imposed */
594static int stat_sync_limit_hit;	/* number of synchronous slowdowns imposed */
595static int stat_indir_blk_ptrs;	/* bufs redirtied as indir ptrs not written */
596static int stat_inode_bitmap;	/* bufs redirtied as inode bitmap not written */
597static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */
598static int stat_dir_entry;	/* bufs redirtied as dir entry cannot write */
599
600SYSCTL_INT(_debug, OID_AUTO, max_softdeps, CTLFLAG_RW, &max_softdeps, 0, "");
601SYSCTL_INT(_debug, OID_AUTO, tickdelay, CTLFLAG_RW, &tickdelay, 0, "");
602SYSCTL_INT(_debug, OID_AUTO, maxindirdeps, CTLFLAG_RW, &maxindirdeps, 0, "");
603SYSCTL_INT(_debug, OID_AUTO, worklist_push, CTLFLAG_RW, &stat_worklist_push, 0,"");
604SYSCTL_INT(_debug, OID_AUTO, blk_limit_push, CTLFLAG_RW, &stat_blk_limit_push, 0,"");
605SYSCTL_INT(_debug, OID_AUTO, ino_limit_push, CTLFLAG_RW, &stat_ino_limit_push, 0,"");
606SYSCTL_INT(_debug, OID_AUTO, blk_limit_hit, CTLFLAG_RW, &stat_blk_limit_hit, 0, "");
607SYSCTL_INT(_debug, OID_AUTO, ino_limit_hit, CTLFLAG_RW, &stat_ino_limit_hit, 0, "");
608SYSCTL_INT(_debug, OID_AUTO, sync_limit_hit, CTLFLAG_RW, &stat_sync_limit_hit, 0, "");
609SYSCTL_INT(_debug, OID_AUTO, indir_blk_ptrs, CTLFLAG_RW, &stat_indir_blk_ptrs, 0, "");
610SYSCTL_INT(_debug, OID_AUTO, inode_bitmap, CTLFLAG_RW, &stat_inode_bitmap, 0, "");
611SYSCTL_INT(_debug, OID_AUTO, direct_blk_ptrs, CTLFLAG_RW, &stat_direct_blk_ptrs, 0, "");
612SYSCTL_INT(_debug, OID_AUTO, dir_entry, CTLFLAG_RW, &stat_dir_entry, 0, "");
613SYSCTL_INT(_debug, OID_AUTO, worklist_num, CTLFLAG_RD, &num_on_worklist, 0, "");
614
615SYSCTL_DECL(_vfs_ffs);
616
617static int compute_summary_at_mount = 0;	/* Whether to recompute the summary at mount time */
618SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW,
619	   &compute_summary_at_mount, 0, "Recompute summary at mount");
620
621/*
622 * Add an item to the end of the work queue.
623 * This routine requires that the lock be held.
624 * This is the only routine that adds items to the list.
625 * The following routine is the only one that removes items
626 * and does so in order from first to last.
627 */
628static void
629add_to_worklist(wk)
630	struct worklist *wk;
631{
632
633	mtx_assert(&lk, MA_OWNED);
634	if (wk->wk_state & ONWORKLIST)
635		panic("add_to_worklist: already on list");
636	wk->wk_state |= ONWORKLIST;
637	if (LIST_FIRST(&softdep_workitem_pending) == NULL)
638		LIST_INSERT_HEAD(&softdep_workitem_pending, wk, wk_list);
639	else
640		LIST_INSERT_AFTER(worklist_tail, wk, wk_list);
641	worklist_tail = wk;
642	num_on_worklist += 1;
643}
644
645/*
646 * Process that runs once per second to handle items in the background queue.
647 *
648 * Note that we ensure that everything is done in the order in which they
649 * appear in the queue. The code below depends on this property to ensure
650 * that blocks of a file are freed before the inode itself is freed. This
651 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated
652 * until all the old ones have been purged from the dependency lists.
653 */
654int
655softdep_process_worklist(matchmnt)
656	struct mount *matchmnt;
657{
658	struct thread *td = curthread;
659	int cnt, matchcnt, loopcount;
660	long starttime;
661
662	/*
663	 * Record the process identifier of our caller so that we can give
664	 * this process preferential treatment in request_cleanup below.
665	 */
666	filesys_syncer = td;
667	matchcnt = 0;
668
669	/*
670	 * There is no danger of having multiple processes run this
671	 * code, but we have to single-thread it when softdep_flushfiles()
672	 * is in operation to get an accurate count of the number of items
673	 * related to its mount point that are in the list.
674	 */
675	ACQUIRE_LOCK(&lk);
676	if (matchmnt == NULL) {
677		if (softdep_worklist_busy < 0) {
678			FREE_LOCK(&lk);
679			return(-1);
680		}
681		softdep_worklist_busy += 1;
682	}
683
684	/*
685	 * If requested, try removing inode or removal dependencies.
686	 */
687	if (req_clear_inodedeps) {
688		clear_inodedeps(td);
689		req_clear_inodedeps -= 1;
690		wakeup_one(&proc_waiting);
691	}
692	if (req_clear_remove) {
693		clear_remove(td);
694		req_clear_remove -= 1;
695		wakeup_one(&proc_waiting);
696	}
697	loopcount = 1;
698	starttime = time_second;
699	while (num_on_worklist > 0) {
700		if ((cnt = process_worklist_item(matchmnt, 0)) == -1)
701			break;
702		else
703			matchcnt += cnt;
704
705		/*
706		 * If a umount operation wants to run the worklist
707		 * accurately, abort.
708		 */
709		if (softdep_worklist_req && matchmnt == NULL) {
710			matchcnt = -1;
711			break;
712		}
713
714		/*
715		 * If requested, try removing inode or removal dependencies.
716		 */
717		if (req_clear_inodedeps) {
718			clear_inodedeps(td);
719			req_clear_inodedeps -= 1;
720			wakeup_one(&proc_waiting);
721		}
722		if (req_clear_remove) {
723			clear_remove(td);
724			req_clear_remove -= 1;
725			wakeup_one(&proc_waiting);
726		}
727		/*
728		 * We do not generally want to stop for buffer space, but if
729		 * we are really being a buffer hog, we will stop and wait.
730		 */
731		if (loopcount++ % 128 == 0) {
732			FREE_LOCK(&lk);
733			bwillwrite();
734			ACQUIRE_LOCK(&lk);
735		}
736		/*
737		 * Never allow processing to run for more than one
738		 * second. Otherwise the other syncer tasks may get
739		 * excessively backlogged.
740		 */
741		if (starttime != time_second && matchmnt == NULL) {
742			matchcnt = -1;
743			break;
744		}
745	}
746	if (matchmnt == NULL) {
747		softdep_worklist_busy -= 1;
748		if (softdep_worklist_req && softdep_worklist_busy == 0)
749			wakeup(&softdep_worklist_req);
750	}
751	FREE_LOCK(&lk);
752	return (matchcnt);
753}
754
755/*
756 * Process one item on the worklist.
757 */
758static int
759process_worklist_item(matchmnt, flags)
760	struct mount *matchmnt;
761	int flags;
762{
763	struct worklist *wk, *wkend;
764	struct mount *mp;
765	struct vnode *vp;
766	int matchcnt = 0;
767
768	mtx_assert(&lk, MA_OWNED);
769	/*
770	 * If we are being called because of a process doing a
771	 * copy-on-write, then it is not safe to write as we may
772	 * recurse into the copy-on-write routine.
773	 */
774	if (curthread->td_pflags & TDP_COWINPROGRESS)
775		return (-1);
776	/*
777	 * Normally we just process each item on the worklist in order.
778	 * However, if we are in a situation where we cannot lock any
779	 * inodes, we have to skip over any dirrem requests whose
780	 * vnodes are resident and locked.
781	 */
782	vp = NULL;
783	LIST_FOREACH(wk, &softdep_workitem_pending, wk_list) {
784		if (wk->wk_state & INPROGRESS)
785			continue;
786		if ((flags & LK_NOWAIT) == 0 || wk->wk_type != D_DIRREM)
787			break;
788		wk->wk_state |= INPROGRESS;
789		FREE_LOCK(&lk);
790		ffs_vget(WK_DIRREM(wk)->dm_mnt, WK_DIRREM(wk)->dm_oldinum,
791		    LK_NOWAIT | LK_EXCLUSIVE, &vp);
792		ACQUIRE_LOCK(&lk);
793		wk->wk_state &= ~INPROGRESS;
794		if (vp != NULL)
795			break;
796	}
797	if (wk == 0)
798		return (-1);
799	/*
800	 * Remove the item to be processed. If we are removing the last
801	 * item on the list, we need to recalculate the tail pointer.
802	 * As this happens rarely and usually when the list is short,
803	 * we just run down the list to find it rather than tracking it
804	 * in the above loop.
805	 */
806	WORKLIST_REMOVE(wk);
807	if (wk == worklist_tail) {
808		LIST_FOREACH(wkend, &softdep_workitem_pending, wk_list)
809			if (LIST_NEXT(wkend, wk_list) == NULL)
810				break;
811		worklist_tail = wkend;
812	}
813	num_on_worklist -= 1;
814	FREE_LOCK(&lk);
815	switch (wk->wk_type) {
816
817	case D_DIRREM:
818		/* removal of a directory entry */
819		mp = WK_DIRREM(wk)->dm_mnt;
820		if (vn_write_suspend_wait(NULL, mp, V_NOWAIT))
821			panic("%s: dirrem on suspended filesystem",
822				"process_worklist_item");
823		if (mp == matchmnt)
824			matchcnt += 1;
825		handle_workitem_remove(WK_DIRREM(wk), vp);
826		break;
827
828	case D_FREEBLKS:
829		/* releasing blocks and/or fragments from a file */
830		mp = WK_FREEBLKS(wk)->fb_mnt;
831		if (vn_write_suspend_wait(NULL, mp, V_NOWAIT))
832			panic("%s: freeblks on suspended filesystem",
833				"process_worklist_item");
834		if (mp == matchmnt)
835			matchcnt += 1;
836		handle_workitem_freeblocks(WK_FREEBLKS(wk), flags & LK_NOWAIT);
837		break;
838
839	case D_FREEFRAG:
840		/* releasing a fragment when replaced as a file grows */
841		mp = WK_FREEFRAG(wk)->ff_mnt;
842		if (vn_write_suspend_wait(NULL, mp, V_NOWAIT))
843			panic("%s: freefrag on suspended filesystem",
844				"process_worklist_item");
845		if (mp == matchmnt)
846			matchcnt += 1;
847		handle_workitem_freefrag(WK_FREEFRAG(wk));
848		break;
849
850	case D_FREEFILE:
851		/* releasing an inode when its link count drops to 0 */
852		mp = WK_FREEFILE(wk)->fx_mnt;
853		if (vn_write_suspend_wait(NULL, mp, V_NOWAIT))
854			panic("%s: freefile on suspended filesystem",
855				"process_worklist_item");
856		if (mp == matchmnt)
857			matchcnt += 1;
858		handle_workitem_freefile(WK_FREEFILE(wk));
859		break;
860
861	default:
862		panic("%s_process_worklist: Unknown type %s",
863		    "softdep", TYPENAME(wk->wk_type));
864		/* NOTREACHED */
865	}
866	ACQUIRE_LOCK(&lk);
867	return (matchcnt);
868}
869
870/*
871 * Move dependencies from one buffer to another.
872 */
873void
874softdep_move_dependencies(oldbp, newbp)
875	struct buf *oldbp;
876	struct buf *newbp;
877{
878	struct worklist *wk, *wktail;
879
880	if (LIST_FIRST(&newbp->b_dep) != NULL)
881		panic("softdep_move_dependencies: need merge code");
882	wktail = 0;
883	ACQUIRE_LOCK(&lk);
884	while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) {
885		LIST_REMOVE(wk, wk_list);
886		if (wktail == 0)
887			LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list);
888		else
889			LIST_INSERT_AFTER(wktail, wk, wk_list);
890		wktail = wk;
891	}
892	FREE_LOCK(&lk);
893}
894
895/*
896 * Purge the work list of all items associated with a particular mount point.
897 */
898int
899softdep_flushworklist(oldmnt, countp, td)
900	struct mount *oldmnt;
901	int *countp;
902	struct thread *td;
903{
904	struct vnode *devvp;
905	int count, error = 0;
906
907	/*
908	 * Await our turn to clear out the queue, then serialize access.
909	 */
910	ACQUIRE_LOCK(&lk);
911	while (softdep_worklist_busy) {
912		softdep_worklist_req += 1;
913		msleep(&softdep_worklist_req, &lk, PRIBIO, "softflush", 0);
914		softdep_worklist_req -= 1;
915	}
916	softdep_worklist_busy = -1;
917	FREE_LOCK(&lk);
918	/*
919	 * Alternately flush the block device associated with the mount
920	 * point and process any dependencies that the flushing
921	 * creates. We continue until no more worklist dependencies
922	 * are found.
923	 */
924	*countp = 0;
925	devvp = VFSTOUFS(oldmnt)->um_devvp;
926	while ((count = softdep_process_worklist(oldmnt)) > 0) {
927		*countp += count;
928		vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY, td);
929		error = VOP_FSYNC(devvp, MNT_WAIT, td);
930		VOP_UNLOCK(devvp, 0, td);
931		if (error)
932			break;
933	}
934	ACQUIRE_LOCK(&lk);
935	softdep_worklist_busy = 0;
936	if (softdep_worklist_req)
937		wakeup(&softdep_worklist_req);
938	FREE_LOCK(&lk);
939	return (error);
940}
941
942/*
943 * Flush all vnodes and worklist items associated with a specified mount point.
944 */
945int
946softdep_flushfiles(oldmnt, flags, td)
947	struct mount *oldmnt;
948	int flags;
949	struct thread *td;
950{
951	int error, count, loopcnt;
952
953	error = 0;
954
955	/*
956	 * Alternately flush the vnodes associated with the mount
957	 * point and process any dependencies that the flushing
958	 * creates. In theory, this loop can happen at most twice,
959	 * but we give it a few extra just to be sure.
960	 */
961	for (loopcnt = 10; loopcnt > 0; loopcnt--) {
962		/*
963		 * Do another flush in case any vnodes were brought in
964		 * as part of the cleanup operations.
965		 */
966		if ((error = ffs_flushfiles(oldmnt, flags, td)) != 0)
967			break;
968		if ((error = softdep_flushworklist(oldmnt, &count, td)) != 0 ||
969		    count == 0)
970			break;
971	}
972	/*
973	 * If we are unmounting then it is an error to fail. If we
974	 * are simply trying to downgrade to read-only, then filesystem
975	 * activity can keep us busy forever, so we just fail with EBUSY.
976	 */
977	if (loopcnt == 0) {
978		if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT)
979			panic("softdep_flushfiles: looping");
980		error = EBUSY;
981	}
982	return (error);
983}
984
985/*
986 * Structure hashing.
987 *
988 * There are three types of structures that can be looked up:
989 *	1) pagedep structures identified by mount point, inode number,
990 *	   and logical block.
991 *	2) inodedep structures identified by mount point and inode number.
992 *	3) newblk structures identified by mount point and
993 *	   physical block number.
994 *
995 * The "pagedep" and "inodedep" dependency structures are hashed
996 * separately from the file blocks and inodes to which they correspond.
997 * This separation helps when the in-memory copy of an inode or
998 * file block must be replaced. It also obviates the need to access
999 * an inode or file page when simply updating (or de-allocating)
1000 * dependency structures. Lookup of newblk structures is needed to
1001 * find newly allocated blocks when trying to associate them with
1002 * their allocdirect or allocindir structure.
1003 *
1004 * The lookup routines optionally create and hash a new instance when
1005 * an existing entry is not found.
1006 */
1007#define DEPALLOC	0x0001	/* allocate structure if lookup fails */
1008#define NODELAY		0x0002	/* cannot do background work */
1009
1010/*
1011 * Structures and routines associated with pagedep caching.
1012 */
1013LIST_HEAD(pagedep_hashhead, pagedep) *pagedep_hashtbl;
1014u_long	pagedep_hash;		/* size of hash table - 1 */
1015#define	PAGEDEP_HASH(mp, inum, lbn) \
1016	(&pagedep_hashtbl[((((register_t)(mp)) >> 13) + (inum) + (lbn)) & \
1017	    pagedep_hash])
1018
1019static int
1020pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp)
1021	struct pagedep_hashhead *pagedephd;
1022	ino_t ino;
1023	ufs_lbn_t lbn;
1024	struct mount *mp;
1025	int flags;
1026	struct pagedep **pagedeppp;
1027{
1028	struct pagedep *pagedep;
1029
1030	LIST_FOREACH(pagedep, pagedephd, pd_hash)
1031		if (ino == pagedep->pd_ino &&
1032		    lbn == pagedep->pd_lbn &&
1033		    mp == pagedep->pd_mnt)
1034			break;
1035	if (pagedep) {
1036		*pagedeppp = pagedep;
1037		if ((flags & DEPALLOC) != 0 &&
1038		    (pagedep->pd_state & ONWORKLIST) == 0)
1039			return (0);
1040		return (1);
1041	}
1042	*pagedeppp = NULL;
1043	return (0);
1044}
1045/*
1046 * Look up a pagedep. Return 1 if found, 0 if not found or found
1047 * when asked to allocate but not associated with any buffer.
1048 * If not found, allocate if DEPALLOC flag is passed.
1049 * Found or allocated entry is returned in pagedeppp.
1050 * This routine must be called with splbio interrupts blocked.
1051 */
1052static int
1053pagedep_lookup(ip, lbn, flags, pagedeppp)
1054	struct inode *ip;
1055	ufs_lbn_t lbn;
1056	int flags;
1057	struct pagedep **pagedeppp;
1058{
1059	struct pagedep *pagedep;
1060	struct pagedep_hashhead *pagedephd;
1061	struct mount *mp;
1062	int ret;
1063	int i;
1064
1065	mtx_assert(&lk, MA_OWNED);
1066	mp = ITOV(ip)->v_mount;
1067	pagedephd = PAGEDEP_HASH(mp, ip->i_number, lbn);
1068
1069	ret = pagedep_find(pagedephd, ip->i_number, lbn, mp, flags, pagedeppp);
1070	if (*pagedeppp || (flags & DEPALLOC) == 0)
1071		return (ret);
1072	FREE_LOCK(&lk);
1073	MALLOC(pagedep, struct pagedep *, sizeof(struct pagedep),
1074	    M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO);
1075	ACQUIRE_LOCK(&lk);
1076	ret = pagedep_find(pagedephd, ip->i_number, lbn, mp, flags, pagedeppp);
1077	if (*pagedeppp) {
1078		FREE(pagedep, M_PAGEDEP);
1079		return (ret);
1080	}
1081	pagedep->pd_list.wk_type = D_PAGEDEP;
1082	pagedep->pd_mnt = mp;
1083	pagedep->pd_ino = ip->i_number;
1084	pagedep->pd_lbn = lbn;
1085	LIST_INIT(&pagedep->pd_dirremhd);
1086	LIST_INIT(&pagedep->pd_pendinghd);
1087	for (i = 0; i < DAHASHSZ; i++)
1088		LIST_INIT(&pagedep->pd_diraddhd[i]);
1089	LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash);
1090	*pagedeppp = pagedep;
1091	return (0);
1092}
1093
1094/*
1095 * Structures and routines associated with inodedep caching.
1096 */
1097LIST_HEAD(inodedep_hashhead, inodedep) *inodedep_hashtbl;
1098static u_long	inodedep_hash;	/* size of hash table - 1 */
1099static long	num_inodedep;	/* number of inodedep allocated */
1100#define	INODEDEP_HASH(fs, inum) \
1101      (&inodedep_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & inodedep_hash])
1102
1103static int
1104inodedep_find(inodedephd, fs, inum, inodedeppp)
1105	struct inodedep_hashhead *inodedephd;
1106	struct fs *fs;
1107	ino_t inum;
1108	struct inodedep **inodedeppp;
1109{
1110	struct inodedep *inodedep;
1111
1112	LIST_FOREACH(inodedep, inodedephd, id_hash)
1113		if (inum == inodedep->id_ino && fs == inodedep->id_fs)
1114			break;
1115	if (inodedep) {
1116		*inodedeppp = inodedep;
1117		return (1);
1118	}
1119	*inodedeppp = NULL;
1120
1121	return (0);
1122}
1123/*
1124 * Look up an inodedep. Return 1 if found, 0 if not found.
1125 * If not found, allocate if DEPALLOC flag is passed.
1126 * Found or allocated entry is returned in inodedeppp.
1127 * This routine must be called with splbio interrupts blocked.
1128 */
1129static int
1130inodedep_lookup(fs, inum, flags, inodedeppp)
1131	struct fs *fs;
1132	ino_t inum;
1133	int flags;
1134	struct inodedep **inodedeppp;
1135{
1136	struct inodedep *inodedep;
1137	struct inodedep_hashhead *inodedephd;
1138
1139	mtx_assert(&lk, MA_OWNED);
1140	inodedephd = INODEDEP_HASH(fs, inum);
1141
1142	if (inodedep_find(inodedephd, fs, inum, inodedeppp))
1143		return (1);
1144	if ((flags & DEPALLOC) == 0)
1145		return (0);
1146	/*
1147	 * If we are over our limit, try to improve the situation.
1148	 */
1149	if (num_inodedep > max_softdeps  && (flags & NODELAY) == 0)
1150		request_cleanup(FLUSH_INODES);
1151	FREE_LOCK(&lk);
1152	MALLOC(inodedep, struct inodedep *, sizeof(struct inodedep),
1153		M_INODEDEP, M_SOFTDEP_FLAGS);
1154	ACQUIRE_LOCK(&lk);
1155	if (inodedep_find(inodedephd, fs, inum, inodedeppp)) {
1156		FREE(inodedep, M_INODEDEP);
1157		return (1);
1158	}
1159	num_inodedep += 1;
1160	inodedep->id_list.wk_type = D_INODEDEP;
1161	inodedep->id_fs = fs;
1162	inodedep->id_ino = inum;
1163	inodedep->id_state = ALLCOMPLETE;
1164	inodedep->id_nlinkdelta = 0;
1165	inodedep->id_savedino1 = NULL;
1166	inodedep->id_savedsize = -1;
1167	inodedep->id_savedextsize = -1;
1168	inodedep->id_buf = NULL;
1169	LIST_INIT(&inodedep->id_pendinghd);
1170	LIST_INIT(&inodedep->id_inowait);
1171	LIST_INIT(&inodedep->id_bufwait);
1172	TAILQ_INIT(&inodedep->id_inoupdt);
1173	TAILQ_INIT(&inodedep->id_newinoupdt);
1174	TAILQ_INIT(&inodedep->id_extupdt);
1175	TAILQ_INIT(&inodedep->id_newextupdt);
1176	LIST_INSERT_HEAD(inodedephd, inodedep, id_hash);
1177	*inodedeppp = inodedep;
1178	return (0);
1179}
1180
1181/*
1182 * Structures and routines associated with newblk caching.
1183 */
1184LIST_HEAD(newblk_hashhead, newblk) *newblk_hashtbl;
1185u_long	newblk_hash;		/* size of hash table - 1 */
1186#define	NEWBLK_HASH(fs, inum) \
1187	(&newblk_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & newblk_hash])
1188
1189static int
1190newblk_find(newblkhd, fs, newblkno, newblkpp)
1191	struct newblk_hashhead *newblkhd;
1192	struct fs *fs;
1193	ufs2_daddr_t newblkno;
1194	struct newblk **newblkpp;
1195{
1196	struct newblk *newblk;
1197
1198	LIST_FOREACH(newblk, newblkhd, nb_hash)
1199		if (newblkno == newblk->nb_newblkno && fs == newblk->nb_fs)
1200			break;
1201	if (newblk) {
1202		*newblkpp = newblk;
1203		return (1);
1204	}
1205	*newblkpp = NULL;
1206	return (0);
1207}
1208
1209/*
1210 * Look up a newblk. Return 1 if found, 0 if not found.
1211 * If not found, allocate if DEPALLOC flag is passed.
1212 * Found or allocated entry is returned in newblkpp.
1213 */
1214static int
1215newblk_lookup(fs, newblkno, flags, newblkpp)
1216	struct fs *fs;
1217	ufs2_daddr_t newblkno;
1218	int flags;
1219	struct newblk **newblkpp;
1220{
1221	struct newblk *newblk;
1222	struct newblk_hashhead *newblkhd;
1223
1224	newblkhd = NEWBLK_HASH(fs, newblkno);
1225	if (newblk_find(newblkhd, fs, newblkno, newblkpp))
1226		return (1);
1227	if ((flags & DEPALLOC) == 0)
1228		return (0);
1229	FREE_LOCK(&lk);
1230	MALLOC(newblk, struct newblk *, sizeof(struct newblk),
1231		M_NEWBLK, M_SOFTDEP_FLAGS);
1232	ACQUIRE_LOCK(&lk);
1233	if (newblk_find(newblkhd, fs, newblkno, newblkpp)) {
1234		FREE(newblk, M_NEWBLK);
1235		return (1);
1236	}
1237	newblk->nb_state = 0;
1238	newblk->nb_fs = fs;
1239	newblk->nb_newblkno = newblkno;
1240	LIST_INSERT_HEAD(newblkhd, newblk, nb_hash);
1241	*newblkpp = newblk;
1242	return (0);
1243}
1244
1245/*
1246 * Executed during filesystem system initialization before
1247 * mounting any filesystems.
1248 */
1249void
1250softdep_initialize()
1251{
1252
1253	LIST_INIT(&mkdirlisthd);
1254	LIST_INIT(&softdep_workitem_pending);
1255	max_softdeps = desiredvnodes * 4;
1256	pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP,
1257	    &pagedep_hash);
1258	inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, &inodedep_hash);
1259	newblk_hashtbl = hashinit(64, M_NEWBLK, &newblk_hash);
1260
1261	/* hooks through which the main kernel code calls us */
1262	softdep_process_worklist_hook = softdep_process_worklist;
1263
1264	/* initialise bioops hack */
1265	bioops.io_start = softdep_disk_io_initiation;
1266	bioops.io_complete = softdep_disk_write_complete;
1267	bioops.io_deallocate = softdep_deallocate_dependencies;
1268	bioops.io_countdeps = softdep_count_dependencies;
1269}
1270
1271/*
1272 * Executed after all filesystems have been unmounted during
1273 * filesystem module unload.
1274 */
1275void
1276softdep_uninitialize()
1277{
1278
1279	softdep_process_worklist_hook = NULL;
1280	hashdestroy(pagedep_hashtbl, M_PAGEDEP, pagedep_hash);
1281	hashdestroy(inodedep_hashtbl, M_INODEDEP, inodedep_hash);
1282	hashdestroy(newblk_hashtbl, M_NEWBLK, newblk_hash);
1283}
1284
1285/*
1286 * Called at mount time to notify the dependency code that a
1287 * filesystem wishes to use it.
1288 */
1289int
1290softdep_mount(devvp, mp, fs, cred)
1291	struct vnode *devvp;
1292	struct mount *mp;
1293	struct fs *fs;
1294	struct ucred *cred;
1295{
1296	struct csum_total cstotal;
1297	struct cg *cgp;
1298	struct buf *bp;
1299	int error, cyl;
1300
1301	mp->mnt_flag &= ~MNT_ASYNC;
1302	mp->mnt_flag |= MNT_SOFTDEP;
1303	/*
1304	 * When doing soft updates, the counters in the
1305	 * superblock may have gotten out of sync. Recomputation
1306	 * can take a long time and can be deferred for background
1307	 * fsck.  However, the old behavior of scanning the cylinder
1308	 * groups and recalculating them at mount time is available
1309	 * by setting vfs.ffs.compute_summary_at_mount to one.
1310	 */
1311	if (compute_summary_at_mount == 0 || fs->fs_clean != 0)
1312		return (0);
1313	bzero(&cstotal, sizeof cstotal);
1314	for (cyl = 0; cyl < fs->fs_ncg; cyl++) {
1315		if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)),
1316		    fs->fs_cgsize, cred, &bp)) != 0) {
1317			brelse(bp);
1318			return (error);
1319		}
1320		cgp = (struct cg *)bp->b_data;
1321		cstotal.cs_nffree += cgp->cg_cs.cs_nffree;
1322		cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree;
1323		cstotal.cs_nifree += cgp->cg_cs.cs_nifree;
1324		cstotal.cs_ndir += cgp->cg_cs.cs_ndir;
1325		fs->fs_cs(fs, cyl) = cgp->cg_cs;
1326		brelse(bp);
1327	}
1328#ifdef DEBUG
1329	if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal))
1330		printf("%s: superblock summary recomputed\n", fs->fs_fsmnt);
1331#endif
1332	bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal);
1333	return (0);
1334}
1335
1336/*
1337 * Protecting the freemaps (or bitmaps).
1338 *
1339 * To eliminate the need to execute fsck before mounting a filesystem
1340 * after a power failure, one must (conservatively) guarantee that the
1341 * on-disk copy of the bitmaps never indicate that a live inode or block is
1342 * free.  So, when a block or inode is allocated, the bitmap should be
1343 * updated (on disk) before any new pointers.  When a block or inode is
1344 * freed, the bitmap should not be updated until all pointers have been
1345 * reset.  The latter dependency is handled by the delayed de-allocation
1346 * approach described below for block and inode de-allocation.  The former
1347 * dependency is handled by calling the following procedure when a block or
1348 * inode is allocated. When an inode is allocated an "inodedep" is created
1349 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk.
1350 * Each "inodedep" is also inserted into the hash indexing structure so
1351 * that any additional link additions can be made dependent on the inode
1352 * allocation.
1353 *
1354 * The ufs filesystem maintains a number of free block counts (e.g., per
1355 * cylinder group, per cylinder and per <cylinder, rotational position> pair)
1356 * in addition to the bitmaps.  These counts are used to improve efficiency
1357 * during allocation and therefore must be consistent with the bitmaps.
1358 * There is no convenient way to guarantee post-crash consistency of these
1359 * counts with simple update ordering, for two main reasons: (1) The counts
1360 * and bitmaps for a single cylinder group block are not in the same disk
1361 * sector.  If a disk write is interrupted (e.g., by power failure), one may
1362 * be written and the other not.  (2) Some of the counts are located in the
1363 * superblock rather than the cylinder group block. So, we focus our soft
1364 * updates implementation on protecting the bitmaps. When mounting a
1365 * filesystem, we recompute the auxiliary counts from the bitmaps.
1366 */
1367
1368/*
1369 * Called just after updating the cylinder group block to allocate an inode.
1370 */
1371void
1372softdep_setup_inomapdep(bp, ip, newinum)
1373	struct buf *bp;		/* buffer for cylgroup block with inode map */
1374	struct inode *ip;	/* inode related to allocation */
1375	ino_t newinum;		/* new inode number being allocated */
1376{
1377	struct inodedep *inodedep;
1378	struct bmsafemap *bmsafemap;
1379
1380	/*
1381	 * Create a dependency for the newly allocated inode.
1382	 * Panic if it already exists as something is seriously wrong.
1383	 * Otherwise add it to the dependency list for the buffer holding
1384	 * the cylinder group map from which it was allocated.
1385	 */
1386	ACQUIRE_LOCK(&lk);
1387	if ((inodedep_lookup(ip->i_fs, newinum, DEPALLOC|NODELAY, &inodedep)))
1388		panic("softdep_setup_inomapdep: found inode");
1389	inodedep->id_buf = bp;
1390	inodedep->id_state &= ~DEPCOMPLETE;
1391	bmsafemap = bmsafemap_lookup(bp);
1392	LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps);
1393	FREE_LOCK(&lk);
1394}
1395
1396/*
1397 * Called just after updating the cylinder group block to
1398 * allocate block or fragment.
1399 */
1400void
1401softdep_setup_blkmapdep(bp, fs, newblkno)
1402	struct buf *bp;		/* buffer for cylgroup block with block map */
1403	struct fs *fs;		/* filesystem doing allocation */
1404	ufs2_daddr_t newblkno;	/* number of newly allocated block */
1405{
1406	struct newblk *newblk;
1407	struct bmsafemap *bmsafemap;
1408
1409	/*
1410	 * Create a dependency for the newly allocated block.
1411	 * Add it to the dependency list for the buffer holding
1412	 * the cylinder group map from which it was allocated.
1413	 */
1414	ACQUIRE_LOCK(&lk);
1415	if (newblk_lookup(fs, newblkno, DEPALLOC, &newblk) != 0)
1416		panic("softdep_setup_blkmapdep: found block");
1417	newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(bp);
1418	LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps);
1419	FREE_LOCK(&lk);
1420}
1421
1422/*
1423 * Find the bmsafemap associated with a cylinder group buffer.
1424 * If none exists, create one. The buffer must be locked when
1425 * this routine is called and this routine must be called with
1426 * splbio interrupts blocked.
1427 */
1428static struct bmsafemap *
1429bmsafemap_lookup(bp)
1430	struct buf *bp;
1431{
1432	struct bmsafemap *bmsafemap;
1433	struct worklist *wk;
1434
1435	mtx_assert(&lk, MA_OWNED);
1436	LIST_FOREACH(wk, &bp->b_dep, wk_list)
1437		if (wk->wk_type == D_BMSAFEMAP)
1438			return (WK_BMSAFEMAP(wk));
1439	FREE_LOCK(&lk);
1440	MALLOC(bmsafemap, struct bmsafemap *, sizeof(struct bmsafemap),
1441		M_BMSAFEMAP, M_SOFTDEP_FLAGS);
1442	bmsafemap->sm_list.wk_type = D_BMSAFEMAP;
1443	bmsafemap->sm_list.wk_state = 0;
1444	bmsafemap->sm_buf = bp;
1445	LIST_INIT(&bmsafemap->sm_allocdirecthd);
1446	LIST_INIT(&bmsafemap->sm_allocindirhd);
1447	LIST_INIT(&bmsafemap->sm_inodedephd);
1448	LIST_INIT(&bmsafemap->sm_newblkhd);
1449	ACQUIRE_LOCK(&lk);
1450	WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list);
1451	return (bmsafemap);
1452}
1453
1454/*
1455 * Direct block allocation dependencies.
1456 *
1457 * When a new block is allocated, the corresponding disk locations must be
1458 * initialized (with zeros or new data) before the on-disk inode points to
1459 * them.  Also, the freemap from which the block was allocated must be
1460 * updated (on disk) before the inode's pointer. These two dependencies are
1461 * independent of each other and are needed for all file blocks and indirect
1462 * blocks that are pointed to directly by the inode.  Just before the
1463 * "in-core" version of the inode is updated with a newly allocated block
1464 * number, a procedure (below) is called to setup allocation dependency
1465 * structures.  These structures are removed when the corresponding
1466 * dependencies are satisfied or when the block allocation becomes obsolete
1467 * (i.e., the file is deleted, the block is de-allocated, or the block is a
1468 * fragment that gets upgraded).  All of these cases are handled in
1469 * procedures described later.
1470 *
1471 * When a file extension causes a fragment to be upgraded, either to a larger
1472 * fragment or to a full block, the on-disk location may change (if the
1473 * previous fragment could not simply be extended). In this case, the old
1474 * fragment must be de-allocated, but not until after the inode's pointer has
1475 * been updated. In most cases, this is handled by later procedures, which
1476 * will construct a "freefrag" structure to be added to the workitem queue
1477 * when the inode update is complete (or obsolete).  The main exception to
1478 * this is when an allocation occurs while a pending allocation dependency
1479 * (for the same block pointer) remains.  This case is handled in the main
1480 * allocation dependency setup procedure by immediately freeing the
1481 * unreferenced fragments.
1482 */
1483void
1484softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
1485	struct inode *ip;	/* inode to which block is being added */
1486	ufs_lbn_t lbn;		/* block pointer within inode */
1487	ufs2_daddr_t newblkno;	/* disk block number being added */
1488	ufs2_daddr_t oldblkno;	/* previous block number, 0 unless frag */
1489	long newsize;		/* size of new block */
1490	long oldsize;		/* size of new block */
1491	struct buf *bp;		/* bp for allocated block */
1492{
1493	struct allocdirect *adp, *oldadp;
1494	struct allocdirectlst *adphead;
1495	struct bmsafemap *bmsafemap;
1496	struct inodedep *inodedep;
1497	struct pagedep *pagedep;
1498	struct newblk *newblk;
1499
1500	MALLOC(adp, struct allocdirect *, sizeof(struct allocdirect),
1501		M_ALLOCDIRECT, M_SOFTDEP_FLAGS|M_ZERO);
1502	adp->ad_list.wk_type = D_ALLOCDIRECT;
1503	adp->ad_lbn = lbn;
1504	adp->ad_newblkno = newblkno;
1505	adp->ad_oldblkno = oldblkno;
1506	adp->ad_newsize = newsize;
1507	adp->ad_oldsize = oldsize;
1508	adp->ad_state = ATTACHED;
1509	LIST_INIT(&adp->ad_newdirblk);
1510	if (newblkno == oldblkno)
1511		adp->ad_freefrag = NULL;
1512	else
1513		adp->ad_freefrag = newfreefrag(ip, oldblkno, oldsize);
1514
1515	ACQUIRE_LOCK(&lk);
1516	if (lbn >= NDADDR) {
1517		/* allocating an indirect block */
1518		if (oldblkno != 0)
1519			panic("softdep_setup_allocdirect: non-zero indir");
1520	} else {
1521		/*
1522		 * Allocating a direct block.
1523		 *
1524		 * If we are allocating a directory block, then we must
1525		 * allocate an associated pagedep to track additions and
1526		 * deletions.
1527		 */
1528		if ((ip->i_mode & IFMT) == IFDIR &&
1529		    pagedep_lookup(ip, lbn, DEPALLOC, &pagedep) == 0)
1530			WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list);
1531	}
1532	if (newblk_lookup(ip->i_fs, newblkno, 0, &newblk) == 0)
1533		panic("softdep_setup_allocdirect: lost block");
1534	if (newblk->nb_state == DEPCOMPLETE) {
1535		adp->ad_state |= DEPCOMPLETE;
1536		adp->ad_buf = NULL;
1537	} else {
1538		bmsafemap = newblk->nb_bmsafemap;
1539		adp->ad_buf = bmsafemap->sm_buf;
1540		LIST_REMOVE(newblk, nb_deps);
1541		LIST_INSERT_HEAD(&bmsafemap->sm_allocdirecthd, adp, ad_deps);
1542	}
1543	LIST_REMOVE(newblk, nb_hash);
1544	FREE(newblk, M_NEWBLK);
1545
1546	inodedep_lookup(ip->i_fs, ip->i_number, DEPALLOC | NODELAY, &inodedep);
1547	adp->ad_inodedep = inodedep;
1548	WORKLIST_INSERT(&bp->b_dep, &adp->ad_list);
1549	/*
1550	 * The list of allocdirects must be kept in sorted and ascending
1551	 * order so that the rollback routines can quickly determine the
1552	 * first uncommitted block (the size of the file stored on disk
1553	 * ends at the end of the lowest committed fragment, or if there
1554	 * are no fragments, at the end of the highest committed block).
1555	 * Since files generally grow, the typical case is that the new
1556	 * block is to be added at the end of the list. We speed this
1557	 * special case by checking against the last allocdirect in the
1558	 * list before laboriously traversing the list looking for the
1559	 * insertion point.
1560	 */
1561	adphead = &inodedep->id_newinoupdt;
1562	oldadp = TAILQ_LAST(adphead, allocdirectlst);
1563	if (oldadp == NULL || oldadp->ad_lbn <= lbn) {
1564		/* insert at end of list */
1565		TAILQ_INSERT_TAIL(adphead, adp, ad_next);
1566		if (oldadp != NULL && oldadp->ad_lbn == lbn)
1567			allocdirect_merge(adphead, adp, oldadp);
1568		FREE_LOCK(&lk);
1569		return;
1570	}
1571	TAILQ_FOREACH(oldadp, adphead, ad_next) {
1572		if (oldadp->ad_lbn >= lbn)
1573			break;
1574	}
1575	if (oldadp == NULL)
1576		panic("softdep_setup_allocdirect: lost entry");
1577	/* insert in middle of list */
1578	TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
1579	if (oldadp->ad_lbn == lbn)
1580		allocdirect_merge(adphead, adp, oldadp);
1581	FREE_LOCK(&lk);
1582}
1583
1584/*
1585 * Replace an old allocdirect dependency with a newer one.
1586 * This routine must be called with splbio interrupts blocked.
1587 */
1588static void
1589allocdirect_merge(adphead, newadp, oldadp)
1590	struct allocdirectlst *adphead;	/* head of list holding allocdirects */
1591	struct allocdirect *newadp;	/* allocdirect being added */
1592	struct allocdirect *oldadp;	/* existing allocdirect being checked */
1593{
1594	struct worklist *wk;
1595	struct freefrag *freefrag;
1596	struct newdirblk *newdirblk;
1597
1598	mtx_assert(&lk, MA_OWNED);
1599	if (newadp->ad_oldblkno != oldadp->ad_newblkno ||
1600	    newadp->ad_oldsize != oldadp->ad_newsize ||
1601	    newadp->ad_lbn >= NDADDR)
1602		panic("%s %jd != new %jd || old size %ld != new %ld",
1603		    "allocdirect_merge: old blkno",
1604		    (intmax_t)newadp->ad_oldblkno,
1605		    (intmax_t)oldadp->ad_newblkno,
1606		    newadp->ad_oldsize, oldadp->ad_newsize);
1607	newadp->ad_oldblkno = oldadp->ad_oldblkno;
1608	newadp->ad_oldsize = oldadp->ad_oldsize;
1609	/*
1610	 * If the old dependency had a fragment to free or had never
1611	 * previously had a block allocated, then the new dependency
1612	 * can immediately post its freefrag and adopt the old freefrag.
1613	 * This action is done by swapping the freefrag dependencies.
1614	 * The new dependency gains the old one's freefrag, and the
1615	 * old one gets the new one and then immediately puts it on
1616	 * the worklist when it is freed by free_allocdirect. It is
1617	 * not possible to do this swap when the old dependency had a
1618	 * non-zero size but no previous fragment to free. This condition
1619	 * arises when the new block is an extension of the old block.
1620	 * Here, the first part of the fragment allocated to the new
1621	 * dependency is part of the block currently claimed on disk by
1622	 * the old dependency, so cannot legitimately be freed until the
1623	 * conditions for the new dependency are fulfilled.
1624	 */
1625	if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) {
1626		freefrag = newadp->ad_freefrag;
1627		newadp->ad_freefrag = oldadp->ad_freefrag;
1628		oldadp->ad_freefrag = freefrag;
1629	}
1630	/*
1631	 * If we are tracking a new directory-block allocation,
1632	 * move it from the old allocdirect to the new allocdirect.
1633	 */
1634	if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) {
1635		newdirblk = WK_NEWDIRBLK(wk);
1636		WORKLIST_REMOVE(&newdirblk->db_list);
1637		if (LIST_FIRST(&oldadp->ad_newdirblk) != NULL)
1638			panic("allocdirect_merge: extra newdirblk");
1639		WORKLIST_INSERT(&newadp->ad_newdirblk, &newdirblk->db_list);
1640	}
1641	free_allocdirect(adphead, oldadp, 0);
1642}
1643
1644/*
1645 * Allocate a new freefrag structure if needed.
1646 */
1647static struct freefrag *
1648newfreefrag(ip, blkno, size)
1649	struct inode *ip;
1650	ufs2_daddr_t blkno;
1651	long size;
1652{
1653	struct freefrag *freefrag;
1654	struct fs *fs;
1655
1656	if (blkno == 0)
1657		return (NULL);
1658	fs = ip->i_fs;
1659	if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag)
1660		panic("newfreefrag: frag size");
1661	MALLOC(freefrag, struct freefrag *, sizeof(struct freefrag),
1662		M_FREEFRAG, M_SOFTDEP_FLAGS);
1663	freefrag->ff_list.wk_type = D_FREEFRAG;
1664	freefrag->ff_state = 0;
1665	freefrag->ff_inum = ip->i_number;
1666	freefrag->ff_mnt = ITOV(ip)->v_mount;
1667	freefrag->ff_blkno = blkno;
1668	freefrag->ff_fragsize = size;
1669	return (freefrag);
1670}
1671
1672/*
1673 * This workitem de-allocates fragments that were replaced during
1674 * file block allocation.
1675 */
1676static void
1677handle_workitem_freefrag(freefrag)
1678	struct freefrag *freefrag;
1679{
1680	struct ufsmount *ump = VFSTOUFS(freefrag->ff_mnt);
1681
1682	ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno,
1683	    freefrag->ff_fragsize, freefrag->ff_inum);
1684	FREE(freefrag, M_FREEFRAG);
1685}
1686
1687/*
1688 * Set up a dependency structure for an external attributes data block.
1689 * This routine follows much of the structure of softdep_setup_allocdirect.
1690 * See the description of softdep_setup_allocdirect above for details.
1691 */
1692void
1693softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
1694	struct inode *ip;
1695	ufs_lbn_t lbn;
1696	ufs2_daddr_t newblkno;
1697	ufs2_daddr_t oldblkno;
1698	long newsize;
1699	long oldsize;
1700	struct buf *bp;
1701{
1702	struct allocdirect *adp, *oldadp;
1703	struct allocdirectlst *adphead;
1704	struct bmsafemap *bmsafemap;
1705	struct inodedep *inodedep;
1706	struct newblk *newblk;
1707
1708	MALLOC(adp, struct allocdirect *, sizeof(struct allocdirect),
1709		M_ALLOCDIRECT, M_SOFTDEP_FLAGS|M_ZERO);
1710	adp->ad_list.wk_type = D_ALLOCDIRECT;
1711	adp->ad_lbn = lbn;
1712	adp->ad_newblkno = newblkno;
1713	adp->ad_oldblkno = oldblkno;
1714	adp->ad_newsize = newsize;
1715	adp->ad_oldsize = oldsize;
1716	adp->ad_state = ATTACHED | EXTDATA;
1717	LIST_INIT(&adp->ad_newdirblk);
1718	if (newblkno == oldblkno)
1719		adp->ad_freefrag = NULL;
1720	else
1721		adp->ad_freefrag = newfreefrag(ip, oldblkno, oldsize);
1722
1723	ACQUIRE_LOCK(&lk);
1724	if (newblk_lookup(ip->i_fs, newblkno, 0, &newblk) == 0)
1725		panic("softdep_setup_allocext: lost block");
1726
1727	inodedep_lookup(ip->i_fs, ip->i_number, DEPALLOC | NODELAY, &inodedep);
1728	adp->ad_inodedep = inodedep;
1729
1730	if (newblk->nb_state == DEPCOMPLETE) {
1731		adp->ad_state |= DEPCOMPLETE;
1732		adp->ad_buf = NULL;
1733	} else {
1734		bmsafemap = newblk->nb_bmsafemap;
1735		adp->ad_buf = bmsafemap->sm_buf;
1736		LIST_REMOVE(newblk, nb_deps);
1737		LIST_INSERT_HEAD(&bmsafemap->sm_allocdirecthd, adp, ad_deps);
1738	}
1739	LIST_REMOVE(newblk, nb_hash);
1740	FREE(newblk, M_NEWBLK);
1741
1742	WORKLIST_INSERT(&bp->b_dep, &adp->ad_list);
1743	if (lbn >= NXADDR)
1744		panic("softdep_setup_allocext: lbn %lld > NXADDR",
1745		    (long long)lbn);
1746	/*
1747	 * The list of allocdirects must be kept in sorted and ascending
1748	 * order so that the rollback routines can quickly determine the
1749	 * first uncommitted block (the size of the file stored on disk
1750	 * ends at the end of the lowest committed fragment, or if there
1751	 * are no fragments, at the end of the highest committed block).
1752	 * Since files generally grow, the typical case is that the new
1753	 * block is to be added at the end of the list. We speed this
1754	 * special case by checking against the last allocdirect in the
1755	 * list before laboriously traversing the list looking for the
1756	 * insertion point.
1757	 */
1758	adphead = &inodedep->id_newextupdt;
1759	oldadp = TAILQ_LAST(adphead, allocdirectlst);
1760	if (oldadp == NULL || oldadp->ad_lbn <= lbn) {
1761		/* insert at end of list */
1762		TAILQ_INSERT_TAIL(adphead, adp, ad_next);
1763		if (oldadp != NULL && oldadp->ad_lbn == lbn)
1764			allocdirect_merge(adphead, adp, oldadp);
1765		FREE_LOCK(&lk);
1766		return;
1767	}
1768	TAILQ_FOREACH(oldadp, adphead, ad_next) {
1769		if (oldadp->ad_lbn >= lbn)
1770			break;
1771	}
1772	if (oldadp == NULL)
1773		panic("softdep_setup_allocext: lost entry");
1774	/* insert in middle of list */
1775	TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
1776	if (oldadp->ad_lbn == lbn)
1777		allocdirect_merge(adphead, adp, oldadp);
1778	FREE_LOCK(&lk);
1779}
1780
1781/*
1782 * Indirect block allocation dependencies.
1783 *
1784 * The same dependencies that exist for a direct block also exist when
1785 * a new block is allocated and pointed to by an entry in a block of
1786 * indirect pointers. The undo/redo states described above are also
1787 * used here. Because an indirect block contains many pointers that
1788 * may have dependencies, a second copy of the entire in-memory indirect
1789 * block is kept. The buffer cache copy is always completely up-to-date.
1790 * The second copy, which is used only as a source for disk writes,
1791 * contains only the safe pointers (i.e., those that have no remaining
1792 * update dependencies). The second copy is freed when all pointers
1793 * are safe. The cache is not allowed to replace indirect blocks with
1794 * pending update dependencies. If a buffer containing an indirect
1795 * block with dependencies is written, these routines will mark it
1796 * dirty again. It can only be successfully written once all the
1797 * dependencies are removed. The ffs_fsync routine in conjunction with
1798 * softdep_sync_metadata work together to get all the dependencies
1799 * removed so that a file can be successfully written to disk. Three
1800 * procedures are used when setting up indirect block pointer
1801 * dependencies. The division is necessary because of the organization
1802 * of the "balloc" routine and because of the distinction between file
1803 * pages and file metadata blocks.
1804 */
1805
1806/*
1807 * Allocate a new allocindir structure.
1808 */
1809static struct allocindir *
1810newallocindir(ip, ptrno, newblkno, oldblkno)
1811	struct inode *ip;	/* inode for file being extended */
1812	int ptrno;		/* offset of pointer in indirect block */
1813	ufs2_daddr_t newblkno;	/* disk block number being added */
1814	ufs2_daddr_t oldblkno;	/* previous block number, 0 if none */
1815{
1816	struct allocindir *aip;
1817
1818	MALLOC(aip, struct allocindir *, sizeof(struct allocindir),
1819		M_ALLOCINDIR, M_SOFTDEP_FLAGS|M_ZERO);
1820	aip->ai_list.wk_type = D_ALLOCINDIR;
1821	aip->ai_state = ATTACHED;
1822	aip->ai_offset = ptrno;
1823	aip->ai_newblkno = newblkno;
1824	aip->ai_oldblkno = oldblkno;
1825	aip->ai_freefrag = newfreefrag(ip, oldblkno, ip->i_fs->fs_bsize);
1826	return (aip);
1827}
1828
1829/*
1830 * Called just before setting an indirect block pointer
1831 * to a newly allocated file page.
1832 */
1833void
1834softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
1835	struct inode *ip;	/* inode for file being extended */
1836	ufs_lbn_t lbn;		/* allocated block number within file */
1837	struct buf *bp;		/* buffer with indirect blk referencing page */
1838	int ptrno;		/* offset of pointer in indirect block */
1839	ufs2_daddr_t newblkno;	/* disk block number being added */
1840	ufs2_daddr_t oldblkno;	/* previous block number, 0 if none */
1841	struct buf *nbp;	/* buffer holding allocated page */
1842{
1843	struct allocindir *aip;
1844	struct pagedep *pagedep;
1845
1846	ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page");
1847	aip = newallocindir(ip, ptrno, newblkno, oldblkno);
1848	ACQUIRE_LOCK(&lk);
1849	/*
1850	 * If we are allocating a directory page, then we must
1851	 * allocate an associated pagedep to track additions and
1852	 * deletions.
1853	 */
1854	if ((ip->i_mode & IFMT) == IFDIR &&
1855	    pagedep_lookup(ip, lbn, DEPALLOC, &pagedep) == 0)
1856		WORKLIST_INSERT(&nbp->b_dep, &pagedep->pd_list);
1857	WORKLIST_INSERT(&nbp->b_dep, &aip->ai_list);
1858	setup_allocindir_phase2(bp, ip, aip);
1859	FREE_LOCK(&lk);
1860}
1861
1862/*
1863 * Called just before setting an indirect block pointer to a
1864 * newly allocated indirect block.
1865 */
1866void
1867softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
1868	struct buf *nbp;	/* newly allocated indirect block */
1869	struct inode *ip;	/* inode for file being extended */
1870	struct buf *bp;		/* indirect block referencing allocated block */
1871	int ptrno;		/* offset of pointer in indirect block */
1872	ufs2_daddr_t newblkno;	/* disk block number being added */
1873{
1874	struct allocindir *aip;
1875
1876	ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta");
1877	aip = newallocindir(ip, ptrno, newblkno, 0);
1878	ACQUIRE_LOCK(&lk);
1879	WORKLIST_INSERT(&nbp->b_dep, &aip->ai_list);
1880	setup_allocindir_phase2(bp, ip, aip);
1881	FREE_LOCK(&lk);
1882}
1883
1884/*
1885 * Called to finish the allocation of the "aip" allocated
1886 * by one of the two routines above.
1887 */
1888static void
1889setup_allocindir_phase2(bp, ip, aip)
1890	struct buf *bp;		/* in-memory copy of the indirect block */
1891	struct inode *ip;	/* inode for file being extended */
1892	struct allocindir *aip;	/* allocindir allocated by the above routines */
1893{
1894	struct worklist *wk;
1895	struct indirdep *indirdep, *newindirdep;
1896	struct bmsafemap *bmsafemap;
1897	struct allocindir *oldaip;
1898	struct freefrag *freefrag;
1899	struct newblk *newblk;
1900	ufs2_daddr_t blkno;
1901
1902	mtx_assert(&lk, MA_OWNED);
1903	if (bp->b_lblkno >= 0)
1904		panic("setup_allocindir_phase2: not indir blk");
1905	for (indirdep = NULL, newindirdep = NULL; ; ) {
1906		LIST_FOREACH(wk, &bp->b_dep, wk_list) {
1907			if (wk->wk_type != D_INDIRDEP)
1908				continue;
1909			indirdep = WK_INDIRDEP(wk);
1910			break;
1911		}
1912		if (indirdep == NULL && newindirdep) {
1913			indirdep = newindirdep;
1914			WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list);
1915			newindirdep = NULL;
1916		}
1917		if (indirdep) {
1918			if (newblk_lookup(ip->i_fs, aip->ai_newblkno, 0,
1919			    &newblk) == 0)
1920				panic("setup_allocindir: lost block");
1921			if (newblk->nb_state == DEPCOMPLETE) {
1922				aip->ai_state |= DEPCOMPLETE;
1923				aip->ai_buf = NULL;
1924			} else {
1925				bmsafemap = newblk->nb_bmsafemap;
1926				aip->ai_buf = bmsafemap->sm_buf;
1927				LIST_REMOVE(newblk, nb_deps);
1928				LIST_INSERT_HEAD(&bmsafemap->sm_allocindirhd,
1929				    aip, ai_deps);
1930			}
1931			LIST_REMOVE(newblk, nb_hash);
1932			FREE(newblk, M_NEWBLK);
1933			aip->ai_indirdep = indirdep;
1934			/*
1935			 * Check to see if there is an existing dependency
1936			 * for this block. If there is, merge the old
1937			 * dependency into the new one.
1938			 */
1939			if (aip->ai_oldblkno == 0)
1940				oldaip = NULL;
1941			else
1942
1943				LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, ai_next)
1944					if (oldaip->ai_offset == aip->ai_offset)
1945						break;
1946			freefrag = NULL;
1947			if (oldaip != NULL) {
1948				if (oldaip->ai_newblkno != aip->ai_oldblkno)
1949					panic("setup_allocindir_phase2: blkno");
1950				aip->ai_oldblkno = oldaip->ai_oldblkno;
1951				freefrag = aip->ai_freefrag;
1952				aip->ai_freefrag = oldaip->ai_freefrag;
1953				oldaip->ai_freefrag = NULL;
1954				free_allocindir(oldaip, NULL);
1955			}
1956			LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next);
1957			if (ip->i_ump->um_fstype == UFS1)
1958				((ufs1_daddr_t *)indirdep->ir_savebp->b_data)
1959				    [aip->ai_offset] = aip->ai_oldblkno;
1960			else
1961				((ufs2_daddr_t *)indirdep->ir_savebp->b_data)
1962				    [aip->ai_offset] = aip->ai_oldblkno;
1963			FREE_LOCK(&lk);
1964			if (freefrag != NULL)
1965				handle_workitem_freefrag(freefrag);
1966		} else
1967			FREE_LOCK(&lk);
1968		if (newindirdep) {
1969			newindirdep->ir_savebp->b_flags |= B_INVAL | B_NOCACHE;
1970			brelse(newindirdep->ir_savebp);
1971			WORKITEM_FREE((caddr_t)newindirdep, D_INDIRDEP);
1972		}
1973		if (indirdep) {
1974			ACQUIRE_LOCK(&lk);
1975			break;
1976		}
1977		MALLOC(newindirdep, struct indirdep *, sizeof(struct indirdep),
1978			M_INDIRDEP, M_SOFTDEP_FLAGS);
1979		newindirdep->ir_list.wk_type = D_INDIRDEP;
1980		newindirdep->ir_state = ATTACHED;
1981		if (ip->i_ump->um_fstype == UFS1)
1982			newindirdep->ir_state |= UFS1FMT;
1983		LIST_INIT(&newindirdep->ir_deplisthd);
1984		LIST_INIT(&newindirdep->ir_donehd);
1985		if (bp->b_blkno == bp->b_lblkno) {
1986			ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp,
1987			    NULL, NULL);
1988			bp->b_blkno = blkno;
1989		}
1990		newindirdep->ir_savebp =
1991		    getblk(ip->i_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0);
1992		BUF_KERNPROC(newindirdep->ir_savebp);
1993		bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount);
1994		ACQUIRE_LOCK(&lk);
1995	}
1996}
1997
1998/*
1999 * Block de-allocation dependencies.
2000 *
2001 * When blocks are de-allocated, the on-disk pointers must be nullified before
2002 * the blocks are made available for use by other files.  (The true
2003 * requirement is that old pointers must be nullified before new on-disk
2004 * pointers are set.  We chose this slightly more stringent requirement to
2005 * reduce complexity.) Our implementation handles this dependency by updating
2006 * the inode (or indirect block) appropriately but delaying the actual block
2007 * de-allocation (i.e., freemap and free space count manipulation) until
2008 * after the updated versions reach stable storage.  After the disk is
2009 * updated, the blocks can be safely de-allocated whenever it is convenient.
2010 * This implementation handles only the common case of reducing a file's
2011 * length to zero. Other cases are handled by the conventional synchronous
2012 * write approach.
2013 *
2014 * The ffs implementation with which we worked double-checks
2015 * the state of the block pointers and file size as it reduces
2016 * a file's length.  Some of this code is replicated here in our
2017 * soft updates implementation.  The freeblks->fb_chkcnt field is
2018 * used to transfer a part of this information to the procedure
2019 * that eventually de-allocates the blocks.
2020 *
2021 * This routine should be called from the routine that shortens
2022 * a file's length, before the inode's size or block pointers
2023 * are modified. It will save the block pointer information for
2024 * later release and zero the inode so that the calling routine
2025 * can release it.
2026 */
2027void
2028softdep_setup_freeblocks(ip, length, flags)
2029	struct inode *ip;	/* The inode whose length is to be reduced */
2030	off_t length;		/* The new length for the file */
2031	int flags;		/* IO_EXT and/or IO_NORMAL */
2032{
2033	struct freeblks *freeblks;
2034	struct inodedep *inodedep;
2035	struct allocdirect *adp;
2036	struct vnode *vp;
2037	struct buf *bp;
2038	struct fs *fs;
2039	ufs2_daddr_t extblocks, datablocks;
2040	int i, delay, error;
2041
2042	fs = ip->i_fs;
2043	if (length != 0)
2044		panic("softdep_setup_freeblocks: non-zero length");
2045	MALLOC(freeblks, struct freeblks *, sizeof(struct freeblks),
2046		M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO);
2047	freeblks->fb_list.wk_type = D_FREEBLKS;
2048	freeblks->fb_state = ATTACHED;
2049	freeblks->fb_uid = ip->i_uid;
2050	freeblks->fb_previousinum = ip->i_number;
2051	freeblks->fb_devvp = ip->i_devvp;
2052	freeblks->fb_mnt = ITOV(ip)->v_mount;
2053	extblocks = 0;
2054	if (fs->fs_magic == FS_UFS2_MAGIC)
2055		extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
2056	datablocks = DIP(ip, i_blocks) - extblocks;
2057	if ((flags & IO_NORMAL) == 0) {
2058		freeblks->fb_oldsize = 0;
2059		freeblks->fb_chkcnt = 0;
2060	} else {
2061		freeblks->fb_oldsize = ip->i_size;
2062		ip->i_size = 0;
2063		DIP_SET(ip, i_size, 0);
2064		freeblks->fb_chkcnt = datablocks;
2065		for (i = 0; i < NDADDR; i++) {
2066			freeblks->fb_dblks[i] = DIP(ip, i_db[i]);
2067			DIP_SET(ip, i_db[i], 0);
2068		}
2069		for (i = 0; i < NIADDR; i++) {
2070			freeblks->fb_iblks[i] = DIP(ip, i_ib[i]);
2071			DIP_SET(ip, i_ib[i], 0);
2072		}
2073		/*
2074		 * If the file was removed, then the space being freed was
2075		 * accounted for then (see softdep_filereleased()). If the
2076		 * file is merely being truncated, then we account for it now.
2077		 */
2078		if ((ip->i_flag & IN_SPACECOUNTED) == 0) {
2079			UFS_LOCK(ip->i_ump);
2080			fs->fs_pendingblocks += datablocks;
2081			UFS_UNLOCK(ip->i_ump);
2082		}
2083	}
2084	if ((flags & IO_EXT) == 0) {
2085		freeblks->fb_oldextsize = 0;
2086	} else {
2087		freeblks->fb_oldextsize = ip->i_din2->di_extsize;
2088		ip->i_din2->di_extsize = 0;
2089		freeblks->fb_chkcnt += extblocks;
2090		for (i = 0; i < NXADDR; i++) {
2091			freeblks->fb_eblks[i] = ip->i_din2->di_extb[i];
2092			ip->i_din2->di_extb[i] = 0;
2093		}
2094	}
2095	DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - freeblks->fb_chkcnt);
2096	/*
2097	 * Push the zero'ed inode to to its disk buffer so that we are free
2098	 * to delete its dependencies below. Once the dependencies are gone
2099	 * the buffer can be safely released.
2100	 */
2101	if ((error = bread(ip->i_devvp,
2102	    fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
2103	    (int)fs->fs_bsize, NOCRED, &bp)) != 0) {
2104		brelse(bp);
2105		softdep_error("softdep_setup_freeblocks", error);
2106	}
2107	if (ip->i_ump->um_fstype == UFS1)
2108		*((struct ufs1_dinode *)bp->b_data +
2109		    ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1;
2110	else
2111		*((struct ufs2_dinode *)bp->b_data +
2112		    ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2;
2113	/*
2114	 * Find and eliminate any inode dependencies.
2115	 */
2116	ACQUIRE_LOCK(&lk);
2117	(void) inodedep_lookup(fs, ip->i_number, DEPALLOC, &inodedep);
2118	if ((inodedep->id_state & IOSTARTED) != 0)
2119		panic("softdep_setup_freeblocks: inode busy");
2120	/*
2121	 * Add the freeblks structure to the list of operations that
2122	 * must await the zero'ed inode being written to disk. If we
2123	 * still have a bitmap dependency (delay == 0), then the inode
2124	 * has never been written to disk, so we can process the
2125	 * freeblks below once we have deleted the dependencies.
2126	 */
2127	delay = (inodedep->id_state & DEPCOMPLETE);
2128	if (delay)
2129		WORKLIST_INSERT(&inodedep->id_bufwait, &freeblks->fb_list);
2130	/*
2131	 * Because the file length has been truncated to zero, any
2132	 * pending block allocation dependency structures associated
2133	 * with this inode are obsolete and can simply be de-allocated.
2134	 * We must first merge the two dependency lists to get rid of
2135	 * any duplicate freefrag structures, then purge the merged list.
2136	 * If we still have a bitmap dependency, then the inode has never
2137	 * been written to disk, so we can free any fragments without delay.
2138	 */
2139	if (flags & IO_NORMAL) {
2140		merge_inode_lists(&inodedep->id_newinoupdt,
2141		    &inodedep->id_inoupdt);
2142		while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != 0)
2143			free_allocdirect(&inodedep->id_inoupdt, adp, delay);
2144	}
2145	if (flags & IO_EXT) {
2146		merge_inode_lists(&inodedep->id_newextupdt,
2147		    &inodedep->id_extupdt);
2148		while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != 0)
2149			free_allocdirect(&inodedep->id_extupdt, adp, delay);
2150	}
2151	FREE_LOCK(&lk);
2152	bdwrite(bp);
2153	/*
2154	 * We must wait for any I/O in progress to finish so that
2155	 * all potential buffers on the dirty list will be visible.
2156	 * Once they are all there, walk the list and get rid of
2157	 * any dependencies.
2158	 */
2159	vp = ITOV(ip);
2160	VI_LOCK(vp);
2161	drain_output(vp);
2162restart:
2163	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
2164		if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) ||
2165		    ((flags & IO_NORMAL) == 0 &&
2166		      (bp->b_xflags & BX_ALTDATA) == 0))
2167			continue;
2168		if ((bp = getdirtybuf(bp, VI_MTX(vp), MNT_WAIT)) == NULL)
2169			goto restart;
2170		VI_UNLOCK(vp);
2171		ACQUIRE_LOCK(&lk);
2172		(void) inodedep_lookup(fs, ip->i_number, 0, &inodedep);
2173		deallocate_dependencies(bp, inodedep);
2174		FREE_LOCK(&lk);
2175		bp->b_flags |= B_INVAL | B_NOCACHE;
2176		brelse(bp);
2177		VI_LOCK(vp);
2178		goto restart;
2179	}
2180	VI_UNLOCK(vp);
2181	ACQUIRE_LOCK(&lk);
2182	if (inodedep_lookup(fs, ip->i_number, 0, &inodedep) != 0)
2183		(void) free_inodedep(inodedep);
2184
2185	if(delay) {
2186		freeblks->fb_state |= DEPCOMPLETE;
2187		/*
2188		 * If the inode with zeroed block pointers is now on disk
2189		 * we can start freeing blocks. Add freeblks to the worklist
2190		 * instead of calling  handle_workitem_freeblocks directly as
2191		 * it is more likely that additional IO is needed to complete
2192		 * the request here than in the !delay case.
2193		 */
2194		if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE)
2195			add_to_worklist(&freeblks->fb_list);
2196	}
2197
2198	FREE_LOCK(&lk);
2199	/*
2200	 * If the inode has never been written to disk (delay == 0),
2201	 * then we can process the freeblks now that we have deleted
2202	 * the dependencies.
2203	 */
2204	if (!delay)
2205		handle_workitem_freeblocks(freeblks, 0);
2206}
2207
2208/*
2209 * Reclaim any dependency structures from a buffer that is about to
2210 * be reallocated to a new vnode. The buffer must be locked, thus,
2211 * no I/O completion operations can occur while we are manipulating
2212 * its associated dependencies. The mutex is held so that other I/O's
2213 * associated with related dependencies do not occur.
2214 */
2215static void
2216deallocate_dependencies(bp, inodedep)
2217	struct buf *bp;
2218	struct inodedep *inodedep;
2219{
2220	struct worklist *wk;
2221	struct indirdep *indirdep;
2222	struct allocindir *aip;
2223	struct pagedep *pagedep;
2224	struct dirrem *dirrem;
2225	struct diradd *dap;
2226	int i;
2227
2228	mtx_assert(&lk, MA_OWNED);
2229	while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) {
2230		switch (wk->wk_type) {
2231
2232		case D_INDIRDEP:
2233			indirdep = WK_INDIRDEP(wk);
2234			/*
2235			 * None of the indirect pointers will ever be visible,
2236			 * so they can simply be tossed. GOINGAWAY ensures
2237			 * that allocated pointers will be saved in the buffer
2238			 * cache until they are freed. Note that they will
2239			 * only be able to be found by their physical address
2240			 * since the inode mapping the logical address will
2241			 * be gone. The save buffer used for the safe copy
2242			 * was allocated in setup_allocindir_phase2 using
2243			 * the physical address so it could be used for this
2244			 * purpose. Hence we swap the safe copy with the real
2245			 * copy, allowing the safe copy to be freed and holding
2246			 * on to the real copy for later use in indir_trunc.
2247			 */
2248			if (indirdep->ir_state & GOINGAWAY)
2249				panic("deallocate_dependencies: already gone");
2250			indirdep->ir_state |= GOINGAWAY;
2251			VFSTOUFS(bp->b_vp->v_mount)->um_numindirdeps += 1;
2252			while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != 0)
2253				free_allocindir(aip, inodedep);
2254			if (bp->b_lblkno >= 0 ||
2255			    bp->b_blkno != indirdep->ir_savebp->b_lblkno)
2256				panic("deallocate_dependencies: not indir");
2257			bcopy(bp->b_data, indirdep->ir_savebp->b_data,
2258			    bp->b_bcount);
2259			WORKLIST_REMOVE(wk);
2260			WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, wk);
2261			continue;
2262
2263		case D_PAGEDEP:
2264			pagedep = WK_PAGEDEP(wk);
2265			/*
2266			 * None of the directory additions will ever be
2267			 * visible, so they can simply be tossed.
2268			 */
2269			for (i = 0; i < DAHASHSZ; i++)
2270				while ((dap =
2271				    LIST_FIRST(&pagedep->pd_diraddhd[i])))
2272					free_diradd(dap);
2273			while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != 0)
2274				free_diradd(dap);
2275			/*
2276			 * Copy any directory remove dependencies to the list
2277			 * to be processed after the zero'ed inode is written.
2278			 * If the inode has already been written, then they
2279			 * can be dumped directly onto the work list.
2280			 */
2281			LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) {
2282				LIST_REMOVE(dirrem, dm_next);
2283				dirrem->dm_dirinum = pagedep->pd_ino;
2284				if (inodedep == NULL ||
2285				    (inodedep->id_state & ALLCOMPLETE) ==
2286				     ALLCOMPLETE)
2287					add_to_worklist(&dirrem->dm_list);
2288				else
2289					WORKLIST_INSERT(&inodedep->id_bufwait,
2290					    &dirrem->dm_list);
2291			}
2292			if ((pagedep->pd_state & NEWBLOCK) != 0) {
2293				LIST_FOREACH(wk, &inodedep->id_bufwait, wk_list)
2294					if (wk->wk_type == D_NEWDIRBLK &&
2295					    WK_NEWDIRBLK(wk)->db_pagedep ==
2296					      pagedep)
2297						break;
2298				if (wk != NULL) {
2299					WORKLIST_REMOVE(wk);
2300					free_newdirblk(WK_NEWDIRBLK(wk));
2301				} else
2302					panic("deallocate_dependencies: "
2303					      "lost pagedep");
2304			}
2305			WORKLIST_REMOVE(&pagedep->pd_list);
2306			LIST_REMOVE(pagedep, pd_hash);
2307			WORKITEM_FREE(pagedep, D_PAGEDEP);
2308			continue;
2309
2310		case D_ALLOCINDIR:
2311			free_allocindir(WK_ALLOCINDIR(wk), inodedep);
2312			continue;
2313
2314		case D_ALLOCDIRECT:
2315		case D_INODEDEP:
2316			panic("deallocate_dependencies: Unexpected type %s",
2317			    TYPENAME(wk->wk_type));
2318			/* NOTREACHED */
2319
2320		default:
2321			panic("deallocate_dependencies: Unknown type %s",
2322			    TYPENAME(wk->wk_type));
2323			/* NOTREACHED */
2324		}
2325	}
2326}
2327
2328/*
2329 * Free an allocdirect. Generate a new freefrag work request if appropriate.
2330 * This routine must be called with splbio interrupts blocked.
2331 */
2332static void
2333free_allocdirect(adphead, adp, delay)
2334	struct allocdirectlst *adphead;
2335	struct allocdirect *adp;
2336	int delay;
2337{
2338	struct newdirblk *newdirblk;
2339	struct worklist *wk;
2340
2341	mtx_assert(&lk, MA_OWNED);
2342	if ((adp->ad_state & DEPCOMPLETE) == 0)
2343		LIST_REMOVE(adp, ad_deps);
2344	TAILQ_REMOVE(adphead, adp, ad_next);
2345	if ((adp->ad_state & COMPLETE) == 0)
2346		WORKLIST_REMOVE(&adp->ad_list);
2347	if (adp->ad_freefrag != NULL) {
2348		if (delay)
2349			WORKLIST_INSERT(&adp->ad_inodedep->id_bufwait,
2350			    &adp->ad_freefrag->ff_list);
2351		else
2352			add_to_worklist(&adp->ad_freefrag->ff_list);
2353	}
2354	if ((wk = LIST_FIRST(&adp->ad_newdirblk)) != NULL) {
2355		newdirblk = WK_NEWDIRBLK(wk);
2356		WORKLIST_REMOVE(&newdirblk->db_list);
2357		if (LIST_FIRST(&adp->ad_newdirblk) != NULL)
2358			panic("free_allocdirect: extra newdirblk");
2359		if (delay)
2360			WORKLIST_INSERT(&adp->ad_inodedep->id_bufwait,
2361			    &newdirblk->db_list);
2362		else
2363			free_newdirblk(newdirblk);
2364	}
2365	WORKITEM_FREE(adp, D_ALLOCDIRECT);
2366}
2367
2368/*
2369 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep.
2370 * This routine must be called with splbio interrupts blocked.
2371 */
2372static void
2373free_newdirblk(newdirblk)
2374	struct newdirblk *newdirblk;
2375{
2376	struct pagedep *pagedep;
2377	struct diradd *dap;
2378	int i;
2379
2380	mtx_assert(&lk, MA_OWNED);
2381	/*
2382	 * If the pagedep is still linked onto the directory buffer
2383	 * dependency chain, then some of the entries on the
2384	 * pd_pendinghd list may not be committed to disk yet. In
2385	 * this case, we will simply clear the NEWBLOCK flag and
2386	 * let the pd_pendinghd list be processed when the pagedep
2387	 * is next written. If the pagedep is no longer on the buffer
2388	 * dependency chain, then all the entries on the pd_pending
2389	 * list are committed to disk and we can free them here.
2390	 */
2391	pagedep = newdirblk->db_pagedep;
2392	pagedep->pd_state &= ~NEWBLOCK;
2393	if ((pagedep->pd_state & ONWORKLIST) == 0)
2394		while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
2395			free_diradd(dap);
2396	/*
2397	 * If no dependencies remain, the pagedep will be freed.
2398	 */
2399	for (i = 0; i < DAHASHSZ; i++)
2400		if (LIST_FIRST(&pagedep->pd_diraddhd[i]) != NULL)
2401			break;
2402	if (i == DAHASHSZ && (pagedep->pd_state & ONWORKLIST) == 0) {
2403		LIST_REMOVE(pagedep, pd_hash);
2404		WORKITEM_FREE(pagedep, D_PAGEDEP);
2405	}
2406	WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
2407}
2408
2409/*
2410 * Prepare an inode to be freed. The actual free operation is not
2411 * done until the zero'ed inode has been written to disk.
2412 */
2413void
2414softdep_freefile(pvp, ino, mode)
2415	struct vnode *pvp;
2416	ino_t ino;
2417	int mode;
2418{
2419	struct inode *ip = VTOI(pvp);
2420	struct inodedep *inodedep;
2421	struct freefile *freefile;
2422
2423	/*
2424	 * This sets up the inode de-allocation dependency.
2425	 */
2426	MALLOC(freefile, struct freefile *, sizeof(struct freefile),
2427		M_FREEFILE, M_SOFTDEP_FLAGS);
2428	freefile->fx_list.wk_type = D_FREEFILE;
2429	freefile->fx_list.wk_state = 0;
2430	freefile->fx_mode = mode;
2431	freefile->fx_oldinum = ino;
2432	freefile->fx_devvp = ip->i_devvp;
2433	freefile->fx_mnt = ITOV(ip)->v_mount;
2434	if ((ip->i_flag & IN_SPACECOUNTED) == 0) {
2435		UFS_LOCK(ip->i_ump);
2436		ip->i_fs->fs_pendinginodes += 1;
2437		UFS_UNLOCK(ip->i_ump);
2438	}
2439
2440	/*
2441	 * If the inodedep does not exist, then the zero'ed inode has
2442	 * been written to disk. If the allocated inode has never been
2443	 * written to disk, then the on-disk inode is zero'ed. In either
2444	 * case we can free the file immediately.
2445	 */
2446	ACQUIRE_LOCK(&lk);
2447	if (inodedep_lookup(ip->i_fs, ino, 0, &inodedep) == 0 ||
2448	    check_inode_unwritten(inodedep)) {
2449		FREE_LOCK(&lk);
2450		handle_workitem_freefile(freefile);
2451		return;
2452	}
2453	WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list);
2454	FREE_LOCK(&lk);
2455}
2456
2457/*
2458 * Check to see if an inode has never been written to disk. If
2459 * so free the inodedep and return success, otherwise return failure.
2460 * This routine must be called with splbio interrupts blocked.
2461 *
2462 * If we still have a bitmap dependency, then the inode has never
2463 * been written to disk. Drop the dependency as it is no longer
2464 * necessary since the inode is being deallocated. We set the
2465 * ALLCOMPLETE flags since the bitmap now properly shows that the
2466 * inode is not allocated. Even if the inode is actively being
2467 * written, it has been rolled back to its zero'ed state, so we
2468 * are ensured that a zero inode is what is on the disk. For short
2469 * lived files, this change will usually result in removing all the
2470 * dependencies from the inode so that it can be freed immediately.
2471 */
2472static int
2473check_inode_unwritten(inodedep)
2474	struct inodedep *inodedep;
2475{
2476
2477	mtx_assert(&lk, MA_OWNED);
2478	if ((inodedep->id_state & DEPCOMPLETE) != 0 ||
2479	    LIST_FIRST(&inodedep->id_pendinghd) != NULL ||
2480	    LIST_FIRST(&inodedep->id_bufwait) != NULL ||
2481	    LIST_FIRST(&inodedep->id_inowait) != NULL ||
2482	    TAILQ_FIRST(&inodedep->id_inoupdt) != NULL ||
2483	    TAILQ_FIRST(&inodedep->id_newinoupdt) != NULL ||
2484	    TAILQ_FIRST(&inodedep->id_extupdt) != NULL ||
2485	    TAILQ_FIRST(&inodedep->id_newextupdt) != NULL ||
2486	    inodedep->id_nlinkdelta != 0)
2487		return (0);
2488
2489	/*
2490	 * Another process might be in initiate_write_inodeblock_ufs[12]
2491	 * trying to allocate memory without holding "Softdep Lock".
2492	 */
2493	if ((inodedep->id_state & IOSTARTED) != 0 &&
2494	    inodedep->id_savedino1 == NULL)
2495		return (0);
2496
2497	inodedep->id_state |= ALLCOMPLETE;
2498	LIST_REMOVE(inodedep, id_deps);
2499	inodedep->id_buf = NULL;
2500	if (inodedep->id_state & ONWORKLIST)
2501		WORKLIST_REMOVE(&inodedep->id_list);
2502	if (inodedep->id_savedino1 != NULL) {
2503		FREE(inodedep->id_savedino1, M_SAVEDINO);
2504		inodedep->id_savedino1 = NULL;
2505	}
2506	if (free_inodedep(inodedep) == 0)
2507		panic("check_inode_unwritten: busy inode");
2508	return (1);
2509}
2510
2511/*
2512 * Try to free an inodedep structure. Return 1 if it could be freed.
2513 */
2514static int
2515free_inodedep(inodedep)
2516	struct inodedep *inodedep;
2517{
2518
2519	mtx_assert(&lk, MA_OWNED);
2520	if ((inodedep->id_state & ONWORKLIST) != 0 ||
2521	    (inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE ||
2522	    LIST_FIRST(&inodedep->id_pendinghd) != NULL ||
2523	    LIST_FIRST(&inodedep->id_bufwait) != NULL ||
2524	    LIST_FIRST(&inodedep->id_inowait) != NULL ||
2525	    TAILQ_FIRST(&inodedep->id_inoupdt) != NULL ||
2526	    TAILQ_FIRST(&inodedep->id_newinoupdt) != NULL ||
2527	    TAILQ_FIRST(&inodedep->id_extupdt) != NULL ||
2528	    TAILQ_FIRST(&inodedep->id_newextupdt) != NULL ||
2529	    inodedep->id_nlinkdelta != 0 || inodedep->id_savedino1 != NULL)
2530		return (0);
2531	LIST_REMOVE(inodedep, id_hash);
2532	WORKITEM_FREE(inodedep, D_INODEDEP);
2533	num_inodedep -= 1;
2534	return (1);
2535}
2536
2537/*
2538 * This workitem routine performs the block de-allocation.
2539 * The workitem is added to the pending list after the updated
2540 * inode block has been written to disk.  As mentioned above,
2541 * checks regarding the number of blocks de-allocated (compared
2542 * to the number of blocks allocated for the file) are also
2543 * performed in this function.
2544 */
2545static void
2546handle_workitem_freeblocks(freeblks, flags)
2547	struct freeblks *freeblks;
2548	int flags;
2549{
2550	struct inode *ip;
2551	struct vnode *vp;
2552	struct fs *fs;
2553	struct ufsmount *ump;
2554	int i, nblocks, level, bsize;
2555	ufs2_daddr_t bn, blocksreleased = 0;
2556	int error, allerror = 0;
2557	ufs_lbn_t baselbns[NIADDR], tmpval;
2558	int fs_pendingblocks;
2559
2560	ump = VFSTOUFS(freeblks->fb_mnt);
2561	fs = ump->um_fs;
2562	fs_pendingblocks = 0;
2563	tmpval = 1;
2564	baselbns[0] = NDADDR;
2565	for (i = 1; i < NIADDR; i++) {
2566		tmpval *= NINDIR(fs);
2567		baselbns[i] = baselbns[i - 1] + tmpval;
2568	}
2569	nblocks = btodb(fs->fs_bsize);
2570	blocksreleased = 0;
2571	/*
2572	 * Release all extended attribute blocks or frags.
2573	 */
2574	if (freeblks->fb_oldextsize > 0) {
2575		for (i = (NXADDR - 1); i >= 0; i--) {
2576			if ((bn = freeblks->fb_eblks[i]) == 0)
2577				continue;
2578			bsize = sblksize(fs, freeblks->fb_oldextsize, i);
2579			ffs_blkfree(ump, fs, freeblks->fb_devvp, bn, bsize,
2580			    freeblks->fb_previousinum);
2581			blocksreleased += btodb(bsize);
2582		}
2583	}
2584	/*
2585	 * Release all data blocks or frags.
2586	 */
2587	if (freeblks->fb_oldsize > 0) {
2588		/*
2589		 * Indirect blocks first.
2590		 */
2591		for (level = (NIADDR - 1); level >= 0; level--) {
2592			if ((bn = freeblks->fb_iblks[level]) == 0)
2593				continue;
2594			if ((error = indir_trunc(freeblks, fsbtodb(fs, bn),
2595			    level, baselbns[level], &blocksreleased)) == 0)
2596				allerror = error;
2597			ffs_blkfree(ump, fs, freeblks->fb_devvp, bn,
2598			    fs->fs_bsize, freeblks->fb_previousinum);
2599			fs_pendingblocks += nblocks;
2600			blocksreleased += nblocks;
2601		}
2602		/*
2603		 * All direct blocks or frags.
2604		 */
2605		for (i = (NDADDR - 1); i >= 0; i--) {
2606			if ((bn = freeblks->fb_dblks[i]) == 0)
2607				continue;
2608			bsize = sblksize(fs, freeblks->fb_oldsize, i);
2609			ffs_blkfree(ump, fs, freeblks->fb_devvp, bn, bsize,
2610			    freeblks->fb_previousinum);
2611			fs_pendingblocks += btodb(bsize);
2612			blocksreleased += btodb(bsize);
2613		}
2614	}
2615	UFS_LOCK(ump);
2616	fs->fs_pendingblocks -= fs_pendingblocks;
2617	UFS_UNLOCK(ump);
2618	/*
2619	 * If we still have not finished background cleanup, then check
2620	 * to see if the block count needs to be adjusted.
2621	 */
2622	if (freeblks->fb_chkcnt != blocksreleased &&
2623	    (fs->fs_flags & FS_UNCLEAN) != 0 &&
2624	    ffs_vget(freeblks->fb_mnt, freeblks->fb_previousinum,
2625	    (flags & LK_NOWAIT) | LK_EXCLUSIVE, &vp) == 0) {
2626		ip = VTOI(vp);
2627		DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + \
2628		    freeblks->fb_chkcnt - blocksreleased);
2629		ip->i_flag |= IN_CHANGE;
2630		vput(vp);
2631	}
2632
2633#ifdef DIAGNOSTIC
2634	if (freeblks->fb_chkcnt != blocksreleased &&
2635	    ((fs->fs_flags & FS_UNCLEAN) == 0 || (flags & LK_NOWAIT) != 0))
2636		printf("handle_workitem_freeblocks: block count\n");
2637	if (allerror)
2638		softdep_error("handle_workitem_freeblks", allerror);
2639#endif /* DIAGNOSTIC */
2640
2641	WORKITEM_FREE(freeblks, D_FREEBLKS);
2642}
2643
2644/*
2645 * Release blocks associated with the inode ip and stored in the indirect
2646 * block dbn. If level is greater than SINGLE, the block is an indirect block
2647 * and recursive calls to indirtrunc must be used to cleanse other indirect
2648 * blocks.
2649 */
2650static int
2651indir_trunc(freeblks, dbn, level, lbn, countp)
2652	struct freeblks *freeblks;
2653	ufs2_daddr_t dbn;
2654	int level;
2655	ufs_lbn_t lbn;
2656	ufs2_daddr_t *countp;
2657{
2658	struct buf *bp;
2659	struct fs *fs;
2660	struct worklist *wk;
2661	struct indirdep *indirdep;
2662	struct ufsmount *ump;
2663	ufs1_daddr_t *bap1 = 0;
2664	ufs2_daddr_t nb, *bap2 = 0;
2665	ufs_lbn_t lbnadd;
2666	int i, nblocks, ufs1fmt;
2667	int error, allerror = 0;
2668	int fs_pendingblocks;
2669
2670	ump = VFSTOUFS(freeblks->fb_mnt);
2671	fs = ump->um_fs;
2672	fs_pendingblocks = 0;
2673	lbnadd = 1;
2674	for (i = level; i > 0; i--)
2675		lbnadd *= NINDIR(fs);
2676	/*
2677	 * Get buffer of block pointers to be freed. This routine is not
2678	 * called until the zero'ed inode has been written, so it is safe
2679	 * to free blocks as they are encountered. Because the inode has
2680	 * been zero'ed, calls to bmap on these blocks will fail. So, we
2681	 * have to use the on-disk address and the block device for the
2682	 * filesystem to look them up. If the file was deleted before its
2683	 * indirect blocks were all written to disk, the routine that set
2684	 * us up (deallocate_dependencies) will have arranged to leave
2685	 * a complete copy of the indirect block in memory for our use.
2686	 * Otherwise we have to read the blocks in from the disk.
2687	 */
2688#ifdef notyet
2689	bp = getblk(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 0, 0,
2690	    GB_NOCREAT);
2691#else
2692	bp = incore(&freeblks->fb_devvp->v_bufobj, dbn);
2693#endif
2694	ACQUIRE_LOCK(&lk);
2695	if (bp != NULL && (wk = LIST_FIRST(&bp->b_dep)) != NULL) {
2696		if (wk->wk_type != D_INDIRDEP ||
2697		    (indirdep = WK_INDIRDEP(wk))->ir_savebp != bp ||
2698		    (indirdep->ir_state & GOINGAWAY) == 0)
2699			panic("indir_trunc: lost indirdep");
2700		WORKLIST_REMOVE(wk);
2701		WORKITEM_FREE(indirdep, D_INDIRDEP);
2702		if (LIST_FIRST(&bp->b_dep) != NULL)
2703			panic("indir_trunc: dangling dep");
2704		VFSTOUFS(freeblks->fb_mnt)->um_numindirdeps -= 1;
2705		FREE_LOCK(&lk);
2706	} else {
2707#ifdef notyet
2708		if (bp)
2709			brelse(bp);
2710#endif
2711		FREE_LOCK(&lk);
2712		error = bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize,
2713		    NOCRED, &bp);
2714		if (error) {
2715			brelse(bp);
2716			return (error);
2717		}
2718	}
2719	/*
2720	 * Recursively free indirect blocks.
2721	 */
2722	if (VFSTOUFS(freeblks->fb_mnt)->um_fstype == UFS1) {
2723		ufs1fmt = 1;
2724		bap1 = (ufs1_daddr_t *)bp->b_data;
2725	} else {
2726		ufs1fmt = 0;
2727		bap2 = (ufs2_daddr_t *)bp->b_data;
2728	}
2729	nblocks = btodb(fs->fs_bsize);
2730	for (i = NINDIR(fs) - 1; i >= 0; i--) {
2731		if (ufs1fmt)
2732			nb = bap1[i];
2733		else
2734			nb = bap2[i];
2735		if (nb == 0)
2736			continue;
2737		if (level != 0) {
2738			if ((error = indir_trunc(freeblks, fsbtodb(fs, nb),
2739			     level - 1, lbn + (i * lbnadd), countp)) != 0)
2740				allerror = error;
2741		}
2742		ffs_blkfree(ump, fs, freeblks->fb_devvp, nb, fs->fs_bsize,
2743		    freeblks->fb_previousinum);
2744		fs_pendingblocks += nblocks;
2745		*countp += nblocks;
2746	}
2747	UFS_LOCK(ump);
2748	fs->fs_pendingblocks -= fs_pendingblocks;
2749	UFS_UNLOCK(ump);
2750	bp->b_flags |= B_INVAL | B_NOCACHE;
2751	brelse(bp);
2752	return (allerror);
2753}
2754
2755/*
2756 * Free an allocindir.
2757 * This routine must be called with splbio interrupts blocked.
2758 */
2759static void
2760free_allocindir(aip, inodedep)
2761	struct allocindir *aip;
2762	struct inodedep *inodedep;
2763{
2764	struct freefrag *freefrag;
2765
2766	mtx_assert(&lk, MA_OWNED);
2767	if ((aip->ai_state & DEPCOMPLETE) == 0)
2768		LIST_REMOVE(aip, ai_deps);
2769	if (aip->ai_state & ONWORKLIST)
2770		WORKLIST_REMOVE(&aip->ai_list);
2771	LIST_REMOVE(aip, ai_next);
2772	if ((freefrag = aip->ai_freefrag) != NULL) {
2773		if (inodedep == NULL)
2774			add_to_worklist(&freefrag->ff_list);
2775		else
2776			WORKLIST_INSERT(&inodedep->id_bufwait,
2777			    &freefrag->ff_list);
2778	}
2779	WORKITEM_FREE(aip, D_ALLOCINDIR);
2780}
2781
2782/*
2783 * Directory entry addition dependencies.
2784 *
2785 * When adding a new directory entry, the inode (with its incremented link
2786 * count) must be written to disk before the directory entry's pointer to it.
2787 * Also, if the inode is newly allocated, the corresponding freemap must be
2788 * updated (on disk) before the directory entry's pointer. These requirements
2789 * are met via undo/redo on the directory entry's pointer, which consists
2790 * simply of the inode number.
2791 *
2792 * As directory entries are added and deleted, the free space within a
2793 * directory block can become fragmented.  The ufs filesystem will compact
2794 * a fragmented directory block to make space for a new entry. When this
2795 * occurs, the offsets of previously added entries change. Any "diradd"
2796 * dependency structures corresponding to these entries must be updated with
2797 * the new offsets.
2798 */
2799
2800/*
2801 * This routine is called after the in-memory inode's link
2802 * count has been incremented, but before the directory entry's
2803 * pointer to the inode has been set.
2804 */
2805int
2806softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
2807	struct buf *bp;		/* buffer containing directory block */
2808	struct inode *dp;	/* inode for directory */
2809	off_t diroffset;	/* offset of new entry in directory */
2810	ino_t newinum;		/* inode referenced by new directory entry */
2811	struct buf *newdirbp;	/* non-NULL => contents of new mkdir */
2812	int isnewblk;		/* entry is in a newly allocated block */
2813{
2814	int offset;		/* offset of new entry within directory block */
2815	ufs_lbn_t lbn;		/* block in directory containing new entry */
2816	struct fs *fs;
2817	struct diradd *dap;
2818	struct allocdirect *adp;
2819	struct pagedep *pagedep;
2820	struct inodedep *inodedep;
2821	struct newdirblk *newdirblk = 0;
2822	struct mkdir *mkdir1, *mkdir2;
2823
2824	/*
2825	 * Whiteouts have no dependencies.
2826	 */
2827	if (newinum == WINO) {
2828		if (newdirbp != NULL)
2829			bdwrite(newdirbp);
2830		return (0);
2831	}
2832
2833	fs = dp->i_fs;
2834	lbn = lblkno(fs, diroffset);
2835	offset = blkoff(fs, diroffset);
2836	MALLOC(dap, struct diradd *, sizeof(struct diradd), M_DIRADD,
2837		M_SOFTDEP_FLAGS|M_ZERO);
2838	dap->da_list.wk_type = D_DIRADD;
2839	dap->da_offset = offset;
2840	dap->da_newinum = newinum;
2841	dap->da_state = ATTACHED;
2842	if (isnewblk && lbn < NDADDR && fragoff(fs, diroffset) == 0) {
2843		MALLOC(newdirblk, struct newdirblk *, sizeof(struct newdirblk),
2844		    M_NEWDIRBLK, M_SOFTDEP_FLAGS);
2845		newdirblk->db_list.wk_type = D_NEWDIRBLK;
2846		newdirblk->db_state = 0;
2847	}
2848	if (newdirbp == NULL) {
2849		dap->da_state |= DEPCOMPLETE;
2850		ACQUIRE_LOCK(&lk);
2851	} else {
2852		dap->da_state |= MKDIR_BODY | MKDIR_PARENT;
2853		MALLOC(mkdir1, struct mkdir *, sizeof(struct mkdir), M_MKDIR,
2854		    M_SOFTDEP_FLAGS);
2855		mkdir1->md_list.wk_type = D_MKDIR;
2856		mkdir1->md_state = MKDIR_BODY;
2857		mkdir1->md_diradd = dap;
2858		MALLOC(mkdir2, struct mkdir *, sizeof(struct mkdir), M_MKDIR,
2859		    M_SOFTDEP_FLAGS);
2860		mkdir2->md_list.wk_type = D_MKDIR;
2861		mkdir2->md_state = MKDIR_PARENT;
2862		mkdir2->md_diradd = dap;
2863		/*
2864		 * Dependency on "." and ".." being written to disk.
2865		 */
2866		mkdir1->md_buf = newdirbp;
2867		ACQUIRE_LOCK(&lk);
2868		LIST_INSERT_HEAD(&mkdirlisthd, mkdir1, md_mkdirs);
2869		WORKLIST_INSERT(&newdirbp->b_dep, &mkdir1->md_list);
2870		FREE_LOCK(&lk);
2871		bdwrite(newdirbp);
2872		/*
2873		 * Dependency on link count increase for parent directory
2874		 */
2875		ACQUIRE_LOCK(&lk);
2876		if (inodedep_lookup(fs, dp->i_number, 0, &inodedep) == 0
2877		    || (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
2878			dap->da_state &= ~MKDIR_PARENT;
2879			WORKITEM_FREE(mkdir2, D_MKDIR);
2880		} else {
2881			LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs);
2882			WORKLIST_INSERT(&inodedep->id_bufwait,&mkdir2->md_list);
2883		}
2884	}
2885	/*
2886	 * Link into parent directory pagedep to await its being written.
2887	 */
2888	if (pagedep_lookup(dp, lbn, DEPALLOC, &pagedep) == 0)
2889		WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list);
2890	dap->da_pagedep = pagedep;
2891	LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap,
2892	    da_pdlist);
2893	/*
2894	 * Link into its inodedep. Put it on the id_bufwait list if the inode
2895	 * is not yet written. If it is written, do the post-inode write
2896	 * processing to put it on the id_pendinghd list.
2897	 */
2898	(void) inodedep_lookup(fs, newinum, DEPALLOC, &inodedep);
2899	if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE)
2900		diradd_inode_written(dap, inodedep);
2901	else
2902		WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
2903	if (isnewblk) {
2904		/*
2905		 * Directories growing into indirect blocks are rare
2906		 * enough and the frequency of new block allocation
2907		 * in those cases even more rare, that we choose not
2908		 * to bother tracking them. Rather we simply force the
2909		 * new directory entry to disk.
2910		 */
2911		if (lbn >= NDADDR) {
2912			FREE_LOCK(&lk);
2913			/*
2914			 * We only have a new allocation when at the
2915			 * beginning of a new block, not when we are
2916			 * expanding into an existing block.
2917			 */
2918			if (blkoff(fs, diroffset) == 0)
2919				return (1);
2920			return (0);
2921		}
2922		/*
2923		 * We only have a new allocation when at the beginning
2924		 * of a new fragment, not when we are expanding into an
2925		 * existing fragment. Also, there is nothing to do if we
2926		 * are already tracking this block.
2927		 */
2928		if (fragoff(fs, diroffset) != 0) {
2929			FREE_LOCK(&lk);
2930			return (0);
2931		}
2932		if ((pagedep->pd_state & NEWBLOCK) != 0) {
2933			FREE_LOCK(&lk);
2934			WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
2935			return (0);
2936		}
2937		/*
2938		 * Find our associated allocdirect and have it track us.
2939		 */
2940		if (inodedep_lookup(fs, dp->i_number, 0, &inodedep) == 0)
2941			panic("softdep_setup_directory_add: lost inodedep");
2942		adp = TAILQ_LAST(&inodedep->id_newinoupdt, allocdirectlst);
2943		if (adp == NULL || adp->ad_lbn != lbn)
2944			panic("softdep_setup_directory_add: lost entry");
2945		pagedep->pd_state |= NEWBLOCK;
2946		newdirblk->db_pagedep = pagedep;
2947		WORKLIST_INSERT(&adp->ad_newdirblk, &newdirblk->db_list);
2948	}
2949	FREE_LOCK(&lk);
2950	return (0);
2951}
2952
2953/*
2954 * This procedure is called to change the offset of a directory
2955 * entry when compacting a directory block which must be owned
2956 * exclusively by the caller. Note that the actual entry movement
2957 * must be done in this procedure to ensure that no I/O completions
2958 * occur while the move is in progress.
2959 */
2960void
2961softdep_change_directoryentry_offset(dp, base, oldloc, newloc, entrysize)
2962	struct inode *dp;	/* inode for directory */
2963	caddr_t base;		/* address of dp->i_offset */
2964	caddr_t oldloc;		/* address of old directory location */
2965	caddr_t newloc;		/* address of new directory location */
2966	int entrysize;		/* size of directory entry */
2967{
2968	int offset, oldoffset, newoffset;
2969	struct pagedep *pagedep;
2970	struct diradd *dap;
2971	ufs_lbn_t lbn;
2972
2973	ACQUIRE_LOCK(&lk);
2974	lbn = lblkno(dp->i_fs, dp->i_offset);
2975	offset = blkoff(dp->i_fs, dp->i_offset);
2976	if (pagedep_lookup(dp, lbn, 0, &pagedep) == 0)
2977		goto done;
2978	oldoffset = offset + (oldloc - base);
2979	newoffset = offset + (newloc - base);
2980
2981	LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(oldoffset)], da_pdlist) {
2982		if (dap->da_offset != oldoffset)
2983			continue;
2984		dap->da_offset = newoffset;
2985		if (DIRADDHASH(newoffset) == DIRADDHASH(oldoffset))
2986			break;
2987		LIST_REMOVE(dap, da_pdlist);
2988		LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(newoffset)],
2989		    dap, da_pdlist);
2990		break;
2991	}
2992	if (dap == NULL) {
2993
2994		LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) {
2995			if (dap->da_offset == oldoffset) {
2996				dap->da_offset = newoffset;
2997				break;
2998			}
2999		}
3000	}
3001done:
3002	bcopy(oldloc, newloc, entrysize);
3003	FREE_LOCK(&lk);
3004}
3005
3006/*
3007 * Free a diradd dependency structure. This routine must be called
3008 * with splbio interrupts blocked.
3009 */
3010static void
3011free_diradd(dap)
3012	struct diradd *dap;
3013{
3014	struct dirrem *dirrem;
3015	struct pagedep *pagedep;
3016	struct inodedep *inodedep;
3017	struct mkdir *mkdir, *nextmd;
3018
3019	mtx_assert(&lk, MA_OWNED);
3020	WORKLIST_REMOVE(&dap->da_list);
3021	LIST_REMOVE(dap, da_pdlist);
3022	if ((dap->da_state & DIRCHG) == 0) {
3023		pagedep = dap->da_pagedep;
3024	} else {
3025		dirrem = dap->da_previous;
3026		pagedep = dirrem->dm_pagedep;
3027		dirrem->dm_dirinum = pagedep->pd_ino;
3028		add_to_worklist(&dirrem->dm_list);
3029	}
3030	if (inodedep_lookup(VFSTOUFS(pagedep->pd_mnt)->um_fs, dap->da_newinum,
3031	    0, &inodedep) != 0)
3032		(void) free_inodedep(inodedep);
3033	if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
3034		for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) {
3035			nextmd = LIST_NEXT(mkdir, md_mkdirs);
3036			if (mkdir->md_diradd != dap)
3037				continue;
3038			dap->da_state &= ~mkdir->md_state;
3039			WORKLIST_REMOVE(&mkdir->md_list);
3040			LIST_REMOVE(mkdir, md_mkdirs);
3041			WORKITEM_FREE(mkdir, D_MKDIR);
3042		}
3043		if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0)
3044			panic("free_diradd: unfound ref");
3045	}
3046	WORKITEM_FREE(dap, D_DIRADD);
3047}
3048
3049/*
3050 * Directory entry removal dependencies.
3051 *
3052 * When removing a directory entry, the entry's inode pointer must be
3053 * zero'ed on disk before the corresponding inode's link count is decremented
3054 * (possibly freeing the inode for re-use). This dependency is handled by
3055 * updating the directory entry but delaying the inode count reduction until
3056 * after the directory block has been written to disk. After this point, the
3057 * inode count can be decremented whenever it is convenient.
3058 */
3059
3060/*
3061 * This routine should be called immediately after removing
3062 * a directory entry.  The inode's link count should not be
3063 * decremented by the calling procedure -- the soft updates
3064 * code will do this task when it is safe.
3065 */
3066void
3067softdep_setup_remove(bp, dp, ip, isrmdir)
3068	struct buf *bp;		/* buffer containing directory block */
3069	struct inode *dp;	/* inode for the directory being modified */
3070	struct inode *ip;	/* inode for directory entry being removed */
3071	int isrmdir;		/* indicates if doing RMDIR */
3072{
3073	struct dirrem *dirrem, *prevdirrem;
3074
3075	/*
3076	 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK.
3077	 */
3078	dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
3079
3080	/*
3081	 * If the COMPLETE flag is clear, then there were no active
3082	 * entries and we want to roll back to a zeroed entry until
3083	 * the new inode is committed to disk. If the COMPLETE flag is
3084	 * set then we have deleted an entry that never made it to
3085	 * disk. If the entry we deleted resulted from a name change,
3086	 * then the old name still resides on disk. We cannot delete
3087	 * its inode (returned to us in prevdirrem) until the zeroed
3088	 * directory entry gets to disk. The new inode has never been
3089	 * referenced on the disk, so can be deleted immediately.
3090	 */
3091	if ((dirrem->dm_state & COMPLETE) == 0) {
3092		LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem,
3093		    dm_next);
3094		FREE_LOCK(&lk);
3095	} else {
3096		if (prevdirrem != NULL)
3097			LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd,
3098			    prevdirrem, dm_next);
3099		dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino;
3100		FREE_LOCK(&lk);
3101		handle_workitem_remove(dirrem, NULL);
3102	}
3103}
3104
3105/*
3106 * Allocate a new dirrem if appropriate and return it along with
3107 * its associated pagedep. Called without a lock, returns with lock.
3108 */
3109static long num_dirrem;		/* number of dirrem allocated */
3110static struct dirrem *
3111newdirrem(bp, dp, ip, isrmdir, prevdirremp)
3112	struct buf *bp;		/* buffer containing directory block */
3113	struct inode *dp;	/* inode for the directory being modified */
3114	struct inode *ip;	/* inode for directory entry being removed */
3115	int isrmdir;		/* indicates if doing RMDIR */
3116	struct dirrem **prevdirremp; /* previously referenced inode, if any */
3117{
3118	int offset;
3119	ufs_lbn_t lbn;
3120	struct diradd *dap;
3121	struct dirrem *dirrem;
3122	struct pagedep *pagedep;
3123
3124	/*
3125	 * Whiteouts have no deletion dependencies.
3126	 */
3127	if (ip == NULL)
3128		panic("newdirrem: whiteout");
3129	/*
3130	 * If we are over our limit, try to improve the situation.
3131	 * Limiting the number of dirrem structures will also limit
3132	 * the number of freefile and freeblks structures.
3133	 */
3134	ACQUIRE_LOCK(&lk);
3135	if (num_dirrem > max_softdeps / 2)
3136		(void) request_cleanup(FLUSH_REMOVE);
3137	num_dirrem += 1;
3138	FREE_LOCK(&lk);
3139	MALLOC(dirrem, struct dirrem *, sizeof(struct dirrem),
3140		M_DIRREM, M_SOFTDEP_FLAGS|M_ZERO);
3141	dirrem->dm_list.wk_type = D_DIRREM;
3142	dirrem->dm_state = isrmdir ? RMDIR : 0;
3143	dirrem->dm_mnt = ITOV(ip)->v_mount;
3144	dirrem->dm_oldinum = ip->i_number;
3145	*prevdirremp = NULL;
3146
3147	ACQUIRE_LOCK(&lk);
3148	lbn = lblkno(dp->i_fs, dp->i_offset);
3149	offset = blkoff(dp->i_fs, dp->i_offset);
3150	if (pagedep_lookup(dp, lbn, DEPALLOC, &pagedep) == 0)
3151		WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list);
3152	dirrem->dm_pagedep = pagedep;
3153	/*
3154	 * Check for a diradd dependency for the same directory entry.
3155	 * If present, then both dependencies become obsolete and can
3156	 * be de-allocated. Check for an entry on both the pd_dirraddhd
3157	 * list and the pd_pendinghd list.
3158	 */
3159
3160	LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist)
3161		if (dap->da_offset == offset)
3162			break;
3163	if (dap == NULL) {
3164
3165		LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist)
3166			if (dap->da_offset == offset)
3167				break;
3168		if (dap == NULL)
3169			return (dirrem);
3170	}
3171	/*
3172	 * Must be ATTACHED at this point.
3173	 */
3174	if ((dap->da_state & ATTACHED) == 0)
3175		panic("newdirrem: not ATTACHED");
3176	if (dap->da_newinum != ip->i_number)
3177		panic("newdirrem: inum %d should be %d",
3178		    ip->i_number, dap->da_newinum);
3179	/*
3180	 * If we are deleting a changed name that never made it to disk,
3181	 * then return the dirrem describing the previous inode (which
3182	 * represents the inode currently referenced from this entry on disk).
3183	 */
3184	if ((dap->da_state & DIRCHG) != 0) {
3185		*prevdirremp = dap->da_previous;
3186		dap->da_state &= ~DIRCHG;
3187		dap->da_pagedep = pagedep;
3188	}
3189	/*
3190	 * We are deleting an entry that never made it to disk.
3191	 * Mark it COMPLETE so we can delete its inode immediately.
3192	 */
3193	dirrem->dm_state |= COMPLETE;
3194	free_diradd(dap);
3195	return (dirrem);
3196}
3197
3198/*
3199 * Directory entry change dependencies.
3200 *
3201 * Changing an existing directory entry requires that an add operation
3202 * be completed first followed by a deletion. The semantics for the addition
3203 * are identical to the description of adding a new entry above except
3204 * that the rollback is to the old inode number rather than zero. Once
3205 * the addition dependency is completed, the removal is done as described
3206 * in the removal routine above.
3207 */
3208
3209/*
3210 * This routine should be called immediately after changing
3211 * a directory entry.  The inode's link count should not be
3212 * decremented by the calling procedure -- the soft updates
3213 * code will perform this task when it is safe.
3214 */
3215void
3216softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
3217	struct buf *bp;		/* buffer containing directory block */
3218	struct inode *dp;	/* inode for the directory being modified */
3219	struct inode *ip;	/* inode for directory entry being removed */
3220	ino_t newinum;		/* new inode number for changed entry */
3221	int isrmdir;		/* indicates if doing RMDIR */
3222{
3223	int offset;
3224	struct diradd *dap = NULL;
3225	struct dirrem *dirrem, *prevdirrem;
3226	struct pagedep *pagedep;
3227	struct inodedep *inodedep;
3228
3229	offset = blkoff(dp->i_fs, dp->i_offset);
3230
3231	/*
3232	 * Whiteouts do not need diradd dependencies.
3233	 */
3234	if (newinum != WINO) {
3235		MALLOC(dap, struct diradd *, sizeof(struct diradd),
3236		    M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO);
3237		dap->da_list.wk_type = D_DIRADD;
3238		dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE;
3239		dap->da_offset = offset;
3240		dap->da_newinum = newinum;
3241	}
3242
3243	/*
3244	 * Allocate a new dirrem and ACQUIRE_LOCK.
3245	 */
3246	dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
3247	pagedep = dirrem->dm_pagedep;
3248	/*
3249	 * The possible values for isrmdir:
3250	 *	0 - non-directory file rename
3251	 *	1 - directory rename within same directory
3252	 *   inum - directory rename to new directory of given inode number
3253	 * When renaming to a new directory, we are both deleting and
3254	 * creating a new directory entry, so the link count on the new
3255	 * directory should not change. Thus we do not need the followup
3256	 * dirrem which is usually done in handle_workitem_remove. We set
3257	 * the DIRCHG flag to tell handle_workitem_remove to skip the
3258	 * followup dirrem.
3259	 */
3260	if (isrmdir > 1)
3261		dirrem->dm_state |= DIRCHG;
3262
3263	/*
3264	 * Whiteouts have no additional dependencies,
3265	 * so just put the dirrem on the correct list.
3266	 */
3267	if (newinum == WINO) {
3268		if ((dirrem->dm_state & COMPLETE) == 0) {
3269			LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem,
3270			    dm_next);
3271		} else {
3272			dirrem->dm_dirinum = pagedep->pd_ino;
3273			add_to_worklist(&dirrem->dm_list);
3274		}
3275		FREE_LOCK(&lk);
3276		return;
3277	}
3278
3279	/*
3280	 * If the COMPLETE flag is clear, then there were no active
3281	 * entries and we want to roll back to the previous inode until
3282	 * the new inode is committed to disk. If the COMPLETE flag is
3283	 * set, then we have deleted an entry that never made it to disk.
3284	 * If the entry we deleted resulted from a name change, then the old
3285	 * inode reference still resides on disk. Any rollback that we do
3286	 * needs to be to that old inode (returned to us in prevdirrem). If
3287	 * the entry we deleted resulted from a create, then there is
3288	 * no entry on the disk, so we want to roll back to zero rather
3289	 * than the uncommitted inode. In either of the COMPLETE cases we
3290	 * want to immediately free the unwritten and unreferenced inode.
3291	 */
3292	if ((dirrem->dm_state & COMPLETE) == 0) {
3293		dap->da_previous = dirrem;
3294	} else {
3295		if (prevdirrem != NULL) {
3296			dap->da_previous = prevdirrem;
3297		} else {
3298			dap->da_state &= ~DIRCHG;
3299			dap->da_pagedep = pagedep;
3300		}
3301		dirrem->dm_dirinum = pagedep->pd_ino;
3302		add_to_worklist(&dirrem->dm_list);
3303	}
3304	/*
3305	 * Link into its inodedep. Put it on the id_bufwait list if the inode
3306	 * is not yet written. If it is written, do the post-inode write
3307	 * processing to put it on the id_pendinghd list.
3308	 */
3309	if (inodedep_lookup(dp->i_fs, newinum, DEPALLOC, &inodedep) == 0 ||
3310	    (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
3311		dap->da_state |= COMPLETE;
3312		LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
3313		WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
3314	} else {
3315		LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)],
3316		    dap, da_pdlist);
3317		WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
3318	}
3319	FREE_LOCK(&lk);
3320}
3321
3322/*
3323 * Called whenever the link count on an inode is changed.
3324 * It creates an inode dependency so that the new reference(s)
3325 * to the inode cannot be committed to disk until the updated
3326 * inode has been written.
3327 */
3328void
3329softdep_change_linkcnt(ip)
3330	struct inode *ip;	/* the inode with the increased link count */
3331{
3332	struct inodedep *inodedep;
3333
3334	ACQUIRE_LOCK(&lk);
3335	(void) inodedep_lookup(ip->i_fs, ip->i_number, DEPALLOC, &inodedep);
3336	if (ip->i_nlink < ip->i_effnlink)
3337		panic("softdep_change_linkcnt: bad delta");
3338	inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
3339	FREE_LOCK(&lk);
3340}
3341
3342/*
3343 * Called when the effective link count and the reference count
3344 * on an inode drops to zero. At this point there are no names
3345 * referencing the file in the filesystem and no active file
3346 * references. The space associated with the file will be freed
3347 * as soon as the necessary soft dependencies are cleared.
3348 */
3349void
3350softdep_releasefile(ip)
3351	struct inode *ip;	/* inode with the zero effective link count */
3352{
3353	struct inodedep *inodedep;
3354	struct fs *fs;
3355	int extblocks;
3356
3357	if (ip->i_effnlink > 0)
3358		panic("softdep_filerelease: file still referenced");
3359	/*
3360	 * We may be called several times as the real reference count
3361	 * drops to zero. We only want to account for the space once.
3362	 */
3363	if (ip->i_flag & IN_SPACECOUNTED)
3364		return;
3365	/*
3366	 * We have to deactivate a snapshot otherwise copyonwrites may
3367	 * add blocks and the cleanup may remove blocks after we have
3368	 * tried to account for them.
3369	 */
3370	if ((ip->i_flags & SF_SNAPSHOT) != 0)
3371		ffs_snapremove(ITOV(ip));
3372	/*
3373	 * If we are tracking an nlinkdelta, we have to also remember
3374	 * whether we accounted for the freed space yet.
3375	 */
3376	ACQUIRE_LOCK(&lk);
3377	if ((inodedep_lookup(ip->i_fs, ip->i_number, 0, &inodedep)))
3378		inodedep->id_state |= SPACECOUNTED;
3379	FREE_LOCK(&lk);
3380	fs = ip->i_fs;
3381	extblocks = 0;
3382	if (fs->fs_magic == FS_UFS2_MAGIC)
3383		extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
3384	UFS_LOCK(ip->i_ump);
3385	ip->i_fs->fs_pendingblocks += DIP(ip, i_blocks) - extblocks;
3386	ip->i_fs->fs_pendinginodes += 1;
3387	UFS_UNLOCK(ip->i_ump);
3388	ip->i_flag |= IN_SPACECOUNTED;
3389}
3390
3391/*
3392 * This workitem decrements the inode's link count.
3393 * If the link count reaches zero, the file is removed.
3394 */
3395static void
3396handle_workitem_remove(dirrem, xp)
3397	struct dirrem *dirrem;
3398	struct vnode *xp;
3399{
3400	struct thread *td = curthread;
3401	struct inodedep *inodedep;
3402	struct vnode *vp;
3403	struct inode *ip;
3404	ino_t oldinum;
3405	int error;
3406
3407	if ((vp = xp) == NULL &&
3408	    (error = ffs_vget(dirrem->dm_mnt, dirrem->dm_oldinum, LK_EXCLUSIVE,
3409	     &vp)) != 0) {
3410		softdep_error("handle_workitem_remove: vget", error);
3411		return;
3412	}
3413	ip = VTOI(vp);
3414	ACQUIRE_LOCK(&lk);
3415	if ((inodedep_lookup(ip->i_fs, dirrem->dm_oldinum, 0, &inodedep)) == 0)
3416		panic("handle_workitem_remove: lost inodedep");
3417	/*
3418	 * Normal file deletion.
3419	 */
3420	if ((dirrem->dm_state & RMDIR) == 0) {
3421		ip->i_nlink--;
3422		DIP_SET(ip, i_nlink, ip->i_nlink);
3423		ip->i_flag |= IN_CHANGE;
3424		if (ip->i_nlink < ip->i_effnlink)
3425			panic("handle_workitem_remove: bad file delta");
3426		inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
3427		num_dirrem -= 1;
3428		FREE_LOCK(&lk);
3429		vput(vp);
3430		WORKITEM_FREE(dirrem, D_DIRREM);
3431		return;
3432	}
3433	/*
3434	 * Directory deletion. Decrement reference count for both the
3435	 * just deleted parent directory entry and the reference for ".".
3436	 * Next truncate the directory to length zero. When the
3437	 * truncation completes, arrange to have the reference count on
3438	 * the parent decremented to account for the loss of "..".
3439	 */
3440	ip->i_nlink -= 2;
3441	DIP_SET(ip, i_nlink, ip->i_nlink);
3442	ip->i_flag |= IN_CHANGE;
3443	if (ip->i_nlink < ip->i_effnlink)
3444		panic("handle_workitem_remove: bad dir delta");
3445	inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
3446	FREE_LOCK(&lk);
3447	if ((error = ffs_truncate(vp, (off_t)0, 0, td->td_ucred, td)) != 0)
3448		softdep_error("handle_workitem_remove: truncate", error);
3449	ACQUIRE_LOCK(&lk);
3450	/*
3451	 * Rename a directory to a new parent. Since, we are both deleting
3452	 * and creating a new directory entry, the link count on the new
3453	 * directory should not change. Thus we skip the followup dirrem.
3454	 */
3455	if (dirrem->dm_state & DIRCHG) {
3456		num_dirrem -= 1;
3457		FREE_LOCK(&lk);
3458		vput(vp);
3459		WORKITEM_FREE(dirrem, D_DIRREM);
3460		return;
3461	}
3462	/*
3463	 * If the inodedep does not exist, then the zero'ed inode has
3464	 * been written to disk. If the allocated inode has never been
3465	 * written to disk, then the on-disk inode is zero'ed. In either
3466	 * case we can remove the file immediately.
3467	 */
3468	dirrem->dm_state = 0;
3469	oldinum = dirrem->dm_oldinum;
3470	dirrem->dm_oldinum = dirrem->dm_dirinum;
3471	if (inodedep_lookup(ip->i_fs, oldinum, 0, &inodedep) == 0 ||
3472	    check_inode_unwritten(inodedep)) {
3473		FREE_LOCK(&lk);
3474		vput(vp);
3475		handle_workitem_remove(dirrem, NULL);
3476		return;
3477	}
3478	WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list);
3479	FREE_LOCK(&lk);
3480	vput(vp);
3481}
3482
3483/*
3484 * Inode de-allocation dependencies.
3485 *
3486 * When an inode's link count is reduced to zero, it can be de-allocated. We
3487 * found it convenient to postpone de-allocation until after the inode is
3488 * written to disk with its new link count (zero).  At this point, all of the
3489 * on-disk inode's block pointers are nullified and, with careful dependency
3490 * list ordering, all dependencies related to the inode will be satisfied and
3491 * the corresponding dependency structures de-allocated.  So, if/when the
3492 * inode is reused, there will be no mixing of old dependencies with new
3493 * ones.  This artificial dependency is set up by the block de-allocation
3494 * procedure above (softdep_setup_freeblocks) and completed by the
3495 * following procedure.
3496 */
3497static void
3498handle_workitem_freefile(freefile)
3499	struct freefile *freefile;
3500{
3501	struct fs *fs;
3502	struct inodedep *idp;
3503	struct ufsmount *ump;
3504	int error;
3505
3506	ump = VFSTOUFS(freefile->fx_mnt);
3507	fs = ump->um_fs;
3508#ifdef DEBUG
3509	ACQUIRE_LOCK(&lk);
3510	error = inodedep_lookup(fs, freefile->fx_oldinum, 0, &idp);
3511	FREE_LOCK(&lk);
3512	if (error)
3513		panic("handle_workitem_freefile: inodedep survived");
3514#endif
3515	UFS_LOCK(ump);
3516	fs->fs_pendinginodes -= 1;
3517	UFS_UNLOCK(ump);
3518	if ((error = ffs_freefile(VFSTOUFS(freefile->fx_mnt), fs,
3519	    freefile->fx_devvp, freefile->fx_oldinum, freefile->fx_mode)) != 0)
3520		softdep_error("handle_workitem_freefile", error);
3521	WORKITEM_FREE(freefile, D_FREEFILE);
3522}
3523
3524
3525/*
3526 * Helper function which unlinks marker element from work list and returns
3527 * the next element on the list.
3528 */
3529static __inline struct worklist *
3530markernext(struct worklist *marker)
3531{
3532	struct worklist *next;
3533
3534	next = LIST_NEXT(marker, wk_list);
3535	LIST_REMOVE(marker, wk_list);
3536	return next;
3537}
3538
3539/*
3540 * Disk writes.
3541 *
3542 * The dependency structures constructed above are most actively used when file
3543 * system blocks are written to disk.  No constraints are placed on when a
3544 * block can be written, but unsatisfied update dependencies are made safe by
3545 * modifying (or replacing) the source memory for the duration of the disk
3546 * write.  When the disk write completes, the memory block is again brought
3547 * up-to-date.
3548 *
3549 * In-core inode structure reclamation.
3550 *
3551 * Because there are a finite number of "in-core" inode structures, they are
3552 * reused regularly.  By transferring all inode-related dependencies to the
3553 * in-memory inode block and indexing them separately (via "inodedep"s), we
3554 * can allow "in-core" inode structures to be reused at any time and avoid
3555 * any increase in contention.
3556 *
3557 * Called just before entering the device driver to initiate a new disk I/O.
3558 * The buffer must be locked, thus, no I/O completion operations can occur
3559 * while we are manipulating its associated dependencies.
3560 */
3561static void
3562softdep_disk_io_initiation(bp)
3563	struct buf *bp;		/* structure describing disk write to occur */
3564{
3565	struct worklist *wk;
3566	struct worklist marker;
3567	struct indirdep *indirdep;
3568	struct inodedep *inodedep;
3569
3570	/*
3571	 * We only care about write operations. There should never
3572	 * be dependencies for reads.
3573	 */
3574	if (bp->b_iocmd != BIO_WRITE)
3575		panic("softdep_disk_io_initiation: not write");
3576
3577	marker.wk_type = D_LAST + 1;	/* Not a normal workitem */
3578	PHOLD(curproc);			/* Don't swap out kernel stack */
3579
3580	ACQUIRE_LOCK(&lk);
3581	/*
3582	 * Do any necessary pre-I/O processing.
3583	 */
3584	for (wk = LIST_FIRST(&bp->b_dep); wk != NULL;
3585	     wk = markernext(&marker)) {
3586		LIST_INSERT_AFTER(wk, &marker, wk_list);
3587		switch (wk->wk_type) {
3588
3589		case D_PAGEDEP:
3590			initiate_write_filepage(WK_PAGEDEP(wk), bp);
3591			continue;
3592
3593		case D_INODEDEP:
3594			inodedep = WK_INODEDEP(wk);
3595			if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC)
3596				initiate_write_inodeblock_ufs1(inodedep, bp);
3597			else
3598				initiate_write_inodeblock_ufs2(inodedep, bp);
3599			continue;
3600
3601		case D_INDIRDEP:
3602			indirdep = WK_INDIRDEP(wk);
3603			if (indirdep->ir_state & GOINGAWAY)
3604				panic("disk_io_initiation: indirdep gone");
3605			/*
3606			 * If there are no remaining dependencies, this
3607			 * will be writing the real pointers, so the
3608			 * dependency can be freed.
3609			 */
3610			if (LIST_FIRST(&indirdep->ir_deplisthd) == NULL) {
3611				struct buf *bp;
3612
3613				bp = indirdep->ir_savebp;
3614				bp->b_flags |= B_INVAL | B_NOCACHE;
3615				/* inline expand WORKLIST_REMOVE(wk); */
3616				wk->wk_state &= ~ONWORKLIST;
3617				LIST_REMOVE(wk, wk_list);
3618				WORKITEM_FREE(indirdep, D_INDIRDEP);
3619				FREE_LOCK(&lk);
3620				brelse(bp);
3621				ACQUIRE_LOCK(&lk);
3622				continue;
3623			}
3624			/*
3625			 * Replace up-to-date version with safe version.
3626			 */
3627			FREE_LOCK(&lk);
3628			MALLOC(indirdep->ir_saveddata, caddr_t, bp->b_bcount,
3629			    M_INDIRDEP, M_SOFTDEP_FLAGS);
3630			ACQUIRE_LOCK(&lk);
3631			indirdep->ir_state &= ~ATTACHED;
3632			indirdep->ir_state |= UNDONE;
3633			bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount);
3634			bcopy(indirdep->ir_savebp->b_data, bp->b_data,
3635			    bp->b_bcount);
3636			continue;
3637
3638		case D_MKDIR:
3639		case D_BMSAFEMAP:
3640		case D_ALLOCDIRECT:
3641		case D_ALLOCINDIR:
3642			continue;
3643
3644		default:
3645			panic("handle_disk_io_initiation: Unexpected type %s",
3646			    TYPENAME(wk->wk_type));
3647			/* NOTREACHED */
3648		}
3649	}
3650	FREE_LOCK(&lk);
3651	PRELE(curproc);			/* Allow swapout of kernel stack */
3652}
3653
3654/*
3655 * Called from within the procedure above to deal with unsatisfied
3656 * allocation dependencies in a directory. The buffer must be locked,
3657 * thus, no I/O completion operations can occur while we are
3658 * manipulating its associated dependencies.
3659 */
3660static void
3661initiate_write_filepage(pagedep, bp)
3662	struct pagedep *pagedep;
3663	struct buf *bp;
3664{
3665	struct diradd *dap;
3666	struct direct *ep;
3667	int i;
3668
3669	if (pagedep->pd_state & IOSTARTED) {
3670		/*
3671		 * This can only happen if there is a driver that does not
3672		 * understand chaining. Here biodone will reissue the call
3673		 * to strategy for the incomplete buffers.
3674		 */
3675		printf("initiate_write_filepage: already started\n");
3676		return;
3677	}
3678	pagedep->pd_state |= IOSTARTED;
3679	for (i = 0; i < DAHASHSZ; i++) {
3680		LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
3681			ep = (struct direct *)
3682			    ((char *)bp->b_data + dap->da_offset);
3683			if (ep->d_ino != dap->da_newinum)
3684				panic("%s: dir inum %d != new %d",
3685				    "initiate_write_filepage",
3686				    ep->d_ino, dap->da_newinum);
3687			if (dap->da_state & DIRCHG)
3688				ep->d_ino = dap->da_previous->dm_oldinum;
3689			else
3690				ep->d_ino = 0;
3691			dap->da_state &= ~ATTACHED;
3692			dap->da_state |= UNDONE;
3693		}
3694	}
3695}
3696
3697/*
3698 * Version of initiate_write_inodeblock that handles UFS1 dinodes.
3699 * Note that any bug fixes made to this routine must be done in the
3700 * version found below.
3701 *
3702 * Called from within the procedure above to deal with unsatisfied
3703 * allocation dependencies in an inodeblock. The buffer must be
3704 * locked, thus, no I/O completion operations can occur while we
3705 * are manipulating its associated dependencies.
3706 */
3707static void
3708initiate_write_inodeblock_ufs1(inodedep, bp)
3709	struct inodedep *inodedep;
3710	struct buf *bp;			/* The inode block */
3711{
3712	struct allocdirect *adp, *lastadp;
3713	struct ufs1_dinode *dp;
3714	struct ufs1_dinode *sip;
3715	struct fs *fs;
3716	ufs_lbn_t i, prevlbn = 0;
3717	int deplist;
3718
3719	if (inodedep->id_state & IOSTARTED)
3720		panic("initiate_write_inodeblock_ufs1: already started");
3721	inodedep->id_state |= IOSTARTED;
3722	fs = inodedep->id_fs;
3723	dp = (struct ufs1_dinode *)bp->b_data +
3724	    ino_to_fsbo(fs, inodedep->id_ino);
3725	/*
3726	 * If the bitmap is not yet written, then the allocated
3727	 * inode cannot be written to disk.
3728	 */
3729	if ((inodedep->id_state & DEPCOMPLETE) == 0) {
3730		if (inodedep->id_savedino1 != NULL)
3731			panic("initiate_write_inodeblock_ufs1: I/O underway");
3732		FREE_LOCK(&lk);
3733		MALLOC(sip, struct ufs1_dinode *,
3734		    sizeof(struct ufs1_dinode), M_SAVEDINO, M_SOFTDEP_FLAGS);
3735		ACQUIRE_LOCK(&lk);
3736		inodedep->id_savedino1 = sip;
3737		*inodedep->id_savedino1 = *dp;
3738		bzero((caddr_t)dp, sizeof(struct ufs1_dinode));
3739		dp->di_gen = inodedep->id_savedino1->di_gen;
3740		return;
3741	}
3742	/*
3743	 * If no dependencies, then there is nothing to roll back.
3744	 */
3745	inodedep->id_savedsize = dp->di_size;
3746	inodedep->id_savedextsize = 0;
3747	if (TAILQ_FIRST(&inodedep->id_inoupdt) == NULL)
3748		return;
3749	/*
3750	 * Set the dependencies to busy.
3751	 */
3752	for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
3753	     adp = TAILQ_NEXT(adp, ad_next)) {
3754#ifdef DIAGNOSTIC
3755		if (deplist != 0 && prevlbn >= adp->ad_lbn)
3756			panic("softdep_write_inodeblock: lbn order");
3757		prevlbn = adp->ad_lbn;
3758		if (adp->ad_lbn < NDADDR &&
3759		    dp->di_db[adp->ad_lbn] != adp->ad_newblkno)
3760			panic("%s: direct pointer #%jd mismatch %d != %jd",
3761			    "softdep_write_inodeblock",
3762			    (intmax_t)adp->ad_lbn,
3763			    dp->di_db[adp->ad_lbn],
3764			    (intmax_t)adp->ad_newblkno);
3765		if (adp->ad_lbn >= NDADDR &&
3766		    dp->di_ib[adp->ad_lbn - NDADDR] != adp->ad_newblkno)
3767			panic("%s: indirect pointer #%jd mismatch %d != %jd",
3768			    "softdep_write_inodeblock",
3769			    (intmax_t)adp->ad_lbn - NDADDR,
3770			    dp->di_ib[adp->ad_lbn - NDADDR],
3771			    (intmax_t)adp->ad_newblkno);
3772		deplist |= 1 << adp->ad_lbn;
3773		if ((adp->ad_state & ATTACHED) == 0)
3774			panic("softdep_write_inodeblock: Unknown state 0x%x",
3775			    adp->ad_state);
3776#endif /* DIAGNOSTIC */
3777		adp->ad_state &= ~ATTACHED;
3778		adp->ad_state |= UNDONE;
3779	}
3780	/*
3781	 * The on-disk inode cannot claim to be any larger than the last
3782	 * fragment that has been written. Otherwise, the on-disk inode
3783	 * might have fragments that were not the last block in the file
3784	 * which would corrupt the filesystem.
3785	 */
3786	for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
3787	     lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
3788		if (adp->ad_lbn >= NDADDR)
3789			break;
3790		dp->di_db[adp->ad_lbn] = adp->ad_oldblkno;
3791		/* keep going until hitting a rollback to a frag */
3792		if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
3793			continue;
3794		dp->di_size = fs->fs_bsize * adp->ad_lbn + adp->ad_oldsize;
3795		for (i = adp->ad_lbn + 1; i < NDADDR; i++) {
3796#ifdef DIAGNOSTIC
3797			if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
3798				panic("softdep_write_inodeblock: lost dep1");
3799#endif /* DIAGNOSTIC */
3800			dp->di_db[i] = 0;
3801		}
3802		for (i = 0; i < NIADDR; i++) {
3803#ifdef DIAGNOSTIC
3804			if (dp->di_ib[i] != 0 &&
3805			    (deplist & ((1 << NDADDR) << i)) == 0)
3806				panic("softdep_write_inodeblock: lost dep2");
3807#endif /* DIAGNOSTIC */
3808			dp->di_ib[i] = 0;
3809		}
3810		return;
3811	}
3812	/*
3813	 * If we have zero'ed out the last allocated block of the file,
3814	 * roll back the size to the last currently allocated block.
3815	 * We know that this last allocated block is a full-sized as
3816	 * we already checked for fragments in the loop above.
3817	 */
3818	if (lastadp != NULL &&
3819	    dp->di_size <= (lastadp->ad_lbn + 1) * fs->fs_bsize) {
3820		for (i = lastadp->ad_lbn; i >= 0; i--)
3821			if (dp->di_db[i] != 0)
3822				break;
3823		dp->di_size = (i + 1) * fs->fs_bsize;
3824	}
3825	/*
3826	 * The only dependencies are for indirect blocks.
3827	 *
3828	 * The file size for indirect block additions is not guaranteed.
3829	 * Such a guarantee would be non-trivial to achieve. The conventional
3830	 * synchronous write implementation also does not make this guarantee.
3831	 * Fsck should catch and fix discrepancies. Arguably, the file size
3832	 * can be over-estimated without destroying integrity when the file
3833	 * moves into the indirect blocks (i.e., is large). If we want to
3834	 * postpone fsck, we are stuck with this argument.
3835	 */
3836	for (; adp; adp = TAILQ_NEXT(adp, ad_next))
3837		dp->di_ib[adp->ad_lbn - NDADDR] = 0;
3838}
3839
3840/*
3841 * Version of initiate_write_inodeblock that handles UFS2 dinodes.
3842 * Note that any bug fixes made to this routine must be done in the
3843 * version found above.
3844 *
3845 * Called from within the procedure above to deal with unsatisfied
3846 * allocation dependencies in an inodeblock. The buffer must be
3847 * locked, thus, no I/O completion operations can occur while we
3848 * are manipulating its associated dependencies.
3849 */
3850static void
3851initiate_write_inodeblock_ufs2(inodedep, bp)
3852	struct inodedep *inodedep;
3853	struct buf *bp;			/* The inode block */
3854{
3855	struct allocdirect *adp, *lastadp;
3856	struct ufs2_dinode *dp;
3857	struct ufs2_dinode *sip;
3858	struct fs *fs;
3859	ufs_lbn_t i, prevlbn = 0;
3860	int deplist;
3861
3862	if (inodedep->id_state & IOSTARTED)
3863		panic("initiate_write_inodeblock_ufs2: already started");
3864	inodedep->id_state |= IOSTARTED;
3865	fs = inodedep->id_fs;
3866	dp = (struct ufs2_dinode *)bp->b_data +
3867	    ino_to_fsbo(fs, inodedep->id_ino);
3868	/*
3869	 * If the bitmap is not yet written, then the allocated
3870	 * inode cannot be written to disk.
3871	 */
3872	if ((inodedep->id_state & DEPCOMPLETE) == 0) {
3873		if (inodedep->id_savedino2 != NULL)
3874			panic("initiate_write_inodeblock_ufs2: I/O underway");
3875		FREE_LOCK(&lk);
3876		MALLOC(sip, struct ufs2_dinode *,
3877		    sizeof(struct ufs2_dinode), M_SAVEDINO, M_SOFTDEP_FLAGS);
3878		ACQUIRE_LOCK(&lk);
3879		inodedep->id_savedino2 = sip;
3880		*inodedep->id_savedino2 = *dp;
3881		bzero((caddr_t)dp, sizeof(struct ufs2_dinode));
3882		dp->di_gen = inodedep->id_savedino2->di_gen;
3883		return;
3884	}
3885	/*
3886	 * If no dependencies, then there is nothing to roll back.
3887	 */
3888	inodedep->id_savedsize = dp->di_size;
3889	inodedep->id_savedextsize = dp->di_extsize;
3890	if (TAILQ_FIRST(&inodedep->id_inoupdt) == NULL &&
3891	    TAILQ_FIRST(&inodedep->id_extupdt) == NULL)
3892		return;
3893	/*
3894	 * Set the ext data dependencies to busy.
3895	 */
3896	for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
3897	     adp = TAILQ_NEXT(adp, ad_next)) {
3898#ifdef DIAGNOSTIC
3899		if (deplist != 0 && prevlbn >= adp->ad_lbn)
3900			panic("softdep_write_inodeblock: lbn order");
3901		prevlbn = adp->ad_lbn;
3902		if (dp->di_extb[adp->ad_lbn] != adp->ad_newblkno)
3903			panic("%s: direct pointer #%jd mismatch %jd != %jd",
3904			    "softdep_write_inodeblock",
3905			    (intmax_t)adp->ad_lbn,
3906			    (intmax_t)dp->di_extb[adp->ad_lbn],
3907			    (intmax_t)adp->ad_newblkno);
3908		deplist |= 1 << adp->ad_lbn;
3909		if ((adp->ad_state & ATTACHED) == 0)
3910			panic("softdep_write_inodeblock: Unknown state 0x%x",
3911			    adp->ad_state);
3912#endif /* DIAGNOSTIC */
3913		adp->ad_state &= ~ATTACHED;
3914		adp->ad_state |= UNDONE;
3915	}
3916	/*
3917	 * The on-disk inode cannot claim to be any larger than the last
3918	 * fragment that has been written. Otherwise, the on-disk inode
3919	 * might have fragments that were not the last block in the ext
3920	 * data which would corrupt the filesystem.
3921	 */
3922	for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
3923	     lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
3924		dp->di_extb[adp->ad_lbn] = adp->ad_oldblkno;
3925		/* keep going until hitting a rollback to a frag */
3926		if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
3927			continue;
3928		dp->di_extsize = fs->fs_bsize * adp->ad_lbn + adp->ad_oldsize;
3929		for (i = adp->ad_lbn + 1; i < NXADDR; i++) {
3930#ifdef DIAGNOSTIC
3931			if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0)
3932				panic("softdep_write_inodeblock: lost dep1");
3933#endif /* DIAGNOSTIC */
3934			dp->di_extb[i] = 0;
3935		}
3936		lastadp = NULL;
3937		break;
3938	}
3939	/*
3940	 * If we have zero'ed out the last allocated block of the ext
3941	 * data, roll back the size to the last currently allocated block.
3942	 * We know that this last allocated block is a full-sized as
3943	 * we already checked for fragments in the loop above.
3944	 */
3945	if (lastadp != NULL &&
3946	    dp->di_extsize <= (lastadp->ad_lbn + 1) * fs->fs_bsize) {
3947		for (i = lastadp->ad_lbn; i >= 0; i--)
3948			if (dp->di_extb[i] != 0)
3949				break;
3950		dp->di_extsize = (i + 1) * fs->fs_bsize;
3951	}
3952	/*
3953	 * Set the file data dependencies to busy.
3954	 */
3955	for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
3956	     adp = TAILQ_NEXT(adp, ad_next)) {
3957#ifdef DIAGNOSTIC
3958		if (deplist != 0 && prevlbn >= adp->ad_lbn)
3959			panic("softdep_write_inodeblock: lbn order");
3960		prevlbn = adp->ad_lbn;
3961		if (adp->ad_lbn < NDADDR &&
3962		    dp->di_db[adp->ad_lbn] != adp->ad_newblkno)
3963			panic("%s: direct pointer #%jd mismatch %jd != %jd",
3964			    "softdep_write_inodeblock",
3965			    (intmax_t)adp->ad_lbn,
3966			    (intmax_t)dp->di_db[adp->ad_lbn],
3967			    (intmax_t)adp->ad_newblkno);
3968		if (adp->ad_lbn >= NDADDR &&
3969		    dp->di_ib[adp->ad_lbn - NDADDR] != adp->ad_newblkno)
3970			panic("%s indirect pointer #%jd mismatch %jd != %jd",
3971			    "softdep_write_inodeblock:",
3972			    (intmax_t)adp->ad_lbn - NDADDR,
3973			    (intmax_t)dp->di_ib[adp->ad_lbn - NDADDR],
3974			    (intmax_t)adp->ad_newblkno);
3975		deplist |= 1 << adp->ad_lbn;
3976		if ((adp->ad_state & ATTACHED) == 0)
3977			panic("softdep_write_inodeblock: Unknown state 0x%x",
3978			    adp->ad_state);
3979#endif /* DIAGNOSTIC */
3980		adp->ad_state &= ~ATTACHED;
3981		adp->ad_state |= UNDONE;
3982	}
3983	/*
3984	 * The on-disk inode cannot claim to be any larger than the last
3985	 * fragment that has been written. Otherwise, the on-disk inode
3986	 * might have fragments that were not the last block in the file
3987	 * which would corrupt the filesystem.
3988	 */
3989	for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
3990	     lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
3991		if (adp->ad_lbn >= NDADDR)
3992			break;
3993		dp->di_db[adp->ad_lbn] = adp->ad_oldblkno;
3994		/* keep going until hitting a rollback to a frag */
3995		if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
3996			continue;
3997		dp->di_size = fs->fs_bsize * adp->ad_lbn + adp->ad_oldsize;
3998		for (i = adp->ad_lbn + 1; i < NDADDR; i++) {
3999#ifdef DIAGNOSTIC
4000			if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
4001				panic("softdep_write_inodeblock: lost dep2");
4002#endif /* DIAGNOSTIC */
4003			dp->di_db[i] = 0;
4004		}
4005		for (i = 0; i < NIADDR; i++) {
4006#ifdef DIAGNOSTIC
4007			if (dp->di_ib[i] != 0 &&
4008			    (deplist & ((1 << NDADDR) << i)) == 0)
4009				panic("softdep_write_inodeblock: lost dep3");
4010#endif /* DIAGNOSTIC */
4011			dp->di_ib[i] = 0;
4012		}
4013		return;
4014	}
4015	/*
4016	 * If we have zero'ed out the last allocated block of the file,
4017	 * roll back the size to the last currently allocated block.
4018	 * We know that this last allocated block is a full-sized as
4019	 * we already checked for fragments in the loop above.
4020	 */
4021	if (lastadp != NULL &&
4022	    dp->di_size <= (lastadp->ad_lbn + 1) * fs->fs_bsize) {
4023		for (i = lastadp->ad_lbn; i >= 0; i--)
4024			if (dp->di_db[i] != 0)
4025				break;
4026		dp->di_size = (i + 1) * fs->fs_bsize;
4027	}
4028	/*
4029	 * The only dependencies are for indirect blocks.
4030	 *
4031	 * The file size for indirect block additions is not guaranteed.
4032	 * Such a guarantee would be non-trivial to achieve. The conventional
4033	 * synchronous write implementation also does not make this guarantee.
4034	 * Fsck should catch and fix discrepancies. Arguably, the file size
4035	 * can be over-estimated without destroying integrity when the file
4036	 * moves into the indirect blocks (i.e., is large). If we want to
4037	 * postpone fsck, we are stuck with this argument.
4038	 */
4039	for (; adp; adp = TAILQ_NEXT(adp, ad_next))
4040		dp->di_ib[adp->ad_lbn - NDADDR] = 0;
4041}
4042
4043/*
4044 * This routine is called during the completion interrupt
4045 * service routine for a disk write (from the procedure called
4046 * by the device driver to inform the filesystem caches of
4047 * a request completion).  It should be called early in this
4048 * procedure, before the block is made available to other
4049 * processes or other routines are called.
4050 */
4051static void
4052softdep_disk_write_complete(bp)
4053	struct buf *bp;		/* describes the completed disk write */
4054{
4055	struct worklist *wk;
4056	struct worklist *owk;
4057	struct workhead reattach;
4058	struct newblk *newblk;
4059	struct allocindir *aip;
4060	struct allocdirect *adp;
4061	struct indirdep *indirdep;
4062	struct inodedep *inodedep;
4063	struct bmsafemap *bmsafemap;
4064
4065	/*
4066	 * If an error occurred while doing the write, then the data
4067	 * has not hit the disk and the dependencies cannot be unrolled.
4068	 */
4069	if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0)
4070		return;
4071	LIST_INIT(&reattach);
4072	/*
4073	 * This lock must not be released anywhere in this code segment.
4074	 */
4075	ACQUIRE_LOCK(&lk);
4076	owk = NULL;
4077	while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) {
4078		WORKLIST_REMOVE(wk);
4079		if (wk == owk)
4080			panic("duplicate worklist: %p\n", wk);
4081		owk = wk;
4082		switch (wk->wk_type) {
4083
4084		case D_PAGEDEP:
4085			if (handle_written_filepage(WK_PAGEDEP(wk), bp))
4086				WORKLIST_INSERT(&reattach, wk);
4087			continue;
4088
4089		case D_INODEDEP:
4090			if (handle_written_inodeblock(WK_INODEDEP(wk), bp))
4091				WORKLIST_INSERT(&reattach, wk);
4092			continue;
4093
4094		case D_BMSAFEMAP:
4095			bmsafemap = WK_BMSAFEMAP(wk);
4096			while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkhd))) {
4097				newblk->nb_state |= DEPCOMPLETE;
4098				newblk->nb_bmsafemap = NULL;
4099				LIST_REMOVE(newblk, nb_deps);
4100			}
4101			while ((adp =
4102			   LIST_FIRST(&bmsafemap->sm_allocdirecthd))) {
4103				adp->ad_state |= DEPCOMPLETE;
4104				adp->ad_buf = NULL;
4105				LIST_REMOVE(adp, ad_deps);
4106				handle_allocdirect_partdone(adp);
4107			}
4108			while ((aip =
4109			    LIST_FIRST(&bmsafemap->sm_allocindirhd))) {
4110				aip->ai_state |= DEPCOMPLETE;
4111				aip->ai_buf = NULL;
4112				LIST_REMOVE(aip, ai_deps);
4113				handle_allocindir_partdone(aip);
4114			}
4115			while ((inodedep =
4116			     LIST_FIRST(&bmsafemap->sm_inodedephd)) != NULL) {
4117				inodedep->id_state |= DEPCOMPLETE;
4118				LIST_REMOVE(inodedep, id_deps);
4119				inodedep->id_buf = NULL;
4120			}
4121			WORKITEM_FREE(bmsafemap, D_BMSAFEMAP);
4122			continue;
4123
4124		case D_MKDIR:
4125			handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY);
4126			continue;
4127
4128		case D_ALLOCDIRECT:
4129			adp = WK_ALLOCDIRECT(wk);
4130			adp->ad_state |= COMPLETE;
4131			handle_allocdirect_partdone(adp);
4132			continue;
4133
4134		case D_ALLOCINDIR:
4135			aip = WK_ALLOCINDIR(wk);
4136			aip->ai_state |= COMPLETE;
4137			handle_allocindir_partdone(aip);
4138			continue;
4139
4140		case D_INDIRDEP:
4141			indirdep = WK_INDIRDEP(wk);
4142			if (indirdep->ir_state & GOINGAWAY)
4143				panic("disk_write_complete: indirdep gone");
4144			bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount);
4145			FREE(indirdep->ir_saveddata, M_INDIRDEP);
4146			indirdep->ir_saveddata = 0;
4147			indirdep->ir_state &= ~UNDONE;
4148			indirdep->ir_state |= ATTACHED;
4149			while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) {
4150				handle_allocindir_partdone(aip);
4151				if (aip == LIST_FIRST(&indirdep->ir_donehd))
4152					panic("disk_write_complete: not gone");
4153			}
4154			WORKLIST_INSERT(&reattach, wk);
4155			if ((bp->b_flags & B_DELWRI) == 0)
4156				stat_indir_blk_ptrs++;
4157			bdirty(bp);
4158			continue;
4159
4160		default:
4161			panic("handle_disk_write_complete: Unknown type %s",
4162			    TYPENAME(wk->wk_type));
4163			/* NOTREACHED */
4164		}
4165	}
4166	/*
4167	 * Reattach any requests that must be redone.
4168	 */
4169	while ((wk = LIST_FIRST(&reattach)) != NULL) {
4170		WORKLIST_REMOVE(wk);
4171		WORKLIST_INSERT(&bp->b_dep, wk);
4172	}
4173	FREE_LOCK(&lk);
4174}
4175
4176/*
4177 * Called from within softdep_disk_write_complete above. Note that
4178 * this routine is always called from interrupt level with further
4179 * splbio interrupts blocked.
4180 */
4181static void
4182handle_allocdirect_partdone(adp)
4183	struct allocdirect *adp;	/* the completed allocdirect */
4184{
4185	struct allocdirectlst *listhead;
4186	struct allocdirect *listadp;
4187	struct inodedep *inodedep;
4188	long bsize, delay;
4189
4190	if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
4191		return;
4192	if (adp->ad_buf != NULL)
4193		panic("handle_allocdirect_partdone: dangling dep");
4194	/*
4195	 * The on-disk inode cannot claim to be any larger than the last
4196	 * fragment that has been written. Otherwise, the on-disk inode
4197	 * might have fragments that were not the last block in the file
4198	 * which would corrupt the filesystem. Thus, we cannot free any
4199	 * allocdirects after one whose ad_oldblkno claims a fragment as
4200	 * these blocks must be rolled back to zero before writing the inode.
4201	 * We check the currently active set of allocdirects in id_inoupdt
4202	 * or id_extupdt as appropriate.
4203	 */
4204	inodedep = adp->ad_inodedep;
4205	bsize = inodedep->id_fs->fs_bsize;
4206	if (adp->ad_state & EXTDATA)
4207		listhead = &inodedep->id_extupdt;
4208	else
4209		listhead = &inodedep->id_inoupdt;
4210	TAILQ_FOREACH(listadp, listhead, ad_next) {
4211		/* found our block */
4212		if (listadp == adp)
4213			break;
4214		/* continue if ad_oldlbn is not a fragment */
4215		if (listadp->ad_oldsize == 0 ||
4216		    listadp->ad_oldsize == bsize)
4217			continue;
4218		/* hit a fragment */
4219		return;
4220	}
4221	/*
4222	 * If we have reached the end of the current list without
4223	 * finding the just finished dependency, then it must be
4224	 * on the future dependency list. Future dependencies cannot
4225	 * be freed until they are moved to the current list.
4226	 */
4227	if (listadp == NULL) {
4228#ifdef DEBUG
4229		if (adp->ad_state & EXTDATA)
4230			listhead = &inodedep->id_newextupdt;
4231		else
4232			listhead = &inodedep->id_newinoupdt;
4233		TAILQ_FOREACH(listadp, listhead, ad_next)
4234			/* found our block */
4235			if (listadp == adp)
4236				break;
4237		if (listadp == NULL)
4238			panic("handle_allocdirect_partdone: lost dep");
4239#endif /* DEBUG */
4240		return;
4241	}
4242	/*
4243	 * If we have found the just finished dependency, then free
4244	 * it along with anything that follows it that is complete.
4245	 * If the inode still has a bitmap dependency, then it has
4246	 * never been written to disk, hence the on-disk inode cannot
4247	 * reference the old fragment so we can free it without delay.
4248	 */
4249	delay = (inodedep->id_state & DEPCOMPLETE);
4250	for (; adp; adp = listadp) {
4251		listadp = TAILQ_NEXT(adp, ad_next);
4252		if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
4253			return;
4254		free_allocdirect(listhead, adp, delay);
4255	}
4256}
4257
4258/*
4259 * Called from within softdep_disk_write_complete above. Note that
4260 * this routine is always called from interrupt level with further
4261 * splbio interrupts blocked.
4262 */
4263static void
4264handle_allocindir_partdone(aip)
4265	struct allocindir *aip;		/* the completed allocindir */
4266{
4267	struct indirdep *indirdep;
4268
4269	if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE)
4270		return;
4271	if (aip->ai_buf != NULL)
4272		panic("handle_allocindir_partdone: dangling dependency");
4273	indirdep = aip->ai_indirdep;
4274	if (indirdep->ir_state & UNDONE) {
4275		LIST_REMOVE(aip, ai_next);
4276		LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next);
4277		return;
4278	}
4279	if (indirdep->ir_state & UFS1FMT)
4280		((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
4281		    aip->ai_newblkno;
4282	else
4283		((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
4284		    aip->ai_newblkno;
4285	LIST_REMOVE(aip, ai_next);
4286	if (aip->ai_freefrag != NULL)
4287		add_to_worklist(&aip->ai_freefrag->ff_list);
4288	WORKITEM_FREE(aip, D_ALLOCINDIR);
4289}
4290
4291/*
4292 * Called from within softdep_disk_write_complete above to restore
4293 * in-memory inode block contents to their most up-to-date state. Note
4294 * that this routine is always called from interrupt level with further
4295 * splbio interrupts blocked.
4296 */
4297static int
4298handle_written_inodeblock(inodedep, bp)
4299	struct inodedep *inodedep;
4300	struct buf *bp;		/* buffer containing the inode block */
4301{
4302	struct worklist *wk, *filefree;
4303	struct allocdirect *adp, *nextadp;
4304	struct ufs1_dinode *dp1 = NULL;
4305	struct ufs2_dinode *dp2 = NULL;
4306	int hadchanges, fstype;
4307
4308	if ((inodedep->id_state & IOSTARTED) == 0)
4309		panic("handle_written_inodeblock: not started");
4310	inodedep->id_state &= ~IOSTARTED;
4311	if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) {
4312		fstype = UFS1;
4313		dp1 = (struct ufs1_dinode *)bp->b_data +
4314		    ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
4315	} else {
4316		fstype = UFS2;
4317		dp2 = (struct ufs2_dinode *)bp->b_data +
4318		    ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
4319	}
4320	/*
4321	 * If we had to rollback the inode allocation because of
4322	 * bitmaps being incomplete, then simply restore it.
4323	 * Keep the block dirty so that it will not be reclaimed until
4324	 * all associated dependencies have been cleared and the
4325	 * corresponding updates written to disk.
4326	 */
4327	if (inodedep->id_savedino1 != NULL) {
4328		if (fstype == UFS1)
4329			*dp1 = *inodedep->id_savedino1;
4330		else
4331			*dp2 = *inodedep->id_savedino2;
4332		FREE(inodedep->id_savedino1, M_SAVEDINO);
4333		inodedep->id_savedino1 = NULL;
4334		if ((bp->b_flags & B_DELWRI) == 0)
4335			stat_inode_bitmap++;
4336		bdirty(bp);
4337		return (1);
4338	}
4339	inodedep->id_state |= COMPLETE;
4340	/*
4341	 * Roll forward anything that had to be rolled back before
4342	 * the inode could be updated.
4343	 */
4344	hadchanges = 0;
4345	for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) {
4346		nextadp = TAILQ_NEXT(adp, ad_next);
4347		if (adp->ad_state & ATTACHED)
4348			panic("handle_written_inodeblock: new entry");
4349		if (fstype == UFS1) {
4350			if (adp->ad_lbn < NDADDR) {
4351				if (dp1->di_db[adp->ad_lbn]!=adp->ad_oldblkno)
4352					panic("%s %s #%jd mismatch %d != %jd",
4353					    "handle_written_inodeblock:",
4354					    "direct pointer",
4355					    (intmax_t)adp->ad_lbn,
4356					    dp1->di_db[adp->ad_lbn],
4357					    (intmax_t)adp->ad_oldblkno);
4358				dp1->di_db[adp->ad_lbn] = adp->ad_newblkno;
4359			} else {
4360				if (dp1->di_ib[adp->ad_lbn - NDADDR] != 0)
4361					panic("%s: %s #%jd allocated as %d",
4362					    "handle_written_inodeblock",
4363					    "indirect pointer",
4364					    (intmax_t)adp->ad_lbn - NDADDR,
4365					    dp1->di_ib[adp->ad_lbn - NDADDR]);
4366				dp1->di_ib[adp->ad_lbn - NDADDR] =
4367				    adp->ad_newblkno;
4368			}
4369		} else {
4370			if (adp->ad_lbn < NDADDR) {
4371				if (dp2->di_db[adp->ad_lbn]!=adp->ad_oldblkno)
4372					panic("%s: %s #%jd %s %jd != %jd",
4373					    "handle_written_inodeblock",
4374					    "direct pointer",
4375					    (intmax_t)adp->ad_lbn, "mismatch",
4376					    (intmax_t)dp2->di_db[adp->ad_lbn],
4377					    (intmax_t)adp->ad_oldblkno);
4378				dp2->di_db[adp->ad_lbn] = adp->ad_newblkno;
4379			} else {
4380				if (dp2->di_ib[adp->ad_lbn - NDADDR] != 0)
4381					panic("%s: %s #%jd allocated as %jd",
4382					    "handle_written_inodeblock",
4383					    "indirect pointer",
4384					    (intmax_t)adp->ad_lbn - NDADDR,
4385					    (intmax_t)
4386					    dp2->di_ib[adp->ad_lbn - NDADDR]);
4387				dp2->di_ib[adp->ad_lbn - NDADDR] =
4388				    adp->ad_newblkno;
4389			}
4390		}
4391		adp->ad_state &= ~UNDONE;
4392		adp->ad_state |= ATTACHED;
4393		hadchanges = 1;
4394	}
4395	for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) {
4396		nextadp = TAILQ_NEXT(adp, ad_next);
4397		if (adp->ad_state & ATTACHED)
4398			panic("handle_written_inodeblock: new entry");
4399		if (dp2->di_extb[adp->ad_lbn] != adp->ad_oldblkno)
4400			panic("%s: direct pointers #%jd %s %jd != %jd",
4401			    "handle_written_inodeblock",
4402			    (intmax_t)adp->ad_lbn, "mismatch",
4403			    (intmax_t)dp2->di_extb[adp->ad_lbn],
4404			    (intmax_t)adp->ad_oldblkno);
4405		dp2->di_extb[adp->ad_lbn] = adp->ad_newblkno;
4406		adp->ad_state &= ~UNDONE;
4407		adp->ad_state |= ATTACHED;
4408		hadchanges = 1;
4409	}
4410	if (hadchanges && (bp->b_flags & B_DELWRI) == 0)
4411		stat_direct_blk_ptrs++;
4412	/*
4413	 * Reset the file size to its most up-to-date value.
4414	 */
4415	if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1)
4416		panic("handle_written_inodeblock: bad size");
4417	if (fstype == UFS1) {
4418		if (dp1->di_size != inodedep->id_savedsize) {
4419			dp1->di_size = inodedep->id_savedsize;
4420			hadchanges = 1;
4421		}
4422	} else {
4423		if (dp2->di_size != inodedep->id_savedsize) {
4424			dp2->di_size = inodedep->id_savedsize;
4425			hadchanges = 1;
4426		}
4427		if (dp2->di_extsize != inodedep->id_savedextsize) {
4428			dp2->di_extsize = inodedep->id_savedextsize;
4429			hadchanges = 1;
4430		}
4431	}
4432	inodedep->id_savedsize = -1;
4433	inodedep->id_savedextsize = -1;
4434	/*
4435	 * If there were any rollbacks in the inode block, then it must be
4436	 * marked dirty so that its will eventually get written back in
4437	 * its correct form.
4438	 */
4439	if (hadchanges)
4440		bdirty(bp);
4441	/*
4442	 * Process any allocdirects that completed during the update.
4443	 */
4444	if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL)
4445		handle_allocdirect_partdone(adp);
4446	if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL)
4447		handle_allocdirect_partdone(adp);
4448	/*
4449	 * Process deallocations that were held pending until the
4450	 * inode had been written to disk. Freeing of the inode
4451	 * is delayed until after all blocks have been freed to
4452	 * avoid creation of new <vfsid, inum, lbn> triples
4453	 * before the old ones have been deleted.
4454	 */
4455	filefree = NULL;
4456	while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) {
4457		WORKLIST_REMOVE(wk);
4458		switch (wk->wk_type) {
4459
4460		case D_FREEFILE:
4461			/*
4462			 * We defer adding filefree to the worklist until
4463			 * all other additions have been made to ensure
4464			 * that it will be done after all the old blocks
4465			 * have been freed.
4466			 */
4467			if (filefree != NULL)
4468				panic("handle_written_inodeblock: filefree");
4469			filefree = wk;
4470			continue;
4471
4472		case D_MKDIR:
4473			handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT);
4474			continue;
4475
4476		case D_DIRADD:
4477			diradd_inode_written(WK_DIRADD(wk), inodedep);
4478			continue;
4479
4480		case D_FREEBLKS:
4481			wk->wk_state |= COMPLETE;
4482			if ((wk->wk_state  & ALLCOMPLETE) != ALLCOMPLETE)
4483				continue;
4484			 /* -- fall through -- */
4485		case D_FREEFRAG:
4486		case D_DIRREM:
4487			add_to_worklist(wk);
4488			continue;
4489
4490		case D_NEWDIRBLK:
4491			free_newdirblk(WK_NEWDIRBLK(wk));
4492			continue;
4493
4494		default:
4495			panic("handle_written_inodeblock: Unknown type %s",
4496			    TYPENAME(wk->wk_type));
4497			/* NOTREACHED */
4498		}
4499	}
4500	if (filefree != NULL) {
4501		if (free_inodedep(inodedep) == 0)
4502			panic("handle_written_inodeblock: live inodedep");
4503		add_to_worklist(filefree);
4504		return (0);
4505	}
4506
4507	/*
4508	 * If no outstanding dependencies, free it.
4509	 */
4510	if (free_inodedep(inodedep) ||
4511	    (TAILQ_FIRST(&inodedep->id_inoupdt) == 0 &&
4512	     TAILQ_FIRST(&inodedep->id_extupdt) == 0))
4513		return (0);
4514	return (hadchanges);
4515}
4516
4517/*
4518 * Process a diradd entry after its dependent inode has been written.
4519 * This routine must be called with splbio interrupts blocked.
4520 */
4521static void
4522diradd_inode_written(dap, inodedep)
4523	struct diradd *dap;
4524	struct inodedep *inodedep;
4525{
4526	struct pagedep *pagedep;
4527
4528	dap->da_state |= COMPLETE;
4529	if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
4530		if (dap->da_state & DIRCHG)
4531			pagedep = dap->da_previous->dm_pagedep;
4532		else
4533			pagedep = dap->da_pagedep;
4534		LIST_REMOVE(dap, da_pdlist);
4535		LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
4536	}
4537	WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
4538}
4539
4540/*
4541 * Handle the completion of a mkdir dependency.
4542 */
4543static void
4544handle_written_mkdir(mkdir, type)
4545	struct mkdir *mkdir;
4546	int type;
4547{
4548	struct diradd *dap;
4549	struct pagedep *pagedep;
4550
4551	if (mkdir->md_state != type)
4552		panic("handle_written_mkdir: bad type");
4553	dap = mkdir->md_diradd;
4554	dap->da_state &= ~type;
4555	if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0)
4556		dap->da_state |= DEPCOMPLETE;
4557	if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
4558		if (dap->da_state & DIRCHG)
4559			pagedep = dap->da_previous->dm_pagedep;
4560		else
4561			pagedep = dap->da_pagedep;
4562		LIST_REMOVE(dap, da_pdlist);
4563		LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
4564	}
4565	LIST_REMOVE(mkdir, md_mkdirs);
4566	WORKITEM_FREE(mkdir, D_MKDIR);
4567}
4568
4569/*
4570 * Called from within softdep_disk_write_complete above.
4571 * A write operation was just completed. Removed inodes can
4572 * now be freed and associated block pointers may be committed.
4573 * Note that this routine is always called from interrupt level
4574 * with further splbio interrupts blocked.
4575 */
4576static int
4577handle_written_filepage(pagedep, bp)
4578	struct pagedep *pagedep;
4579	struct buf *bp;		/* buffer containing the written page */
4580{
4581	struct dirrem *dirrem;
4582	struct diradd *dap, *nextdap;
4583	struct direct *ep;
4584	int i, chgs;
4585
4586	if ((pagedep->pd_state & IOSTARTED) == 0)
4587		panic("handle_written_filepage: not started");
4588	pagedep->pd_state &= ~IOSTARTED;
4589	/*
4590	 * Process any directory removals that have been committed.
4591	 */
4592	while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) {
4593		LIST_REMOVE(dirrem, dm_next);
4594		dirrem->dm_dirinum = pagedep->pd_ino;
4595		add_to_worklist(&dirrem->dm_list);
4596	}
4597	/*
4598	 * Free any directory additions that have been committed.
4599	 * If it is a newly allocated block, we have to wait until
4600	 * the on-disk directory inode claims the new block.
4601	 */
4602	if ((pagedep->pd_state & NEWBLOCK) == 0)
4603		while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
4604			free_diradd(dap);
4605	/*
4606	 * Uncommitted directory entries must be restored.
4607	 */
4608	for (chgs = 0, i = 0; i < DAHASHSZ; i++) {
4609		for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap;
4610		     dap = nextdap) {
4611			nextdap = LIST_NEXT(dap, da_pdlist);
4612			if (dap->da_state & ATTACHED)
4613				panic("handle_written_filepage: attached");
4614			ep = (struct direct *)
4615			    ((char *)bp->b_data + dap->da_offset);
4616			ep->d_ino = dap->da_newinum;
4617			dap->da_state &= ~UNDONE;
4618			dap->da_state |= ATTACHED;
4619			chgs = 1;
4620			/*
4621			 * If the inode referenced by the directory has
4622			 * been written out, then the dependency can be
4623			 * moved to the pending list.
4624			 */
4625			if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
4626				LIST_REMOVE(dap, da_pdlist);
4627				LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap,
4628				    da_pdlist);
4629			}
4630		}
4631	}
4632	/*
4633	 * If there were any rollbacks in the directory, then it must be
4634	 * marked dirty so that its will eventually get written back in
4635	 * its correct form.
4636	 */
4637	if (chgs) {
4638		if ((bp->b_flags & B_DELWRI) == 0)
4639			stat_dir_entry++;
4640		bdirty(bp);
4641		return (1);
4642	}
4643	/*
4644	 * If we are not waiting for a new directory block to be
4645	 * claimed by its inode, then the pagedep will be freed.
4646	 * Otherwise it will remain to track any new entries on
4647	 * the page in case they are fsync'ed.
4648	 */
4649	if ((pagedep->pd_state & NEWBLOCK) == 0) {
4650		LIST_REMOVE(pagedep, pd_hash);
4651		WORKITEM_FREE(pagedep, D_PAGEDEP);
4652	}
4653	return (0);
4654}
4655
4656/*
4657 * Writing back in-core inode structures.
4658 *
4659 * The filesystem only accesses an inode's contents when it occupies an
4660 * "in-core" inode structure.  These "in-core" structures are separate from
4661 * the page frames used to cache inode blocks.  Only the latter are
4662 * transferred to/from the disk.  So, when the updated contents of the
4663 * "in-core" inode structure are copied to the corresponding in-memory inode
4664 * block, the dependencies are also transferred.  The following procedure is
4665 * called when copying a dirty "in-core" inode to a cached inode block.
4666 */
4667
4668/*
4669 * Called when an inode is loaded from disk. If the effective link count
4670 * differed from the actual link count when it was last flushed, then we
4671 * need to ensure that the correct effective link count is put back.
4672 */
4673void
4674softdep_load_inodeblock(ip)
4675	struct inode *ip;	/* the "in_core" copy of the inode */
4676{
4677	struct inodedep *inodedep;
4678
4679	/*
4680	 * Check for alternate nlink count.
4681	 */
4682	ip->i_effnlink = ip->i_nlink;
4683	ACQUIRE_LOCK(&lk);
4684	if (inodedep_lookup(ip->i_fs, ip->i_number, 0, &inodedep) == 0) {
4685		FREE_LOCK(&lk);
4686		return;
4687	}
4688	ip->i_effnlink -= inodedep->id_nlinkdelta;
4689	if (inodedep->id_state & SPACECOUNTED)
4690		ip->i_flag |= IN_SPACECOUNTED;
4691	FREE_LOCK(&lk);
4692}
4693
4694/*
4695 * This routine is called just before the "in-core" inode
4696 * information is to be copied to the in-memory inode block.
4697 * Recall that an inode block contains several inodes. If
4698 * the force flag is set, then the dependencies will be
4699 * cleared so that the update can always be made. Note that
4700 * the buffer is locked when this routine is called, so we
4701 * will never be in the middle of writing the inode block
4702 * to disk.
4703 */
4704void
4705softdep_update_inodeblock(ip, bp, waitfor)
4706	struct inode *ip;	/* the "in_core" copy of the inode */
4707	struct buf *bp;		/* the buffer containing the inode block */
4708	int waitfor;		/* nonzero => update must be allowed */
4709{
4710	struct inodedep *inodedep;
4711	struct worklist *wk;
4712	struct buf *ibp;
4713	int error;
4714
4715	/*
4716	 * If the effective link count is not equal to the actual link
4717	 * count, then we must track the difference in an inodedep while
4718	 * the inode is (potentially) tossed out of the cache. Otherwise,
4719	 * if there is no existing inodedep, then there are no dependencies
4720	 * to track.
4721	 */
4722	ACQUIRE_LOCK(&lk);
4723	if (inodedep_lookup(ip->i_fs, ip->i_number, 0, &inodedep) == 0) {
4724		FREE_LOCK(&lk);
4725		if (ip->i_effnlink != ip->i_nlink)
4726			panic("softdep_update_inodeblock: bad link count");
4727		return;
4728	}
4729	if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink)
4730		panic("softdep_update_inodeblock: bad delta");
4731	/*
4732	 * Changes have been initiated. Anything depending on these
4733	 * changes cannot occur until this inode has been written.
4734	 */
4735	inodedep->id_state &= ~COMPLETE;
4736	if ((inodedep->id_state & ONWORKLIST) == 0)
4737		WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list);
4738	/*
4739	 * Any new dependencies associated with the incore inode must
4740	 * now be moved to the list associated with the buffer holding
4741	 * the in-memory copy of the inode. Once merged process any
4742	 * allocdirects that are completed by the merger.
4743	 */
4744	merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt);
4745	if (TAILQ_FIRST(&inodedep->id_inoupdt) != NULL)
4746		handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt));
4747	merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt);
4748	if (TAILQ_FIRST(&inodedep->id_extupdt) != NULL)
4749		handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt));
4750	/*
4751	 * Now that the inode has been pushed into the buffer, the
4752	 * operations dependent on the inode being written to disk
4753	 * can be moved to the id_bufwait so that they will be
4754	 * processed when the buffer I/O completes.
4755	 */
4756	while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) {
4757		WORKLIST_REMOVE(wk);
4758		WORKLIST_INSERT(&inodedep->id_bufwait, wk);
4759	}
4760	/*
4761	 * Newly allocated inodes cannot be written until the bitmap
4762	 * that allocates them have been written (indicated by
4763	 * DEPCOMPLETE being set in id_state). If we are doing a
4764	 * forced sync (e.g., an fsync on a file), we force the bitmap
4765	 * to be written so that the update can be done.
4766	 */
4767	if (waitfor == 0) {
4768		FREE_LOCK(&lk);
4769		return;
4770	}
4771retry:
4772	if ((inodedep->id_state & DEPCOMPLETE) != 0) {
4773		FREE_LOCK(&lk);
4774		return;
4775	}
4776	ibp = inodedep->id_buf;
4777	ibp = getdirtybuf(ibp, &lk, MNT_WAIT);
4778	if (ibp == NULL) {
4779		/*
4780		 * If ibp came back as NULL, the dependency could have been
4781		 * freed while we slept.  Look it up again, and check to see
4782		 * that it has completed.
4783		 */
4784		if (inodedep_lookup(ip->i_fs, ip->i_number, 0, &inodedep) != 0)
4785			goto retry;
4786		FREE_LOCK(&lk);
4787		return;
4788	}
4789	FREE_LOCK(&lk);
4790	if ((error = bwrite(ibp)) != 0)
4791		softdep_error("softdep_update_inodeblock: bwrite", error);
4792}
4793
4794/*
4795 * Merge the a new inode dependency list (such as id_newinoupdt) into an
4796 * old inode dependency list (such as id_inoupdt). This routine must be
4797 * called with splbio interrupts blocked.
4798 */
4799static void
4800merge_inode_lists(newlisthead, oldlisthead)
4801	struct allocdirectlst *newlisthead;
4802	struct allocdirectlst *oldlisthead;
4803{
4804	struct allocdirect *listadp, *newadp;
4805
4806	newadp = TAILQ_FIRST(newlisthead);
4807	for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) {
4808		if (listadp->ad_lbn < newadp->ad_lbn) {
4809			listadp = TAILQ_NEXT(listadp, ad_next);
4810			continue;
4811		}
4812		TAILQ_REMOVE(newlisthead, newadp, ad_next);
4813		TAILQ_INSERT_BEFORE(listadp, newadp, ad_next);
4814		if (listadp->ad_lbn == newadp->ad_lbn) {
4815			allocdirect_merge(oldlisthead, newadp,
4816			    listadp);
4817			listadp = newadp;
4818		}
4819		newadp = TAILQ_FIRST(newlisthead);
4820	}
4821	while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) {
4822		TAILQ_REMOVE(newlisthead, newadp, ad_next);
4823		TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next);
4824	}
4825}
4826
4827/*
4828 * If we are doing an fsync, then we must ensure that any directory
4829 * entries for the inode have been written after the inode gets to disk.
4830 */
4831int
4832softdep_fsync(vp)
4833	struct vnode *vp;	/* the "in_core" copy of the inode */
4834{
4835	struct inodedep *inodedep;
4836	struct pagedep *pagedep;
4837	struct worklist *wk;
4838	struct diradd *dap;
4839	struct mount *mnt;
4840	struct vnode *pvp;
4841	struct inode *ip;
4842	struct buf *bp;
4843	struct fs *fs;
4844	struct thread *td = curthread;
4845	int error, flushparent;
4846	ino_t parentino;
4847	ufs_lbn_t lbn;
4848
4849	ip = VTOI(vp);
4850	fs = ip->i_fs;
4851	ACQUIRE_LOCK(&lk);
4852	if (inodedep_lookup(fs, ip->i_number, 0, &inodedep) == 0) {
4853		FREE_LOCK(&lk);
4854		return (0);
4855	}
4856	if (LIST_FIRST(&inodedep->id_inowait) != NULL ||
4857	    LIST_FIRST(&inodedep->id_bufwait) != NULL ||
4858	    TAILQ_FIRST(&inodedep->id_extupdt) != NULL ||
4859	    TAILQ_FIRST(&inodedep->id_newextupdt) != NULL ||
4860	    TAILQ_FIRST(&inodedep->id_inoupdt) != NULL ||
4861	    TAILQ_FIRST(&inodedep->id_newinoupdt) != NULL)
4862		panic("softdep_fsync: pending ops");
4863	for (error = 0, flushparent = 0; ; ) {
4864		if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL)
4865			break;
4866		if (wk->wk_type != D_DIRADD)
4867			panic("softdep_fsync: Unexpected type %s",
4868			    TYPENAME(wk->wk_type));
4869		dap = WK_DIRADD(wk);
4870		/*
4871		 * Flush our parent if this directory entry has a MKDIR_PARENT
4872		 * dependency or is contained in a newly allocated block.
4873		 */
4874		if (dap->da_state & DIRCHG)
4875			pagedep = dap->da_previous->dm_pagedep;
4876		else
4877			pagedep = dap->da_pagedep;
4878		mnt = pagedep->pd_mnt;
4879		parentino = pagedep->pd_ino;
4880		lbn = pagedep->pd_lbn;
4881		if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE)
4882			panic("softdep_fsync: dirty");
4883		if ((dap->da_state & MKDIR_PARENT) ||
4884		    (pagedep->pd_state & NEWBLOCK))
4885			flushparent = 1;
4886		else
4887			flushparent = 0;
4888		/*
4889		 * If we are being fsync'ed as part of vgone'ing this vnode,
4890		 * then we will not be able to release and recover the
4891		 * vnode below, so we just have to give up on writing its
4892		 * directory entry out. It will eventually be written, just
4893		 * not now, but then the user was not asking to have it
4894		 * written, so we are not breaking any promises.
4895		 */
4896		if (vp->v_iflag & VI_DOOMED)
4897			break;
4898		/*
4899		 * We prevent deadlock by always fetching inodes from the
4900		 * root, moving down the directory tree. Thus, when fetching
4901		 * our parent directory, we first try to get the lock. If
4902		 * that fails, we must unlock ourselves before requesting
4903		 * the lock on our parent. See the comment in ufs_lookup
4904		 * for details on possible races.
4905		 */
4906		FREE_LOCK(&lk);
4907		if (ffs_vget(mnt, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp)) {
4908			VOP_UNLOCK(vp, 0, td);
4909			error = ffs_vget(mnt, parentino, LK_EXCLUSIVE, &pvp);
4910			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
4911			if (error != 0)
4912				return (error);
4913		}
4914		/*
4915		 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps
4916		 * that are contained in direct blocks will be resolved by
4917		 * doing a ffs_update. Pagedeps contained in indirect blocks
4918		 * may require a complete sync'ing of the directory. So, we
4919		 * try the cheap and fast ffs_update first, and if that fails,
4920		 * then we do the slower ffs_syncvnode of the directory.
4921		 */
4922		if (flushparent) {
4923			if ((error = ffs_update(pvp, 1)) != 0) {
4924				vput(pvp);
4925				return (error);
4926			}
4927			if ((pagedep->pd_state & NEWBLOCK) &&
4928			    (error = ffs_syncvnode(pvp, MNT_WAIT))) {
4929				vput(pvp);
4930				return (error);
4931			}
4932		}
4933		/*
4934		 * Flush directory page containing the inode's name.
4935		 */
4936		error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred,
4937		    &bp);
4938		if (error == 0)
4939			error = bwrite(bp);
4940		else
4941			brelse(bp);
4942		vput(pvp);
4943		if (error != 0)
4944			return (error);
4945		ACQUIRE_LOCK(&lk);
4946		if (inodedep_lookup(fs, ip->i_number, 0, &inodedep) == 0)
4947			break;
4948	}
4949	FREE_LOCK(&lk);
4950	return (0);
4951}
4952
4953/*
4954 * Flush all the dirty bitmaps associated with the block device
4955 * before flushing the rest of the dirty blocks so as to reduce
4956 * the number of dependencies that will have to be rolled back.
4957 */
4958void
4959softdep_fsync_mountdev(vp)
4960	struct vnode *vp;
4961{
4962	struct buf *bp, *nbp;
4963	struct worklist *wk;
4964
4965	if (!vn_isdisk(vp, NULL))
4966		panic("softdep_fsync_mountdev: vnode not a disk");
4967restart:
4968	ACQUIRE_LOCK(&lk);
4969	VI_LOCK(vp);
4970	TAILQ_FOREACH_SAFE(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs, nbp) {
4971		/*
4972		 * If it is already scheduled, skip to the next buffer.
4973		 */
4974		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL))
4975			continue;
4976
4977		if ((bp->b_flags & B_DELWRI) == 0)
4978			panic("softdep_fsync_mountdev: not dirty");
4979		/*
4980		 * We are only interested in bitmaps with outstanding
4981		 * dependencies.
4982		 */
4983		if ((wk = LIST_FIRST(&bp->b_dep)) == NULL ||
4984		    wk->wk_type != D_BMSAFEMAP ||
4985		    (bp->b_vflags & BV_BKGRDINPROG)) {
4986			BUF_UNLOCK(bp);
4987			continue;
4988		}
4989		VI_UNLOCK(vp);
4990		FREE_LOCK(&lk);
4991		bremfree(bp);
4992		(void) bawrite(bp);
4993		goto restart;
4994	}
4995	FREE_LOCK(&lk);
4996	drain_output(vp);
4997	VI_UNLOCK(vp);
4998}
4999
5000/*
5001 * This routine is called when we are trying to synchronously flush a
5002 * file. This routine must eliminate any filesystem metadata dependencies
5003 * so that the syncing routine can succeed by pushing the dirty blocks
5004 * associated with the file. If any I/O errors occur, they are returned.
5005 */
5006int
5007softdep_sync_metadata(struct vnode *vp)
5008{
5009	struct pagedep *pagedep;
5010	struct allocdirect *adp;
5011	struct allocindir *aip;
5012	struct buf *bp, *nbp;
5013	struct worklist *wk;
5014	int i, error, waitfor;
5015
5016	if (!DOINGSOFTDEP(vp))
5017		return (0);
5018	/*
5019	 * Ensure that any direct block dependencies have been cleared.
5020	 */
5021	ACQUIRE_LOCK(&lk);
5022	if ((error = flush_inodedep_deps(VTOI(vp)->i_fs, VTOI(vp)->i_number))) {
5023		FREE_LOCK(&lk);
5024		return (error);
5025	}
5026	FREE_LOCK(&lk);
5027	/*
5028	 * For most files, the only metadata dependencies are the
5029	 * cylinder group maps that allocate their inode or blocks.
5030	 * The block allocation dependencies can be found by traversing
5031	 * the dependency lists for any buffers that remain on their
5032	 * dirty buffer list. The inode allocation dependency will
5033	 * be resolved when the inode is updated with MNT_WAIT.
5034	 * This work is done in two passes. The first pass grabs most
5035	 * of the buffers and begins asynchronously writing them. The
5036	 * only way to wait for these asynchronous writes is to sleep
5037	 * on the filesystem vnode which may stay busy for a long time
5038	 * if the filesystem is active. So, instead, we make a second
5039	 * pass over the dependencies blocking on each write. In the
5040	 * usual case we will be blocking against a write that we
5041	 * initiated, so when it is done the dependency will have been
5042	 * resolved. Thus the second pass is expected to end quickly.
5043	 */
5044	waitfor = MNT_NOWAIT;
5045
5046top:
5047	/*
5048	 * We must wait for any I/O in progress to finish so that
5049	 * all potential buffers on the dirty list will be visible.
5050	 */
5051	VI_LOCK(vp);
5052	drain_output(vp);
5053	while ((bp = TAILQ_FIRST(&vp->v_bufobj.bo_dirty.bv_hd)) != NULL) {
5054		bp = getdirtybuf(bp, VI_MTX(vp), MNT_WAIT);
5055		if (bp)
5056			break;
5057	}
5058	VI_UNLOCK(vp);
5059	if (bp == NULL)
5060		return (0);
5061loop:
5062	/* While syncing snapshots, we must allow recursive lookups */
5063	bp->b_lock.lk_flags |= LK_CANRECURSE;
5064	ACQUIRE_LOCK(&lk);
5065	/*
5066	 * As we hold the buffer locked, none of its dependencies
5067	 * will disappear.
5068	 */
5069	LIST_FOREACH(wk, &bp->b_dep, wk_list) {
5070		switch (wk->wk_type) {
5071
5072		case D_ALLOCDIRECT:
5073			adp = WK_ALLOCDIRECT(wk);
5074			if (adp->ad_state & DEPCOMPLETE)
5075				continue;
5076			nbp = adp->ad_buf;
5077			nbp = getdirtybuf(nbp, &lk, waitfor);
5078			if (nbp == NULL)
5079				continue;
5080			FREE_LOCK(&lk);
5081			if (waitfor == MNT_NOWAIT) {
5082				bawrite(nbp);
5083			} else if ((error = bwrite(nbp)) != 0) {
5084				break;
5085			}
5086			ACQUIRE_LOCK(&lk);
5087			continue;
5088
5089		case D_ALLOCINDIR:
5090			aip = WK_ALLOCINDIR(wk);
5091			if (aip->ai_state & DEPCOMPLETE)
5092				continue;
5093			nbp = aip->ai_buf;
5094			nbp = getdirtybuf(nbp, &lk, waitfor);
5095			if (nbp == NULL)
5096				continue;
5097			FREE_LOCK(&lk);
5098			if (waitfor == MNT_NOWAIT) {
5099				bawrite(nbp);
5100			} else if ((error = bwrite(nbp)) != 0) {
5101				break;
5102			}
5103			ACQUIRE_LOCK(&lk);
5104			continue;
5105
5106		case D_INDIRDEP:
5107		restart:
5108
5109			LIST_FOREACH(aip, &WK_INDIRDEP(wk)->ir_deplisthd, ai_next) {
5110				if (aip->ai_state & DEPCOMPLETE)
5111					continue;
5112				nbp = aip->ai_buf;
5113				nbp = getdirtybuf(nbp, &lk, MNT_WAIT);
5114				if (nbp == NULL)
5115					goto restart;
5116				FREE_LOCK(&lk);
5117				if ((error = bwrite(nbp)) != 0) {
5118					break;
5119				}
5120				ACQUIRE_LOCK(&lk);
5121				goto restart;
5122			}
5123			continue;
5124
5125		case D_INODEDEP:
5126			if ((error = flush_inodedep_deps(WK_INODEDEP(wk)->id_fs,
5127			    WK_INODEDEP(wk)->id_ino)) != 0) {
5128				FREE_LOCK(&lk);
5129				break;
5130			}
5131			continue;
5132
5133		case D_PAGEDEP:
5134			/*
5135			 * We are trying to sync a directory that may
5136			 * have dependencies on both its own metadata
5137			 * and/or dependencies on the inodes of any
5138			 * recently allocated files. We walk its diradd
5139			 * lists pushing out the associated inode.
5140			 */
5141			pagedep = WK_PAGEDEP(wk);
5142			for (i = 0; i < DAHASHSZ; i++) {
5143				if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0)
5144					continue;
5145				if ((error =
5146				    flush_pagedep_deps(vp, pagedep->pd_mnt,
5147						&pagedep->pd_diraddhd[i]))) {
5148					FREE_LOCK(&lk);
5149					break;
5150				}
5151			}
5152			continue;
5153
5154		case D_MKDIR:
5155			/*
5156			 * This case should never happen if the vnode has
5157			 * been properly sync'ed. However, if this function
5158			 * is used at a place where the vnode has not yet
5159			 * been sync'ed, this dependency can show up. So,
5160			 * rather than panic, just flush it.
5161			 */
5162			nbp = WK_MKDIR(wk)->md_buf;
5163			nbp = getdirtybuf(nbp, &lk, waitfor);
5164			if (nbp == NULL)
5165				continue;
5166			FREE_LOCK(&lk);
5167			if (waitfor == MNT_NOWAIT) {
5168				bawrite(nbp);
5169			} else if ((error = bwrite(nbp)) != 0) {
5170				break;
5171			}
5172			ACQUIRE_LOCK(&lk);
5173			continue;
5174
5175		case D_BMSAFEMAP:
5176			/*
5177			 * This case should never happen if the vnode has
5178			 * been properly sync'ed. However, if this function
5179			 * is used at a place where the vnode has not yet
5180			 * been sync'ed, this dependency can show up. So,
5181			 * rather than panic, just flush it.
5182			 */
5183			nbp = WK_BMSAFEMAP(wk)->sm_buf;
5184			nbp = getdirtybuf(nbp, &lk, waitfor);
5185			if (nbp == NULL)
5186				continue;
5187			FREE_LOCK(&lk);
5188			if (waitfor == MNT_NOWAIT) {
5189				bawrite(nbp);
5190			} else if ((error = bwrite(nbp)) != 0) {
5191				break;
5192			}
5193			ACQUIRE_LOCK(&lk);
5194			continue;
5195
5196		default:
5197			panic("softdep_sync_metadata: Unknown type %s",
5198			    TYPENAME(wk->wk_type));
5199			/* NOTREACHED */
5200		}
5201		/* We reach here only in error and unlocked */
5202		if (error == 0)
5203			panic("softdep_sync_metadata: zero error");
5204		bp->b_lock.lk_flags &= ~LK_CANRECURSE;
5205		bawrite(bp);
5206		return (error);
5207	}
5208	FREE_LOCK(&lk);
5209	VI_LOCK(vp);
5210	while ((nbp = TAILQ_NEXT(bp, b_bobufs)) != NULL) {
5211		nbp = getdirtybuf(nbp, VI_MTX(vp), MNT_WAIT);
5212		if (nbp)
5213			break;
5214	}
5215	VI_UNLOCK(vp);
5216	bp->b_lock.lk_flags &= ~LK_CANRECURSE;
5217	bawrite(bp);
5218	if (nbp != NULL) {
5219		bp = nbp;
5220		goto loop;
5221	}
5222	/*
5223	 * The brief unlock is to allow any pent up dependency
5224	 * processing to be done. Then proceed with the second pass.
5225	 */
5226	if (waitfor == MNT_NOWAIT) {
5227		waitfor = MNT_WAIT;
5228		goto top;
5229	}
5230
5231	/*
5232	 * If we have managed to get rid of all the dirty buffers,
5233	 * then we are done. For certain directories and block
5234	 * devices, we may need to do further work.
5235	 *
5236	 * We must wait for any I/O in progress to finish so that
5237	 * all potential buffers on the dirty list will be visible.
5238	 */
5239	VI_LOCK(vp);
5240	drain_output(vp);
5241	VI_UNLOCK(vp);
5242	return (0);
5243}
5244
5245/*
5246 * Flush the dependencies associated with an inodedep.
5247 * Called with splbio blocked.
5248 */
5249static int
5250flush_inodedep_deps(fs, ino)
5251	struct fs *fs;
5252	ino_t ino;
5253{
5254	struct inodedep *inodedep;
5255	int error, waitfor;
5256
5257	/*
5258	 * This work is done in two passes. The first pass grabs most
5259	 * of the buffers and begins asynchronously writing them. The
5260	 * only way to wait for these asynchronous writes is to sleep
5261	 * on the filesystem vnode which may stay busy for a long time
5262	 * if the filesystem is active. So, instead, we make a second
5263	 * pass over the dependencies blocking on each write. In the
5264	 * usual case we will be blocking against a write that we
5265	 * initiated, so when it is done the dependency will have been
5266	 * resolved. Thus the second pass is expected to end quickly.
5267	 * We give a brief window at the top of the loop to allow
5268	 * any pending I/O to complete.
5269	 */
5270	for (error = 0, waitfor = MNT_NOWAIT; ; ) {
5271		if (error)
5272			return (error);
5273		FREE_LOCK(&lk);
5274		ACQUIRE_LOCK(&lk);
5275		if (inodedep_lookup(fs, ino, 0, &inodedep) == 0)
5276			return (0);
5277		if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) ||
5278		    flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) ||
5279		    flush_deplist(&inodedep->id_extupdt, waitfor, &error) ||
5280		    flush_deplist(&inodedep->id_newextupdt, waitfor, &error))
5281			continue;
5282		/*
5283		 * If pass2, we are done, otherwise do pass 2.
5284		 */
5285		if (waitfor == MNT_WAIT)
5286			break;
5287		waitfor = MNT_WAIT;
5288	}
5289	/*
5290	 * Try freeing inodedep in case all dependencies have been removed.
5291	 */
5292	if (inodedep_lookup(fs, ino, 0, &inodedep) != 0)
5293		(void) free_inodedep(inodedep);
5294	return (0);
5295}
5296
5297/*
5298 * Flush an inode dependency list.
5299 * Called with splbio blocked.
5300 */
5301static int
5302flush_deplist(listhead, waitfor, errorp)
5303	struct allocdirectlst *listhead;
5304	int waitfor;
5305	int *errorp;
5306{
5307	struct allocdirect *adp;
5308	struct buf *bp;
5309
5310	mtx_assert(&lk, MA_OWNED);
5311	TAILQ_FOREACH(adp, listhead, ad_next) {
5312		if (adp->ad_state & DEPCOMPLETE)
5313			continue;
5314		bp = adp->ad_buf;
5315		bp = getdirtybuf(bp, &lk, waitfor);
5316		if (bp == NULL) {
5317			if (waitfor == MNT_NOWAIT)
5318				continue;
5319			return (1);
5320		}
5321		FREE_LOCK(&lk);
5322		if (waitfor == MNT_NOWAIT) {
5323			bawrite(bp);
5324		} else if ((*errorp = bwrite(bp)) != 0) {
5325			ACQUIRE_LOCK(&lk);
5326			return (1);
5327		}
5328		ACQUIRE_LOCK(&lk);
5329		return (1);
5330	}
5331	return (0);
5332}
5333
5334/*
5335 * Eliminate a pagedep dependency by flushing out all its diradd dependencies.
5336 * Called with splbio blocked.
5337 */
5338static int
5339flush_pagedep_deps(pvp, mp, diraddhdp)
5340	struct vnode *pvp;
5341	struct mount *mp;
5342	struct diraddhd *diraddhdp;
5343{
5344	struct inodedep *inodedep;
5345	struct ufsmount *ump;
5346	struct diradd *dap;
5347	struct vnode *vp;
5348	int error = 0;
5349	struct buf *bp;
5350	ino_t inum;
5351
5352	ump = VFSTOUFS(mp);
5353	while ((dap = LIST_FIRST(diraddhdp)) != NULL) {
5354		/*
5355		 * Flush ourselves if this directory entry
5356		 * has a MKDIR_PARENT dependency.
5357		 */
5358		if (dap->da_state & MKDIR_PARENT) {
5359			FREE_LOCK(&lk);
5360			if ((error = ffs_update(pvp, 1)) != 0)
5361				break;
5362			ACQUIRE_LOCK(&lk);
5363			/*
5364			 * If that cleared dependencies, go on to next.
5365			 */
5366			if (dap != LIST_FIRST(diraddhdp))
5367				continue;
5368			if (dap->da_state & MKDIR_PARENT)
5369				panic("flush_pagedep_deps: MKDIR_PARENT");
5370		}
5371		/*
5372		 * A newly allocated directory must have its "." and
5373		 * ".." entries written out before its name can be
5374		 * committed in its parent. We do not want or need
5375		 * the full semantics of a synchronous ffs_syncvnode as
5376		 * that may end up here again, once for each directory
5377		 * level in the filesystem. Instead, we push the blocks
5378		 * and wait for them to clear. We have to fsync twice
5379		 * because the first call may choose to defer blocks
5380		 * that still have dependencies, but deferral will
5381		 * happen at most once.
5382		 */
5383		inum = dap->da_newinum;
5384		if (dap->da_state & MKDIR_BODY) {
5385			FREE_LOCK(&lk);
5386			if ((error = ffs_vget(mp, inum, LK_EXCLUSIVE, &vp)))
5387				break;
5388			if ((error=ffs_syncvnode(vp, MNT_NOWAIT)) ||
5389			    (error=ffs_syncvnode(vp, MNT_NOWAIT))) {
5390				vput(vp);
5391				break;
5392			}
5393			VI_LOCK(vp);
5394			drain_output(vp);
5395			VI_UNLOCK(vp);
5396			vput(vp);
5397			ACQUIRE_LOCK(&lk);
5398			/*
5399			 * If that cleared dependencies, go on to next.
5400			 */
5401			if (dap != LIST_FIRST(diraddhdp))
5402				continue;
5403			if (dap->da_state & MKDIR_BODY)
5404				panic("flush_pagedep_deps: MKDIR_BODY");
5405		}
5406		/*
5407		 * Flush the inode on which the directory entry depends.
5408		 * Having accounted for MKDIR_PARENT and MKDIR_BODY above,
5409		 * the only remaining dependency is that the updated inode
5410		 * count must get pushed to disk. The inode has already
5411		 * been pushed into its inode buffer (via VOP_UPDATE) at
5412		 * the time of the reference count change. So we need only
5413		 * locate that buffer, ensure that there will be no rollback
5414		 * caused by a bitmap dependency, then write the inode buffer.
5415		 */
5416retry:
5417		if (inodedep_lookup(ump->um_fs, inum, 0, &inodedep) == 0)
5418			panic("flush_pagedep_deps: lost inode");
5419		/*
5420		 * If the inode still has bitmap dependencies,
5421		 * push them to disk.
5422		 */
5423		if ((inodedep->id_state & DEPCOMPLETE) == 0) {
5424			bp = inodedep->id_buf;
5425			bp = getdirtybuf(bp, &lk, MNT_WAIT);
5426			if (bp == NULL)
5427				goto retry;
5428			FREE_LOCK(&lk);
5429			if ((error = bwrite(bp)) != 0)
5430				break;
5431			ACQUIRE_LOCK(&lk);
5432			if (dap != LIST_FIRST(diraddhdp))
5433				continue;
5434		}
5435		/*
5436		 * If the inode is still sitting in a buffer waiting
5437		 * to be written, push it to disk.
5438		 */
5439		FREE_LOCK(&lk);
5440		if ((error = bread(ump->um_devvp,
5441		    fsbtodb(ump->um_fs, ino_to_fsba(ump->um_fs, inum)),
5442		    (int)ump->um_fs->fs_bsize, NOCRED, &bp)) != 0) {
5443			brelse(bp);
5444			break;
5445		}
5446		if ((error = bwrite(bp)) != 0)
5447			break;
5448		ACQUIRE_LOCK(&lk);
5449		/*
5450		 * If we have failed to get rid of all the dependencies
5451		 * then something is seriously wrong.
5452		 */
5453		if (dap == LIST_FIRST(diraddhdp))
5454			panic("flush_pagedep_deps: flush failed");
5455	}
5456	if (error)
5457		ACQUIRE_LOCK(&lk);
5458	return (error);
5459}
5460
5461/*
5462 * A large burst of file addition or deletion activity can drive the
5463 * memory load excessively high. First attempt to slow things down
5464 * using the techniques below. If that fails, this routine requests
5465 * the offending operations to fall back to running synchronously
5466 * until the memory load returns to a reasonable level.
5467 */
5468int
5469softdep_slowdown(vp)
5470	struct vnode *vp;
5471{
5472	int max_softdeps_hard;
5473
5474	max_softdeps_hard = max_softdeps * 11 / 10;
5475	if (num_dirrem < max_softdeps_hard / 2 &&
5476	    num_inodedep < max_softdeps_hard &&
5477	    VFSTOUFS(vp->v_mount)->um_numindirdeps < maxindirdeps)
5478  		return (0);
5479	if (VFSTOUFS(vp->v_mount)->um_numindirdeps >= maxindirdeps)
5480		speedup_syncer();
5481	stat_sync_limit_hit += 1;
5482	return (1);
5483}
5484
5485/*
5486 * Called by the allocation routines when they are about to fail
5487 * in the hope that we can free up some disk space.
5488 *
5489 * First check to see if the work list has anything on it. If it has,
5490 * clean up entries until we successfully free some space. Because this
5491 * process holds inodes locked, we cannot handle any remove requests
5492 * that might block on a locked inode as that could lead to deadlock.
5493 * If the worklist yields no free space, encourage the syncer daemon
5494 * to help us. In no event will we try for longer than tickdelay seconds.
5495 */
5496int
5497softdep_request_cleanup(fs, vp)
5498	struct fs *fs;
5499	struct vnode *vp;
5500{
5501	struct ufsmount *ump;
5502	long starttime;
5503	ufs2_daddr_t needed;
5504	int error;
5505
5506	ump = VTOI(vp)->i_ump;
5507	mtx_assert(UFS_MTX(ump), MA_OWNED);
5508	needed = fs->fs_cstotal.cs_nbfree + fs->fs_contigsumsize;
5509	starttime = time_second + tickdelay;
5510	/*
5511	 * If we are being called because of a process doing a
5512	 * copy-on-write, then it is not safe to update the vnode
5513	 * as we may recurse into the copy-on-write routine.
5514	 */
5515	if (!(curthread->td_pflags & TDP_COWINPROGRESS)) {
5516		UFS_UNLOCK(ump);
5517		error = ffs_update(vp, 1);
5518		UFS_LOCK(ump);
5519		if (error != 0)
5520			return (0);
5521	}
5522	while (fs->fs_pendingblocks > 0 && fs->fs_cstotal.cs_nbfree <= needed) {
5523		if (time_second > starttime)
5524			return (0);
5525		UFS_UNLOCK(ump);
5526		ACQUIRE_LOCK(&lk);
5527		if (num_on_worklist > 0 &&
5528		    process_worklist_item(NULL, LK_NOWAIT) != -1) {
5529			stat_worklist_push += 1;
5530			FREE_LOCK(&lk);
5531			UFS_LOCK(ump);
5532			continue;
5533		}
5534		request_cleanup(FLUSH_REMOVE_WAIT);
5535		FREE_LOCK(&lk);
5536		UFS_LOCK(ump);
5537	}
5538	return (1);
5539}
5540
5541/*
5542 * If memory utilization has gotten too high, deliberately slow things
5543 * down and speed up the I/O processing.
5544 */
5545static int
5546request_cleanup(resource)
5547	int resource;
5548{
5549	struct thread *td = curthread;
5550
5551	mtx_assert(&lk, MA_OWNED);
5552	/*
5553	 * We never hold up the filesystem syncer process.
5554	 */
5555	if (td == filesys_syncer || (td->td_pflags & TDP_SOFTDEP))
5556		return (0);
5557	/*
5558	 * First check to see if the work list has gotten backlogged.
5559	 * If it has, co-opt this process to help clean up two entries.
5560	 * Because this process may hold inodes locked, we cannot
5561	 * handle any remove requests that might block on a locked
5562	 * inode as that could lead to deadlock.  We set TDP_SOFTDEP
5563	 * to avoid recursively processing the worklist.
5564	 */
5565	if (num_on_worklist > max_softdeps / 10) {
5566		td->td_pflags |= TDP_SOFTDEP;
5567		process_worklist_item(NULL, LK_NOWAIT);
5568		process_worklist_item(NULL, LK_NOWAIT);
5569		td->td_pflags &= ~TDP_SOFTDEP;
5570		stat_worklist_push += 2;
5571		return(1);
5572	}
5573	/*
5574	 * Next, we attempt to speed up the syncer process. If that
5575	 * is successful, then we allow the process to continue.
5576	 */
5577	if (speedup_syncer() && resource != FLUSH_REMOVE_WAIT)
5578		return(0);
5579	/*
5580	 * If we are resource constrained on inode dependencies, try
5581	 * flushing some dirty inodes. Otherwise, we are constrained
5582	 * by file deletions, so try accelerating flushes of directories
5583	 * with removal dependencies. We would like to do the cleanup
5584	 * here, but we probably hold an inode locked at this point and
5585	 * that might deadlock against one that we try to clean. So,
5586	 * the best that we can do is request the syncer daemon to do
5587	 * the cleanup for us.
5588	 */
5589	switch (resource) {
5590
5591	case FLUSH_INODES:
5592		stat_ino_limit_push += 1;
5593		req_clear_inodedeps += 1;
5594		stat_countp = &stat_ino_limit_hit;
5595		break;
5596
5597	case FLUSH_REMOVE:
5598	case FLUSH_REMOVE_WAIT:
5599		stat_blk_limit_push += 1;
5600		req_clear_remove += 1;
5601		stat_countp = &stat_blk_limit_hit;
5602		break;
5603
5604	default:
5605		panic("request_cleanup: unknown type");
5606	}
5607	/*
5608	 * Hopefully the syncer daemon will catch up and awaken us.
5609	 * We wait at most tickdelay before proceeding in any case.
5610	 */
5611	proc_waiting += 1;
5612	if (handle.callout == NULL)
5613		handle = timeout(pause_timer, 0, tickdelay > 2 ? tickdelay : 2);
5614	msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0);
5615	proc_waiting -= 1;
5616	return (1);
5617}
5618
5619/*
5620 * Awaken processes pausing in request_cleanup and clear proc_waiting
5621 * to indicate that there is no longer a timer running.
5622 */
5623static void
5624pause_timer(arg)
5625	void *arg;
5626{
5627
5628	ACQUIRE_LOCK(&lk);
5629	*stat_countp += 1;
5630	wakeup_one(&proc_waiting);
5631	if (proc_waiting > 0)
5632		handle = timeout(pause_timer, 0, tickdelay > 2 ? tickdelay : 2);
5633	else
5634		handle.callout = NULL;
5635	FREE_LOCK(&lk);
5636}
5637
5638/*
5639 * Flush out a directory with at least one removal dependency in an effort to
5640 * reduce the number of dirrem, freefile, and freeblks dependency structures.
5641 */
5642static void
5643clear_remove(td)
5644	struct thread *td;
5645{
5646	struct pagedep_hashhead *pagedephd;
5647	struct pagedep *pagedep;
5648	static int next = 0;
5649	struct mount *mp;
5650	struct vnode *vp;
5651	int error, cnt;
5652	ino_t ino;
5653
5654	mtx_assert(&lk, MA_OWNED);
5655
5656	for (cnt = 0; cnt < pagedep_hash; cnt++) {
5657		pagedephd = &pagedep_hashtbl[next++];
5658		if (next >= pagedep_hash)
5659			next = 0;
5660		LIST_FOREACH(pagedep, pagedephd, pd_hash) {
5661			if (LIST_FIRST(&pagedep->pd_dirremhd) == NULL)
5662				continue;
5663			mp = pagedep->pd_mnt;
5664			ino = pagedep->pd_ino;
5665			if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
5666				continue;
5667			FREE_LOCK(&lk);
5668			if ((error = ffs_vget(mp, ino, LK_EXCLUSIVE, &vp))) {
5669				softdep_error("clear_remove: vget", error);
5670				vn_finished_write(mp);
5671				ACQUIRE_LOCK(&lk);
5672				return;
5673			}
5674			if ((error = ffs_syncvnode(vp, MNT_NOWAIT)))
5675				softdep_error("clear_remove: fsync", error);
5676			VI_LOCK(vp);
5677			drain_output(vp);
5678			VI_UNLOCK(vp);
5679			vput(vp);
5680			vn_finished_write(mp);
5681			ACQUIRE_LOCK(&lk);
5682			return;
5683		}
5684	}
5685}
5686
5687/*
5688 * Clear out a block of dirty inodes in an effort to reduce
5689 * the number of inodedep dependency structures.
5690 */
5691static void
5692clear_inodedeps(td)
5693	struct thread *td;
5694{
5695	struct inodedep_hashhead *inodedephd;
5696	struct inodedep *inodedep;
5697	static int next = 0;
5698	struct mount *mp;
5699	struct vnode *vp;
5700	struct fs *fs;
5701	int error, cnt;
5702	ino_t firstino, lastino, ino;
5703
5704	mtx_assert(&lk, MA_OWNED);
5705	/*
5706	 * Pick a random inode dependency to be cleared.
5707	 * We will then gather up all the inodes in its block
5708	 * that have dependencies and flush them out.
5709	 */
5710	for (cnt = 0; cnt < inodedep_hash; cnt++) {
5711		inodedephd = &inodedep_hashtbl[next++];
5712		if (next >= inodedep_hash)
5713			next = 0;
5714		if ((inodedep = LIST_FIRST(inodedephd)) != NULL)
5715			break;
5716	}
5717	if (inodedep == NULL)
5718		return;
5719	/*
5720	 * Ugly code to find mount point given pointer to superblock.
5721	 */
5722	fs = inodedep->id_fs;
5723	TAILQ_FOREACH(mp, &mountlist, mnt_list)
5724		if ((mp->mnt_flag & MNT_SOFTDEP) && fs == VFSTOUFS(mp)->um_fs)
5725			break;
5726	/*
5727	 * Find the last inode in the block with dependencies.
5728	 */
5729	firstino = inodedep->id_ino & ~(INOPB(fs) - 1);
5730	for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--)
5731		if (inodedep_lookup(fs, lastino, 0, &inodedep) != 0)
5732			break;
5733	/*
5734	 * Asynchronously push all but the last inode with dependencies.
5735	 * Synchronously push the last inode with dependencies to ensure
5736	 * that the inode block gets written to free up the inodedeps.
5737	 */
5738	for (ino = firstino; ino <= lastino; ino++) {
5739		if (inodedep_lookup(fs, ino, 0, &inodedep) == 0)
5740			continue;
5741		if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
5742			continue;
5743		FREE_LOCK(&lk);
5744		if ((error = ffs_vget(mp, ino, LK_EXCLUSIVE, &vp)) != 0) {
5745			softdep_error("clear_inodedeps: vget", error);
5746			vn_finished_write(mp);
5747			ACQUIRE_LOCK(&lk);
5748			return;
5749		}
5750		if (ino == lastino) {
5751			if ((error = ffs_syncvnode(vp, MNT_WAIT)))
5752				softdep_error("clear_inodedeps: fsync1", error);
5753		} else {
5754			if ((error = ffs_syncvnode(vp, MNT_NOWAIT)))
5755				softdep_error("clear_inodedeps: fsync2", error);
5756			VI_LOCK(vp);
5757			drain_output(vp);
5758			VI_UNLOCK(vp);
5759		}
5760		vput(vp);
5761		vn_finished_write(mp);
5762		ACQUIRE_LOCK(&lk);
5763	}
5764}
5765
5766/*
5767 * Function to determine if the buffer has outstanding dependencies
5768 * that will cause a roll-back if the buffer is written. If wantcount
5769 * is set, return number of dependencies, otherwise just yes or no.
5770 */
5771static int
5772softdep_count_dependencies(bp, wantcount)
5773	struct buf *bp;
5774	int wantcount;
5775{
5776	struct worklist *wk;
5777	struct inodedep *inodedep;
5778	struct indirdep *indirdep;
5779	struct allocindir *aip;
5780	struct pagedep *pagedep;
5781	struct diradd *dap;
5782	int i, retval;
5783
5784	retval = 0;
5785	ACQUIRE_LOCK(&lk);
5786	LIST_FOREACH(wk, &bp->b_dep, wk_list) {
5787		switch (wk->wk_type) {
5788
5789		case D_INODEDEP:
5790			inodedep = WK_INODEDEP(wk);
5791			if ((inodedep->id_state & DEPCOMPLETE) == 0) {
5792				/* bitmap allocation dependency */
5793				retval += 1;
5794				if (!wantcount)
5795					goto out;
5796			}
5797			if (TAILQ_FIRST(&inodedep->id_inoupdt)) {
5798				/* direct block pointer dependency */
5799				retval += 1;
5800				if (!wantcount)
5801					goto out;
5802			}
5803			if (TAILQ_FIRST(&inodedep->id_extupdt)) {
5804				/* direct block pointer dependency */
5805				retval += 1;
5806				if (!wantcount)
5807					goto out;
5808			}
5809			continue;
5810
5811		case D_INDIRDEP:
5812			indirdep = WK_INDIRDEP(wk);
5813
5814			LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) {
5815				/* indirect block pointer dependency */
5816				retval += 1;
5817				if (!wantcount)
5818					goto out;
5819			}
5820			continue;
5821
5822		case D_PAGEDEP:
5823			pagedep = WK_PAGEDEP(wk);
5824			for (i = 0; i < DAHASHSZ; i++) {
5825
5826				LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
5827					/* directory entry dependency */
5828					retval += 1;
5829					if (!wantcount)
5830						goto out;
5831				}
5832			}
5833			continue;
5834
5835		case D_BMSAFEMAP:
5836		case D_ALLOCDIRECT:
5837		case D_ALLOCINDIR:
5838		case D_MKDIR:
5839			/* never a dependency on these blocks */
5840			continue;
5841
5842		default:
5843			panic("softdep_check_for_rollback: Unexpected type %s",
5844			    TYPENAME(wk->wk_type));
5845			/* NOTREACHED */
5846		}
5847	}
5848out:
5849	FREE_LOCK(&lk);
5850	return retval;
5851}
5852
5853/*
5854 * Acquire exclusive access to a buffer.
5855 * Must be called with a locked mtx parameter.
5856 * Return acquired buffer or NULL on failure.
5857 */
5858static struct buf *
5859getdirtybuf(bp, mtx, waitfor)
5860	struct buf *bp;
5861	struct mtx *mtx;
5862	int waitfor;
5863{
5864	int error;
5865
5866	mtx_assert(mtx, MA_OWNED);
5867	if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) {
5868		if (waitfor != MNT_WAIT)
5869			return (NULL);
5870		error = BUF_LOCK(bp,
5871		    LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, mtx);
5872		/*
5873		 * Even if we sucessfully acquire bp here, we have dropped
5874		 * mtx, which may violates our guarantee.
5875		 */
5876		if (error == 0)
5877			BUF_UNLOCK(bp);
5878		else if (error != ENOLCK)
5879			panic("getdirtybuf: inconsistent lock: %d", error);
5880		mtx_lock(mtx);
5881		return (NULL);
5882	}
5883	if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
5884		BUF_UNLOCK(bp);
5885		if (waitfor != MNT_WAIT)
5886			return (NULL);
5887		/*
5888		 * The mtx argument must be bp->b_vp's mutex in
5889		 * this case.
5890		 */
5891#ifdef	DEBUG_VFS_LOCKS
5892		if (bp->b_vp->v_type != VCHR)
5893			ASSERT_VI_LOCKED(bp->b_vp, "getdirtybuf");
5894#endif
5895		bp->b_vflags |= BV_BKGRDWAIT;
5896		msleep(&bp->b_xflags, mtx, PRIBIO, "getbuf", 0);
5897		return (NULL);
5898	}
5899	if ((bp->b_flags & B_DELWRI) == 0) {
5900		BUF_UNLOCK(bp);
5901		return (NULL);
5902	}
5903	bremfree(bp);
5904	return (bp);
5905}
5906
5907/*
5908 * Wait for pending output on a vnode to complete.
5909 * Must be called with vnode lock and interlock locked.
5910 *
5911 * XXX: Should just be a call to bufobj_wwait().
5912 */
5913static void
5914drain_output(vp)
5915	struct vnode *vp;
5916{
5917	ASSERT_VOP_LOCKED(vp, "drain_output");
5918	ASSERT_VI_LOCKED(vp, "drain_output");
5919
5920	while (vp->v_bufobj.bo_numoutput) {
5921		vp->v_bufobj.bo_flag |= BO_WWAIT;
5922		msleep((caddr_t)&vp->v_bufobj.bo_numoutput,
5923		    VI_MTX(vp), PRIBIO + 1, "drainvp", 0);
5924	}
5925}
5926
5927/*
5928 * Called whenever a buffer that is being invalidated or reallocated
5929 * contains dependencies. This should only happen if an I/O error has
5930 * occurred. The routine is called with the buffer locked.
5931 */
5932static void
5933softdep_deallocate_dependencies(bp)
5934	struct buf *bp;
5935{
5936
5937	if ((bp->b_ioflags & BIO_ERROR) == 0)
5938		panic("softdep_deallocate_dependencies: dangling deps");
5939	softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error);
5940	panic("softdep_deallocate_dependencies: unrecovered I/O error");
5941}
5942
5943/*
5944 * Function to handle asynchronous write errors in the filesystem.
5945 */
5946static void
5947softdep_error(func, error)
5948	char *func;
5949	int error;
5950{
5951
5952	/* XXX should do something better! */
5953	printf("%s: got error %d while accessing filesystem\n", func, error);
5954}
5955
5956#endif /* SOFTUPDATES */
5957