Lines Matching refs:bp
115 int bcleanbuf(buf_t bp, boolean_t discard);
116 static int brecover_data(buf_t bp);
120 static void bremfree_locked(buf_t bp);
121 static void buf_reassign(buf_t bp, vnode_t newvp);
122 static errno_t buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo);
126 static buf_t buf_brelse_shadow(buf_t bp);
127 static void buf_free_meta_store(buf_t bp);
129 static buf_t buf_create_shadow_internal(buf_t bp, boolean_t force_copy,
188 #define binsheadfree(bp, dp, whichq) do { \
189 TAILQ_INSERT_HEAD(dp, bp, b_freelist); \
192 #define binstailfree(bp, dp, whichq) do { \
193 TAILQ_INSERT_TAIL(dp, bp, b_freelist); \
196 #define BHASHENTCHECK(bp) \
197 if ((bp)->b_hash.le_prev != (struct buf **)0xdeadbeef) \
198 panic("%p: b_hash.le_prev is not deadbeef", (bp));
200 #define BLISTNONE(bp) \
201 (bp)->b_hash.le_next = (struct buf *)0; \
202 (bp)->b_hash.le_prev = (struct buf **)0xdeadbeef;
207 #define bufinsvn(bp, dp) LIST_INSERT_HEAD(dp, bp, b_vnbufs)
208 #define bufremvn(bp) { \
209 LIST_REMOVE(bp, b_vnbufs); \
210 (bp)->b_vnbufs.le_next = NOLIST; \
229 blistenterhead(struct bufhashhdr * head, buf_t bp)
231 if ((bp->b_hash.le_next = (head)->lh_first) != NULL)
232 (head)->lh_first->b_hash.le_prev = &(bp)->b_hash.le_next;
233 (head)->lh_first = bp;
234 bp->b_hash.le_prev = &(head)->lh_first;
235 if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
240 binshash(buf_t bp, struct bufhashhdr *dp)
246 BHASHENTCHECK(bp);
251 if(nbp == bp)
256 blistenterhead(dp, bp);
260 bremhash(buf_t bp)
262 if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef)
264 if (bp->b_hash.le_next == bp)
267 if (bp->b_hash.le_next != NULL)
268 bp->b_hash.le_next->b_hash.le_prev = bp->b_hash.le_prev;
269 *bp->b_hash.le_prev = (bp)->b_hash.le_next;
276 bmovelaundry(buf_t bp)
278 bp->b_whichq = BQ_LAUNDRY;
279 bp->b_timestamp = buf_timestamp();
280 binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
285 buf_release_credentials(buf_t bp)
287 if (IS_VALID_CRED(bp->b_rcred)) {
288 kauth_cred_unref(&bp->b_rcred);
290 if (IS_VALID_CRED(bp->b_wcred)) {
291 kauth_cred_unref(&bp->b_wcred);
297 buf_valid(buf_t bp) {
299 if ( (bp->b_flags & (B_DONE | B_DELWRI)) )
305 buf_fromcache(buf_t bp) {
307 if ( (bp->b_flags & B_CACHE) )
313 buf_markinvalid(buf_t bp) {
315 SET(bp->b_flags, B_INVAL);
319 buf_markdelayed(buf_t bp) {
321 if (!ISSET(bp->b_flags, B_DELWRI)) {
322 SET(bp->b_flags, B_DELWRI);
325 buf_reassign(bp, bp->b_vp);
327 SET(bp->b_flags, B_DONE);
331 buf_markclean(buf_t bp) {
333 if (ISSET(bp->b_flags, B_DELWRI)) {
334 CLR(bp->b_flags, B_DELWRI);
337 buf_reassign(bp, bp->b_vp);
342 buf_markeintr(buf_t bp) {
344 SET(bp->b_flags, B_EINTR);
349 buf_markaged(buf_t bp) {
351 SET(bp->b_flags, B_AGE);
355 buf_fua(buf_t bp) {
357 if ((bp->b_flags & B_FUA) == B_FUA)
363 buf_markfua(buf_t bp) {
365 SET(bp->b_flags, B_FUA);
370 buf_setcpaddr(buf_t bp, struct cprotect *entry) {
371 bp->b_attr.ba_cpentry = entry;
375 buf_setcpoff (buf_t bp, uint64_t foffset) {
376 bp->b_attr.ba_cp_file_off = foffset;
496 buf_attr(buf_t bp) {
497 return &bp->b_attr;
501 buf_markstatic(buf_t bp __unused) {
502 SET(bp->b_flags, B_STATICCONTENT);
506 buf_static(buf_t bp) {
507 if ( (bp->b_flags & B_STATICCONTENT) )
549 buf_error(buf_t bp) {
551 return (bp->b_error);
555 buf_seterror(buf_t bp, errno_t error) {
557 if ((bp->b_error = error))
558 SET(bp->b_flags, B_ERROR);
560 CLR(bp->b_flags, B_ERROR);
564 buf_setflags(buf_t bp, int32_t flags) {
566 SET(bp->b_flags, (flags & BUF_X_WRFLAGS));
570 buf_clearflags(buf_t bp, int32_t flags) {
572 CLR(bp->b_flags, (flags & BUF_X_WRFLAGS));
576 buf_flags(buf_t bp) {
578 return ((bp->b_flags & BUF_X_RDFLAGS));
582 buf_reset(buf_t bp, int32_t io_flags) {
584 CLR(bp->b_flags, (B_READ | B_WRITE | B_ERROR | B_DONE | B_INVAL | B_ASYNC | B_NOCACHE | B_FUA));
585 SET(bp->b_flags, (io_flags & (B_ASYNC | B_READ | B_WRITE | B_NOCACHE)));
587 bp->b_error = 0;
591 buf_count(buf_t bp) {
593 return (bp->b_bcount);
597 buf_setcount(buf_t bp, uint32_t bcount) {
599 bp->b_bcount = bcount;
603 buf_size(buf_t bp) {
605 return (bp->b_bufsize);
609 buf_setsize(buf_t bp, uint32_t bufsize) {
611 bp->b_bufsize = bufsize;
615 buf_resid(buf_t bp) {
617 return (bp->b_resid);
621 buf_setresid(buf_t bp, uint32_t resid) {
623 bp->b_resid = resid;
627 buf_dirtyoff(buf_t bp) {
629 return (bp->b_dirtyoff);
633 buf_dirtyend(buf_t bp) {
635 return (bp->b_dirtyend);
639 buf_setdirtyoff(buf_t bp, uint32_t dirtyoff) {
641 bp->b_dirtyoff = dirtyoff;
645 buf_setdirtyend(buf_t bp, uint32_t dirtyend) {
647 bp->b_dirtyend = dirtyend;
651 buf_dataptr(buf_t bp) {
653 return (bp->b_datap);
657 buf_setdataptr(buf_t bp, uintptr_t data) {
659 bp->b_datap = data;
663 buf_vnode(buf_t bp) {
665 return (bp->b_vp);
669 buf_setvnode(buf_t bp, vnode_t vp) {
671 bp->b_vp = vp;
676 buf_callback(buf_t bp)
678 if ( !(bp->b_flags & B_CALL) )
681 return ((void *)bp->b_iodone);
686 buf_setcallback(buf_t bp, void (*callback)(buf_t, void *), void *transaction)
689 bp->b_flags |= (B_CALL | B_ASYNC);
691 bp->b_flags &= ~B_CALL;
692 bp->b_transaction = transaction;
693 bp->b_iodone = callback;
699 buf_setupl(buf_t bp, upl_t upl, uint32_t offset)
702 if ( !(bp->b_lflags & BL_IOBUF) )
706 bp->b_flags |= B_CLUSTER;
708 bp->b_flags &= ~B_CLUSTER;
709 bp->b_upl = upl;
710 bp->b_uploffset = offset;
716 buf_clone(buf_t bp, int io_offset, int io_size, void (*iodone)(buf_t, void *), void *arg)
723 if ((unsigned)(io_offset + io_size) > (unsigned)bp->b_bcount)
726 if (bp->b_flags & B_CLUSTER) {
727 if (io_offset && ((bp->b_uploffset + io_offset) & PAGE_MASK))
730 if (((bp->b_uploffset + io_offset + io_size) & PAGE_MASK) && ((io_offset + io_size) < bp->b_bcount))
733 io_bp = alloc_io_buf(bp->b_vp, 0);
735 io_bp->b_flags = bp->b_flags & (B_COMMIT_UPL | B_META | B_PAGEIO | B_CLUSTER | B_PHYS | B_RAW | B_ASYNC | B_READ | B_FUA);
742 if (bp->b_flags & B_CLUSTER) {
743 io_bp->b_upl = bp->b_upl;
744 io_bp->b_uploffset = bp->b_uploffset + io_offset;
746 io_bp->b_datap = (uintptr_t)(((char *)bp->b_datap) + io_offset);
755 buf_shadow(buf_t bp)
757 if (bp->b_lflags & BL_SHADOW)
764 buf_create_shadow_priv(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg)
766 return (buf_create_shadow_internal(bp, force_copy, external_storage, iodone, arg, 1));
770 buf_create_shadow(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg)
772 return (buf_create_shadow_internal(bp, force_copy, external_storage, iodone, arg, 0));
777 buf_create_shadow_internal(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg, int priv)
781 KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_START, bp, 0, 0, 0, 0);
783 if ( !(bp->b_flags & B_META) || (bp->b_lflags & BL_IOBUF)) {
785 KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_END, bp, 0, 0, 0, 0);
789 if (bp->b_shadow_ref && bp->b_data_ref == 0 && external_storage == 0)
790 panic("buf_create_shadow: %p is in the private state (%d, %d)", bp, bp->b_shadow_ref, bp->b_data_ref);
792 io_bp = alloc_io_buf(bp->b_vp, priv);
794 io_bp->b_flags = bp->b_flags & (B_META | B_ZALLOC | B_ASYNC | B_READ | B_FUA);
795 io_bp->b_blkno = bp->b_blkno;
796 io_bp->b_lblkno = bp->b_lblkno;
804 io_bp->b_bcount = bp->b_bcount;
805 io_bp->b_bufsize = bp->b_bufsize;
813 io_bp->b_datap = bp->b_datap;
815 io_bp->b_data_store = bp;
818 *(buf_t *)(&io_bp->b_orig) = bp;
823 io_bp->b_shadow = bp->b_shadow;
824 bp->b_shadow = io_bp;
825 bp->b_shadow_ref++;
831 bp->b_data_ref++;
839 io_bp->b_bcount = bp->b_bcount;
840 io_bp->b_bufsize = bp->b_bufsize;
843 allocbuf(io_bp, bp->b_bcount);
847 bcopy((caddr_t)bp->b_datap, (caddr_t)io_bp->b_datap, bp->b_bcount);
853 KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, io_bp, 0);
861 buf_make_private(buf_t bp)
867 KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_START, bp, bp->b_shadow_ref, 0, 0, 0);
869 if (bp->b_shadow_ref == 0 || bp->b_data_ref == 0 || ISSET(bp->b_lflags, BL_SHADOW)) {
871 KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, EINVAL, 0);
876 allocbuf(&my_buf, bp->b_bcount);
878 bcopy((caddr_t)bp->b_datap, (caddr_t)my_buf.b_datap, bp->b_bcount);
882 for (t_bp = bp->b_shadow; t_bp; t_bp = t_bp->b_shadow) {
883 if ( !ISSET(bp->b_lflags, BL_EXTERNAL))
888 if (ds_bp == NULL && bp->b_data_ref)
891 if (ds_bp && (bp->b_data_ref == 0 || bp->b_shadow_ref == 0))
899 KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, EINVAL, 0);
902 for (t_bp = bp->b_shadow; t_bp; t_bp = t_bp->b_shadow) {
906 ds_bp->b_data_ref = bp->b_data_ref;
908 bp->b_data_ref = 0;
909 bp->b_datap = my_buf.b_datap;
913 KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, 0, 0);
920 buf_setfilter(buf_t bp, void (*filter)(buf_t, void *), void *transaction,
924 *old_iodone = bp->b_iodone;
926 *old_transaction = bp->b_transaction;
928 bp->b_transaction = transaction;
929 bp->b_iodone = filter;
931 bp->b_flags |= B_FILTER;
933 bp->b_flags &= ~B_FILTER;
938 buf_blkno(buf_t bp) {
940 return (bp->b_blkno);
944 buf_lblkno(buf_t bp) {
946 return (bp->b_lblkno);
950 buf_setblkno(buf_t bp, daddr64_t blkno) {
952 bp->b_blkno = blkno;
956 buf_setlblkno(buf_t bp, daddr64_t lblkno) {
958 bp->b_lblkno = lblkno;
962 buf_device(buf_t bp) {
964 return (bp->b_dev);
968 buf_setdevice(buf_t bp, vnode_t vp) {
972 bp->b_dev = vp->v_rdev;
979 buf_drvdata(buf_t bp) {
981 return (bp->b_drvdata);
985 buf_setdrvdata(buf_t bp, void *drvdata) {
987 bp->b_drvdata = drvdata;
991 buf_fsprivate(buf_t bp) {
993 return (bp->b_fsprivate);
997 buf_setfsprivate(buf_t bp, void *fsprivate) {
999 bp->b_fsprivate = fsprivate;
1003 buf_rcred(buf_t bp) {
1005 return (bp->b_rcred);
1009 buf_wcred(buf_t bp) {
1011 return (bp->b_wcred);
1015 buf_upl(buf_t bp) {
1017 return (bp->b_upl);
1021 buf_uploffset(buf_t bp) {
1023 return ((uint32_t)(bp->b_uploffset));
1027 buf_proc(buf_t bp) {
1029 return (bp->b_proc);
1034 buf_map(buf_t bp, caddr_t *io_addr)
1040 if ( !(bp->b_flags & B_CLUSTER)) {
1041 *io_addr = (caddr_t)bp->b_datap;
1044 real_bp = (buf_t)(bp->b_real_bp);
1057 kret = ubc_upl_map(bp->b_upl, &vaddr); /* Map it in */
1064 vaddr += bp->b_uploffset;
1072 buf_unmap(buf_t bp)
1077 if ( !(bp->b_flags & B_CLUSTER))
1082 real_bp = (buf_t)(bp->b_real_bp);
1087 if ((bp->b_lflags & BL_IOBUF) &&
1088 ((bp->b_flags & (B_PAGEIO | B_READ)) != (B_PAGEIO | B_READ))) {
1099 bp->b_flags |= B_AGE;
1101 kret = ubc_upl_unmap(bp->b_upl);
1110 buf_clear(buf_t bp) {
1113 if (buf_map(bp, &baddr) == 0) {
1114 bzero(baddr, bp->b_bcount);
1115 buf_unmap(bp);
1117 bp->b_resid = 0;
1125 buf_strategy_fragmented(vnode_t devvp, buf_t bp, off_t f_offset, size_t contig_bytes)
1127 vnode_t vp = buf_vnode(bp);
1137 * save our starting point... the bp was already mapped
1141 io_blkno = bp->b_blkno;
1146 bp->b_blkno = bp->b_lblkno;
1153 io_bp->b_lblkno = bp->b_lblkno;
1154 io_bp->b_datap = bp->b_datap;
1155 io_resid = bp->b_bcount;
1156 io_direction = bp->b_flags & B_READ;
1159 if (bp->b_flags & B_READ)
1182 if (!ISSET(bp->b_flags, B_READ))
1208 buf_seterror(bp, error);
1209 bp->b_resid = io_resid;
1213 buf_biodone(bp);
1227 buf_t bp = ((struct vnop_strategy_args *)ap)->a_bp;
1228 vnode_t vp = bp->b_vp;
1246 bp->b_dev = devvp->v_rdev;
1248 if (bp->b_flags & B_READ)
1253 if ( !(bp->b_flags & B_CLUSTER)) {
1255 if ( (bp->b_upl) ) {
1257 * we have a UPL associated with this bp
1262 DTRACE_IO1(start, buf_t, bp);
1263 return (cluster_bp(bp));
1265 if (bp->b_blkno == bp->b_lblkno) {
1269 if ((error = VNOP_BLKTOOFF(vp, bp->b_lblkno, &f_offset))) {
1270 DTRACE_IO1(start, buf_t, bp);
1271 buf_seterror(bp, error);
1272 buf_biodone(bp);
1277 if ((error = VNOP_BLOCKMAP(vp, f_offset, bp->b_bcount, &bp->b_blkno, &contig_bytes, NULL, bmap_flags, NULL))) {
1278 DTRACE_IO1(start, buf_t, bp);
1279 buf_seterror(bp, error);
1280 buf_biodone(bp);
1285 DTRACE_IO1(start, buf_t, bp);
1290 if ((bp->b_blkno == -1) || (contig_bytes == 0)) {
1292 bp->b_blkno = -1;
1293 buf_clear(bp);
1295 else if ((long)contig_bytes < bp->b_bcount) {
1296 return (buf_strategy_fragmented(devvp, bp, f_offset, contig_bytes));
1302 DTRACE_IO1(start, buf_t, bp);
1307 if (bp->b_blkno == -1) {
1308 buf_biodone(bp);
1315 DTRACE_IO1(start, buf_t, bp);
1320 if (bp->b_attr.ba_cpentry != 0) {
1322 if(bp->b_attr.ba_cpentry->cp_flags & CP_OFF_IV_ENABLED) {
1324 if ((error = VNOP_BLKTOOFF(bp->b_vp, bp->b_lblkno, &f_offset)))
1340 bufattr_setcpoff (&(bp->b_attr), (u_int64_t)f_offset);
1341 CP_DEBUG((CPDBG_OFFSET_IO | DBG_FUNC_NONE), (uint32_t) f_offset, (uint32_t) bp->b_lblkno, (uint32_t) bp->b_blkno, (uint32_t) bp->b_bcount, 0);
1368 buf_free(buf_t bp) {
1370 free_io_buf(bp);
1379 * if BUF_NOTIFY_BUSY is set, call the callout function using a NULL bp for busy pages
1390 buf_t bp;
1427 bp = LIST_FIRST(&local_iterblkhd);
1428 LIST_REMOVE(bp, b_vnbufs);
1429 LIST_INSERT_HEAD(list[i].listhead, bp, b_vnbufs);
1431 if (buf_acquire_locked(bp, lock_flags, 0, 0)) {
1433 bp = NULL;
1441 retval = callout(bp, arg);
1445 if (bp)
1446 buf_brelse(bp);
1451 if (bp)
1452 buf_brelse(bp);
1474 buf_t bp;
1507 bp = LIST_FIRST(&local_iterblkhd);
1509 LIST_REMOVE(bp, b_vnbufs);
1510 LIST_INSERT_HEAD(&vp->v_cleanblkhd, bp, b_vnbufs);
1515 if ((flags & BUF_SKIP_META) && (bp->b_lblkno < 0 || ISSET(bp->b_flags, B_META)))
1523 if ( (error = (int)buf_acquire_locked(bp, aflags, slpflag, slptimeo)) ) {
1551 if (bp->b_flags & B_LOCKED)
1552 KERNEL_DEBUG(0xbbbbc038, bp, 0, 0, 0, 0);
1554 CLR(bp->b_flags, B_LOCKED);
1555 SET(bp->b_flags, B_INVAL);
1556 buf_brelse(bp);
1578 bp = LIST_FIRST(&local_iterblkhd);
1580 LIST_REMOVE(bp, b_vnbufs);
1581 LIST_INSERT_HEAD(&vp->v_dirtyblkhd, bp, b_vnbufs);
1586 if ((flags & BUF_SKIP_META) && (bp->b_lblkno < 0 || ISSET(bp->b_flags, B_META)))
1594 if ( (error = (int)buf_acquire_locked(bp, aflags, slpflag, slptimeo)) ) {
1622 if (bp->b_flags & B_LOCKED)
1623 KERNEL_DEBUG(0xbbbbc038, bp, 0, 0, 1, 0);
1625 CLR(bp->b_flags, B_LOCKED);
1626 SET(bp->b_flags, B_INVAL);
1628 if (ISSET(bp->b_flags, B_DELWRI) && (flags & BUF_WRITE_DATA))
1629 (void) VNOP_BWRITE(bp);
1631 buf_brelse(bp);
1658 buf_t bp;
1675 bp = LIST_FIRST(&local_iterblkhd);
1676 LIST_REMOVE(bp, b_vnbufs);
1677 LIST_INSERT_HEAD(&vp->v_dirtyblkhd, bp, b_vnbufs);
1679 if ((error = buf_acquire_locked(bp, lock_flags, 0, 0)) == EBUSY) {
1700 bp->b_flags &= ~B_LOCKED;
1706 if ((bp->b_vp == vp) || (wait == 0))
1707 (void) buf_bawrite(bp);
1709 (void) VNOP_BWRITE(bp);
1787 buf_t bp;
1795 bp = LIST_FIRST(iterheadp);
1796 LIST_REMOVE(bp, b_vnbufs);
1797 LIST_INSERT_HEAD(listheadp, bp, b_vnbufs);
1809 bremfree_locked(buf_t bp)
1814 whichq = bp->b_whichq;
1817 if (bp->b_shadow_ref == 0)
1818 panic("bremfree_locked: %p not on freelist", bp);
1820 * there are clones pointing to 'bp'...
1822 * when buf_brelse was last called on 'bp'
1833 if (bp->b_freelist.tqe_next == NULL) {
1836 if (dp->tqh_last != &bp->b_freelist.tqe_next)
1839 TAILQ_REMOVE(dp, bp, b_freelist);
1844 bp->b_whichq = -1;
1845 bp->b_timestamp = 0;
1846 bp->b_shadow = 0;
1854 bgetvp_locked(vnode_t vp, buf_t bp)
1857 if (bp->b_vp != vp)
1861 bp->b_dev = vp->v_rdev;
1863 bp->b_dev = NODEV;
1867 bufinsvn(bp, &vp->v_cleanblkhd);
1875 brelvp_locked(buf_t bp)
1880 if (bp->b_vnbufs.le_next != NOLIST)
1881 bufremvn(bp);
1883 bp->b_vp = (vnode_t)NULL;
1892 buf_reassign(buf_t bp, vnode_t newvp)
1905 if (bp->b_vnbufs.le_next != NOLIST)
1906 bufremvn(bp);
1911 if (ISSET(bp->b_flags, B_DELWRI))
1915 bufinsvn(bp, listheadp);
1921 bufhdrinit(buf_t bp)
1923 bzero((char *)bp, sizeof *bp);
1924 bp->b_dev = NODEV;
1925 bp->b_rcred = NOCRED;
1926 bp->b_wcred = NOCRED;
1927 bp->b_vnbufs.le_next = NOLIST;
1928 bp->b_flags = B_INVAL;
1939 buf_t bp;
1954 bp = &buf_headers[i];
1955 bufhdrinit(bp);
1957 BLISTNONE(bp);
1959 bp->b_whichq = BQ_EMPTY;
1960 bp->b_timestamp = buf_timestamp();
1961 binsheadfree(bp, dp, BQ_EMPTY);
1962 binshash(bp, &invalhash);
1970 bp = &buf_headers[i];
1971 bufhdrinit(bp);
1972 bp->b_whichq = -1;
1973 binsheadfree(bp, &iobufqueue, -1);
2084 buf_t bp;
2086 bp = buf_getblk(vp, blkno, size, 0, 0, queuetype);
2093 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) {
2099 SET(bp->b_flags, B_READ | async);
2100 if (IS_VALID_CRED(cred) && !IS_VALID_CRED(bp->b_rcred)) {
2102 bp->b_rcred = cred;
2105 VNOP_STRATEGY(bp);
2118 * we don't want to pass back a bp
2121 bp = NULL;
2124 buf_brelse(bp);
2125 bp = NULL;
2130 return (bp);
2141 buf_t bp;
2144 bp = *bpp = bio_doread(vp, blkno, size, cred, 0, queuetype);
2159 return (buf_biowait(bp));
2170 buf_t bp;
2173 bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_READ);
2176 return (buf_biowait(bp));
2186 buf_t bp;
2189 bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_META);
2192 return (buf_biowait(bp));
2218 buf_bwrite(buf_t bp)
2223 vnode_t vp = bp->b_vp;
2225 if (bp->b_datap == 0) {
2226 if (brecover_data(bp) == 0)
2230 sync = !ISSET(bp->b_flags, B_ASYNC);
2231 wasdelayed = ISSET(bp->b_flags, B_DELWRI);
2232 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI));
2245 buf_reassign(bp, vp);
2251 trace(TR_BUFWRITE, pack(vp, bp->b_bcount), bp->b_lblkno);
2257 VNOP_STRATEGY(bp);
2263 rv = buf_biowait(bp);
2271 buf_reassign(bp, vp);
2279 if (!ISSET(bp->b_flags, B_NORELSE)) {
2280 buf_brelse(bp);
2282 CLR(bp->b_flags, B_NORELSE);
2317 bdwrite_internal(buf_t bp, int return_error)
2320 vnode_t vp = bp->b_vp;
2328 if (!ISSET(bp->b_flags, B_DELWRI)) {
2329 SET(bp->b_flags, B_DELWRI);
2334 buf_reassign(bp, vp);
2351 if (!ISSET(bp->b_flags, B_LOCKED) && nbdwrite > ((nbuf_headers/4)*3)) {
2360 return (buf_bawrite(bp));
2364 SET(bp->b_flags, B_DONE);
2365 buf_brelse(bp);
2370 buf_bdwrite(buf_t bp)
2372 return (bdwrite_internal(bp, 0));
2387 bawrite_internal(buf_t bp, int throttle)
2389 vnode_t vp = bp->b_vp;
2405 SET(bp->b_flags, B_ASYNC);
2407 return (VNOP_BWRITE(bp));
2411 buf_bawrite(buf_t bp)
2413 return (bawrite_internal(bp, 1));
2419 buf_free_meta_store(buf_t bp)
2421 if (bp->b_bufsize) {
2422 if (ISSET(bp->b_flags, B_ZALLOC)) {
2425 z = getbufzone(bp->b_bufsize);
2426 zfree(z, (void *)bp->b_datap);
2428 kmem_free(kernel_map, bp->b_datap, bp->b_bufsize);
2430 bp->b_datap = (uintptr_t)NULL;
2431 bp->b_bufsize = 0;
2437 buf_brelse_shadow(buf_t bp)
2450 bp_head = (buf_t)bp->b_orig;
2456 if (bp_data = bp->b_data_store) {
2466 KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_START, bp, bp_head, bp_head->b_shadow_ref, 0, 0);
2470 for (bp_temp = bp_head; bp_temp && bp != bp_temp->b_shadow; bp_temp = bp_temp->b_shadow);
2473 panic("buf_brelse_shadow: bp not on list %p", bp_head);
2483 if (bp == bp_data && data_ref) {
2485 panic("buf_brelse_shadow: data_ref mismatch bp(%p)", bp);
2493 panic("buf_relse_shadow: b_shadow != NULL && b_shadow_ref == 0 bp(%p)", bp);
2495 panic("buf_relse_shadow: b_shadow == NULL && b_shadow_ref != 0 bp(%p)", bp);
2526 if (bp == bp_data && data_ref == 0)
2527 buf_free_meta_store(bp);
2529 bp->b_data_store = NULL;
2531 KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_END, bp, 0, 0, 0, 0);
2542 buf_brelse(buf_t bp)
2551 if (bp->b_whichq != -1 || !(bp->b_lflags & BL_BUSY))
2552 panic("buf_brelse: bad buffer = %p\n", bp);
2555 (void) OSBacktrace(&bp->b_stackbrelse[0], 6);
2557 bp->b_lastbrelse = current_thread();
2558 bp->b_tag = 0;
2560 if (bp->b_lflags & BL_IOBUF) {
2563 if (ISSET(bp->b_lflags, BL_SHADOW))
2564 shadow_master_bp = buf_brelse_shadow(bp);
2565 else if (ISSET(bp->b_lflags, BL_IOBUF_ALLOC))
2566 buf_free_meta_store(bp);
2567 free_io_buf(bp);
2570 bp = shadow_master_bp;
2577 bp->b_lblkno * PAGE_SIZE, bp, bp->b_datap,
2578 bp->b_flags, 0);
2580 trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
2589 if (ISSET(bp->b_flags, B_META) && ISSET(bp->b_flags, B_INVAL)) {
2590 if (ISSET(bp->b_flags, B_FILTER)) { /* if necessary, call out */
2591 void (*iodone_func)(struct buf *, void *) = bp->b_iodone;
2592 void *arg = bp->b_transaction;
2594 CLR(bp->b_flags, B_FILTER); /* but note callout done */
2595 bp->b_iodone = NULL;
2596 bp->b_transaction = NULL;
2599 panic("brelse: bp @ %p has NULL b_iodone!\n", bp);
2601 (*iodone_func)(bp, arg);
2607 upl = bp->b_upl;
2609 if ( !ISSET(bp->b_flags, B_META) && UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) {
2614 if ( !ISSET(bp->b_flags, B_INVAL)) {
2615 kret = ubc_create_upl(bp->b_vp,
2616 ubc_blktooff(bp->b_vp, bp->b_lblkno),
2617 bp->b_bufsize,
2625 upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 5);
2629 if (bp->b_datap) {
2634 bp->b_datap = (uintptr_t)NULL;
2638 if (bp->b_flags & (B_ERROR | B_INVAL)) {
2639 if (bp->b_flags & (B_READ | B_INVAL))
2646 if (ISSET(bp->b_flags, B_DELWRI | B_WASDIRTY))
2651 ubc_upl_commit_range(upl, 0, bp->b_bufsize, upl_flags |
2654 bp->b_upl = NULL;
2658 panic("brelse: UPL set for non VREG; vp=%p", bp->b_vp);
2664 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR))
2665 CLR(bp->b_flags, B_ERROR);
2669 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR)))
2670 SET(bp->b_flags, B_INVAL);
2672 if ((bp->b_bufsize <= 0) ||
2673 ISSET(bp->b_flags, B_INVAL) ||
2674 (ISSET(bp->b_lflags, BL_WANTDEALLOC) && !ISSET(bp->b_flags, B_DELWRI))) {
2683 if (ISSET(bp->b_flags, B_DELWRI))
2686 if (ISSET(bp->b_flags, B_META)) {
2687 if (bp->b_shadow_ref)
2690 buf_free_meta_store(bp);
2695 buf_release_credentials(bp);
2699 if (bp->b_shadow_ref) {
2700 SET(bp->b_lflags, BL_WAITSHADOW);
2710 buf_free_meta_store(bp);
2714 CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA));
2716 if (bp->b_vp)
2717 brelvp_locked(bp);
2719 bremhash(bp);
2720 BLISTNONE(bp);
2721 binshash(bp, &invalhash);
2723 bp->b_whichq = BQ_EMPTY;
2724 binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
2731 if (ISSET(bp->b_flags, B_LOCKED))
2733 else if (ISSET(bp->b_flags, B_META))
2735 else if (ISSET(bp->b_flags, B_AGE))
2741 bp->b_timestamp = buf_timestamp();
2754 if (bp->b_shadow_ref == 0) {
2755 CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE));
2756 bp->b_whichq = whichq;
2757 binstailfree(bp, bufq, whichq);
2765 CLR(bp->b_flags, (B_ASYNC | B_NOCACHE));
2778 if (ISSET(bp->b_lflags, BL_WANTED)) {
2788 CLR(bp->b_lflags, (BL_BUSY | BL_WANTED));
2803 wakeup(bp);
2806 bp, bp->b_datap, bp->b_flags, 0, 0);
2839 struct buf *bp;
2842 for (bp = dp->lh_first; bp != NULL; bp = bp->b_hash.le_next) {
2843 if (bp->b_lblkno == blkno && bp->b_vp == vp &&
2844 !ISSET(bp->b_flags, B_INVAL)) {
2845 return (bp);
2855 buf_t bp;
2863 if ((bp = incore_locked(vp, blkno, dp)) == NULL)
2866 if (bp->b_shadow_ref == 0)
2869 SET(bp->b_lflags, BL_WANTED_REF);
2871 (void) msleep(bp, buf_mtxp, PSPIN | (PRIBIO+1), "buf_wait_for_shadow", NULL);
2888 buf_t bp;
2907 if ((bp = incore_locked(vp, blkno, dp))) {
2911 if (ISSET(bp->b_lflags, BL_BUSY)) {
2919 SET(bp->b_lflags, BL_WANTED);
2932 err = msleep(bp, buf_mtxp, slpflag | PDROP | (PRIBIO + 1), "buf_getblk", &ts);
2956 SET(bp->b_lflags, BL_BUSY);
2957 SET(bp->b_flags, B_CACHE);
2960 bremfree_locked(bp);
2965 bp->b_owner = current_thread();
2966 bp->b_tag = 1;
2968 if ( (bp->b_upl) )
2969 panic("buffer has UPL, but not marked BUSY: %p", bp);
2971 if ( !ret_only_valid && bp->b_bufsize != size)
2972 allocbuf(bp, size);
2985 if (UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) {
2987 ubc_blktooff(vp, bp->b_lblkno),
2988 bp->b_bufsize,
2995 bp->b_upl = upl;
2999 SET(bp->b_flags, B_WASDIRTY);
3001 CLR(bp->b_flags, B_WASDIRTY);
3003 CLR(bp->b_flags, (B_DONE | B_CACHE | B_WASDIRTY | B_DELWRI));
3005 kret = ubc_upl_map(upl, (vm_offset_t*)&(bp->b_datap));
3035 if ((bp = getnewbuf(slpflag, slptimeo, &queue)) == NULL)
3046 SET(bp->b_flags, B_INVAL);
3047 binshash(bp, &invalhash);
3051 buf_brelse(bp);
3064 SET(bp->b_flags, B_META);
3066 bp->b_blkno = bp->b_lblkno = blkno;
3067 bp->b_vp = vp;
3072 binshash(bp, BUFHASH(vp, blkno));
3074 bgetvp_locked(vp, bp);
3078 allocbuf(bp, size);
3107 if ( (bp->b_upl) )
3108 panic("bp already has UPL: %p",bp);
3115 bp->b_bufsize,
3123 upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 4);
3125 bp->b_upl = upl;
3134 SET(bp->b_flags, B_CACHE | B_DONE);
3138 bp->b_validoff = 0;
3139 bp->b_dirtyoff = 0;
3143 SET(bp->b_flags, B_WASDIRTY);
3145 bp->b_validend = bp->b_bcount;
3146 bp->b_dirtyend = bp->b_bcount;
3149 bp->b_validend = bp->b_bcount;
3150 bp->b_dirtyend = 0;
3156 if (VNOP_BLOCKMAP(vp, f_offset, bp->b_bcount, &bp->b_blkno, &contig_bytes, NULL, bmap_flags, NULL))
3164 if ((long)contig_bytes < bp->b_bcount)
3165 bp->b_blkno = bp->b_lblkno;
3169 kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap));
3182 bp, bp->b_datap, bp->b_flags, 3, 0);
3185 (void) OSBacktrace(&bp->b_stackgetblk[0], 6);
3187 return (bp);
3196 buf_t bp = NULL;
3202 bp = getnewbuf(0, 0, &queue);
3203 } while (bp == NULL);
3205 SET(bp->b_flags, (B_META|B_INVAL));
3212 binshash(bp, &invalhash);
3217 allocbuf(bp, size);
3219 return (bp);
3223 buf_redundancy_flags(buf_t bp)
3225 return bp->b_redundancy_flags;
3229 buf_set_redundancy_flags(buf_t bp, uint32_t flags)
3231 SET(bp->b_redundancy_flags, flags);
3235 buf_clear_redundancy_flags(buf_t bp, uint32_t flags)
3237 CLR(bp->b_redundancy_flags, flags);
3245 buf_t bp;
3250 TAILQ_FOREACH(bp, &bufqueues[BQ_META], b_freelist) {
3251 if (ISSET(bp->b_flags, B_DELWRI) || bp->b_bufsize != nsize)
3253 ptr = (void *)bp->b_datap;
3254 bp->b_bufsize = 0;
3256 bcleanbuf(bp, TRUE);
3327 allocbuf(buf_t bp, int size)
3338 if (ISSET(bp->b_flags, B_META)) {
3341 if (bp->b_datap) {
3342 vm_offset_t elem = (vm_offset_t)bp->b_datap;
3344 if (ISSET(bp->b_flags, B_ZALLOC)) {
3345 if (bp->b_bufsize < nsize) {
3350 zprev = getbufzone(bp->b_bufsize);
3355 *(void **)(&bp->b_datap) = grab_memory_for_meta_buf(nsize);
3357 bp->b_datap = (uintptr_t)NULL;
3358 kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
3359 CLR(bp->b_flags, B_ZALLOC);
3361 bcopy((void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
3364 desired_size = bp->b_bufsize;
3368 if ((vm_size_t)bp->b_bufsize < desired_size) {
3370 bp->b_datap = (uintptr_t)NULL;
3371 kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
3372 bcopy((const void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
3373 kmem_free(kernel_map, elem, bp->b_bufsize);
3375 desired_size = bp->b_bufsize;
3384 *(void **)(&bp->b_datap) = grab_memory_for_meta_buf(nsize);
3385 SET(bp->b_flags, B_ZALLOC);
3387 kmem_alloc_kobject(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
3390 if (bp->b_datap == 0)
3393 bp->b_bufsize = desired_size;
3394 bp->b_bcount = size;
3424 buf_t bp;
3441 if (*queue == BQ_EMPTY && (bp = bufqueues[*queue].tqh_first))
3457 bp = bufqueues[*queue].tqh_first;
3458 if (bp)
3471 bp = bufqueues[BQ_EMPTY].tqh_first;
3472 if (bp) {
3486 bp = (struct buf *)zalloc(buf_hdr_zone);
3488 if (bp) {
3489 bufhdrinit(bp);
3490 bp->b_whichq = BQ_EMPTY;
3491 bp->b_timestamp = buf_timestamp();
3492 BLISTNONE(bp);
3493 SET(bp->b_flags, B_HDRALLOC);
3498 if (bp) {
3499 binshash(bp, &invalhash);
3500 binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
3521 bp = NULL;
3526 bp = lru_bp;
3529 bp = age_bp;
3537 bp = age_bp;
3545 bp = lru_bp;
3548 bp = age_bp;
3554 if (!bp) { /* Neither on AGE nor on LRU */
3555 bp = meta_bp;
3560 bp_time = t - bp->b_timestamp;
3571 bp = meta_bp;
3577 if (ISSET(bp->b_flags, B_LOCKED) || ISSET(bp->b_lflags, BL_BUSY))
3578 panic("getnewbuf: bp @ %p is LOCKED or BUSY! (flags 0x%x)\n", bp, bp->b_flags);
3581 if (bcleanbuf(bp, FALSE)) {
3588 return (bp);
3602 bcleanbuf(buf_t bp, boolean_t discard)
3605 bremfree_locked(bp);
3608 bp->b_owner = current_thread();
3609 bp->b_tag = 2;
3615 if (ISSET(bp->b_flags, B_DELWRI)) {
3617 SET(bp->b_lflags, BL_WANTDEALLOC);
3620 bmovelaundry(bp);
3635 bp->b_owner = current_thread();
3636 bp->b_tag = 8;
3641 SET(bp->b_lflags, BL_BUSY);
3644 bremhash(bp);
3649 if (bp->b_vp)
3650 brelvp_locked(bp);
3654 BLISTNONE(bp);
3656 if (ISSET(bp->b_flags, B_META))
3657 buf_free_meta_store(bp);
3659 trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
3661 buf_release_credentials(bp);
3663 bp->b_redundancy_flags = 0;
3668 CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA));
3669 bp->b_whichq = BQ_EMPTY;
3670 binshash(bp, &invalhash);
3671 binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
3672 CLR(bp->b_lflags, BL_BUSY);
3676 bp->b_bufsize = 0;
3677 bp->b_datap = (uintptr_t)NULL;
3678 bp->b_upl = (void *)NULL;
3686 bp->b_owner = current_thread();
3687 bp->b_tag = 3;
3689 bp->b_lflags = BL_BUSY;
3690 bp->b_flags = (bp->b_flags & B_HDRALLOC);
3691 bp->b_dev = NODEV;
3692 bp->b_blkno = bp->b_lblkno = 0;
3693 bp->b_iodone = NULL;
3694 bp->b_error = 0;
3695 bp->b_resid = 0;
3696 bp->b_bcount = 0;
3697 bp->b_dirtyoff = bp->b_dirtyend = 0;
3698 bp->b_validoff = bp->b_validend = 0;
3699 bzero(&bp->b_attr, sizeof(struct bufattr));
3711 buf_t bp;
3720 if ((bp = incore_locked(vp, lblkno, dp)) == (struct buf *)0) {
3724 if (ISSET(bp->b_lflags, BL_BUSY)) {
3729 SET(bp->b_lflags, BL_WANTED);
3731 error = msleep((caddr_t)bp, buf_mtxp, PDROP | (PRIBIO + 1), "buf_invalblkno", NULL);
3738 bremfree_locked(bp);
3739 SET(bp->b_lflags, BL_BUSY);
3740 SET(bp->b_flags, B_INVAL);
3743 bp->b_owner = current_thread();
3744 bp->b_tag = 4;
3747 buf_brelse(bp);
3754 buf_drop(buf_t bp)
3760 if (ISSET(bp->b_lflags, BL_WANTED)) {
3768 bp->b_owner = current_thread();
3769 bp->b_tag = 9;
3774 CLR(bp->b_lflags, (BL_BUSY | BL_WANTED));
3783 wakeup(bp);
3789 buf_acquire(buf_t bp, int flags, int slpflag, int slptimeo) {
3794 error = buf_acquire_locked(bp, flags, slpflag, slptimeo);
3803 buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo)
3808 if (ISSET(bp->b_flags, B_LOCKED)) {
3815 if (ISSET(bp->b_lflags, BL_BUSY)) {
3823 SET(bp->b_lflags, BL_WANTED);
3828 error = msleep((caddr_t)bp, buf_mtxp, slpflag | (PRIBIO + 1), "buf_acquire", &ts);
3835 bremfree_locked(bp);
3836 SET(bp->b_lflags, BL_BUSY);
3840 bp->b_owner = current_thread();
3841 bp->b_tag = 5;
3852 buf_biowait(buf_t bp)
3854 while (!ISSET(bp->b_flags, B_DONE)) {
3858 if (!ISSET(bp->b_flags, B_DONE)) {
3859 DTRACE_IO1(wait__start, buf_t, bp);
3860 (void) msleep(bp, buf_mtxp, PDROP | (PRIBIO+1), "buf_biowait", NULL);
3861 DTRACE_IO1(wait__done, buf_t, bp);
3866 if (ISSET(bp->b_flags, B_EINTR)) {
3867 CLR(bp->b_flags, B_EINTR);
3869 } else if (ISSET(bp->b_flags, B_ERROR))
3870 return (bp->b_error ? bp->b_error : EIO);
3894 buf_biodone(buf_t bp)
3900 bp, bp->b_datap, bp->b_flags, 0, 0);
3902 if (ISSET(bp->b_flags, B_DONE))
3905 bap = &bp->b_attr;
3907 if (bp->b_vp && bp->b_vp->v_mount) {
3908 mp = bp->b_vp->v_mount;
3913 if (mp && (bp->b_flags & B_READ) == 0) {
3915 INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_write_size);
3917 INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_read_size);
3924 if (bp->b_flags & B_READ)
3926 if (bp->b_flags & B_ASYNC)
3929 if (bp->b_flags & B_META)
3931 else if (bp->b_flags & B_PAGEIO)
3939 if (bp->b_flags & B_PASSIVE)
3946 buf_kernel_addrperm_addr(bp), (uintptr_t)VM_KERNEL_ADDRPERM(bp->b_vp), bp->b_resid, bp->b_error, 0);
3955 CLR(bp->b_flags, (B_WASDIRTY | B_PASSIVE));
3960 DTRACE_IO1(done, buf_t, bp);
3962 if (!ISSET(bp->b_flags, B_READ) && !ISSET(bp->b_flags, B_RAW))
3968 vnode_writedone(bp->b_vp);
3970 if (ISSET(bp->b_flags, (B_CALL | B_FILTER))) { /* if necessary, call out */
3971 void (*iodone_func)(struct buf *, void *) = bp->b_iodone;
3972 void *arg = bp->b_transaction;
3973 int callout = ISSET(bp->b_flags, B_CALL);
3976 panic("biodone: bp @ %p has NULL b_iodone!\n", bp);
3978 CLR(bp->b_flags, (B_CALL | B_FILTER)); /* filters and callouts are one-shot */
3979 bp->b_iodone = NULL;
3980 bp->b_transaction = NULL;
3983 SET(bp->b_flags, B_DONE); /* note that it's done */
3985 (*iodone_func)(bp, arg);
3990 * ownership of the bp and deals with releasing it if necessary
3997 * ownership of the bp and is expecting us
4002 if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release it */
4003 SET(bp->b_flags, B_DONE); /* note that it's done */
4005 buf_brelse(bp);
4023 CLR(bp->b_lflags, BL_WANTED);
4024 SET(bp->b_flags, B_DONE); /* note that it's done */
4028 wakeup(bp);
4032 (uintptr_t)bp, (uintptr_t)bp->b_datap, bp->b_flags, 0, 0);
4053 buf_t bp;
4058 for (bp = bufqueues[BQ_LOCKED].tqh_first; bp;
4059 bp = bp->b_freelist.tqe_next)
4086 struct buf *bp;
4099 for (bp = dp->tqh_first; bp; bp = bp->b_freelist.tqe_next) {
4100 counts[bp->b_bufsize/CLBYTES]++;
4120 buf_t bp;
4125 (bp = iobufqueue.tqh_first) == NULL) {
4131 TAILQ_REMOVE(&iobufqueue, bp, b_freelist);
4145 bp->b_timestamp = 0;
4146 bp->b_proc = NULL;
4148 bp->b_datap = 0;
4149 bp->b_flags = 0;
4150 bp->b_lflags = BL_BUSY | BL_IOBUF;
4151 bp->b_redundancy_flags = 0;
4152 bp->b_blkno = bp->b_lblkno = 0;
4154 bp->b_owner = current_thread();
4155 bp->b_tag = 6;
4157 bp->b_iodone = NULL;
4158 bp->b_error = 0;
4159 bp->b_resid = 0;
4160 bp->b_bcount = 0;
4161 bp->b_bufsize = 0;
4162 bp->b_upl = NULL;
4163 bp->b_vp = vp;
4164 bzero(&bp->b_attr, sizeof(struct bufattr));
4167 bp->b_dev = vp->v_rdev;
4169 bp->b_dev = NODEV;
4171 return (bp);
4176 free_io_buf(buf_t bp)
4183 bp->b_vp = NULL;
4184 bp->b_flags = B_INVAL;
4187 bzero (&bp->b_attr, sizeof(struct bufattr));
4191 binsheadfree(bp, &iobufqueue, -1);
4207 panic("free_io_buf: bp(%p) - bufstats.bufs_iobufinuse < 0", bp);
4252 struct buf *bp;
4259 while ( (bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY])) == NULL) {
4266 bremfree_locked(bp);
4271 SET(bp->b_lflags, BL_BUSY);
4275 bp->b_owner = current_thread();
4276 bp->b_tag = 10;
4283 error = bawrite_internal(bp, 0);
4286 bp->b_whichq = BQ_LAUNDRY;
4287 bp->b_timestamp = buf_timestamp();
4291 binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
4295 CLR(bp->b_lflags, BL_BUSY);
4298 bp->b_owner = current_thread();
4299 bp->b_tag = 11;
4323 brecover_data(buf_t bp)
4329 vnode_t vp = bp->b_vp;
4333 if ( !UBCINFOEXISTS(vp) || bp->b_bufsize == 0)
4337 if (! (buf_flags(bp) & B_READ)) {
4347 ubc_blktooff(vp, bp->b_lblkno),
4348 bp->b_bufsize,
4355 for (upl_offset = 0; upl_offset < bp->b_bufsize; upl_offset += PAGE_SIZE) {
4362 bp->b_upl = upl;
4364 kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap));
4371 bp->b_bufsize = 0;
4372 SET(bp->b_flags, B_INVAL);
4373 buf_brelse(bp);
4381 buf_t bp;
4407 while (((bp = TAILQ_FIRST(&bufqueues[BQ_META]))) &&
4408 (now > bp->b_timestamp) &&
4409 (now - bp->b_timestamp > thresh_hold) &&
4413 bremfree_locked(bp);
4417 bp->b_owner = current_thread();
4418 bp->b_tag = 12;
4422 if (ISSET(bp->b_flags, B_DELWRI)) {
4423 SET(bp->b_lflags, BL_WANTDEALLOC);
4425 bmovelaundry(bp);
4435 SET(bp->b_lflags, BL_BUSY);
4441 bremhash(bp);
4442 if (bp->b_vp) {
4443 brelvp_locked(bp);
4446 TAILQ_INSERT_TAIL(&privq, bp, b_freelist);
4463 TAILQ_FOREACH(bp, &privq, b_freelist) {
4465 if ((ISSET(bp->b_flags, B_ZALLOC)) && (buf_size(bp) >= PAGE_SIZE)) {
4469 trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno);
4472 buf_free_meta_store(bp);
4475 buf_release_credentials(bp);
4478 CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED
4480 bp->b_whichq = BQ_EMPTY;
4481 BLISTNONE(bp);
4486 TAILQ_FOREACH(bp, &privq, b_freelist) {
4487 binshash(bp, &invalhash);
4488 CLR(bp->b_lflags, BL_BUSY);
4492 if (bp->b_owner != current_thread()) {
4495 bp->b_owner = current_thread();
4496 bp->b_tag = 13;
4538 buf_t bp, next;
4550 bp = TAILQ_FIRST(&bufqueues[whichq]);
4552 for (buf_count = 0; bp; bp = next) {
4553 next = bp->b_freelist.tqe_next;
4555 if (bp->b_vp == NULL || bp->b_vp->v_mount != mp) {
4559 if (ISSET(bp->b_flags, B_DELWRI) && !ISSET(bp->b_lflags, BL_BUSY)) {
4561 bremfree_locked(bp);
4563 bp->b_owner = current_thread();
4564 bp->b_tag = 7;
4566 SET(bp->b_lflags, BL_BUSY);
4569 flush_table[buf_count] = bp;