Lines Matching defs:bp
109 static void vm_hold_free_pages(struct buf *bp, int newbsize);
110 static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
112 static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
113 static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
115 static void vfs_clean_pages_dirty_buf(struct buf *bp);
116 static void vfs_setdirty_locked_object(struct buf *bp);
117 static void vfs_vmio_invalidate(struct buf *bp);
118 static void vfs_vmio_truncate(struct buf *bp, int npages);
119 static void vfs_vmio_extend(struct buf *bp, int npages, int size);
127 static void bremfreel(struct buf *bp);
494 * bp's get placed back in the queues.
539 bufspace_adjust(struct buf *bp, int bufsize)
544 KASSERT((bp->b_flags & B_MALLOC) == 0,
545 ("bufspace_adjust: malloc buf %p", bp));
546 diff = bufsize - bp->b_bufsize;
556 bp->b_bufsize = bufsize;
744 bufmallocadjust(struct buf *bp, int bufsize)
748 KASSERT((bp->b_flags & B_MALLOC) != 0,
749 ("bufmallocadjust: non-malloc buf %p", bp));
750 diff = bufsize - bp->b_bufsize;
755 bp->b_bufsize = bufsize;
782 runningbufwakeup(struct buf *bp)
786 bspace = bp->b_runningbufspace;
792 bp->b_runningbufspace = 0;
837 vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
842 if (bp->b_flags & B_CACHE) {
845 bp->b_flags &= ~B_CACHE;
1040 struct buf *bp;
1063 bp = &buf[i];
1064 bzero(bp, sizeof *bp);
1065 bp->b_flags = B_INVAL;
1066 bp->b_rcred = NOCRED;
1067 bp->b_wcred = NOCRED;
1068 bp->b_qindex = QUEUE_EMPTY;
1069 bp->b_xflags = 0;
1070 bp->b_data = bp->b_kvabase = unmapped_buf;
1071 LIST_INIT(&bp->b_dep);
1072 BUF_LOCKINIT(bp);
1073 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
1167 vfs_buf_check_mapped(struct buf *bp)
1170 KASSERT(bp->b_kvabase != unmapped_buf,
1171 ("mapped buf: b_kvabase was not updated %p", bp));
1172 KASSERT(bp->b_data != unmapped_buf,
1173 ("mapped buf: b_data was not updated %p", bp));
1174 KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
1175 MAXPHYS, ("b_data + b_offset unmapped %p", bp));
1179 vfs_buf_check_unmapped(struct buf *bp)
1182 KASSERT(bp->b_data == unmapped_buf,
1183 ("unmapped buf: corrupted b_data %p", bp));
1186 #define BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
1187 #define BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
1189 #define BUF_CHECK_MAPPED(bp) do {} while (0)
1190 #define BUF_CHECK_UNMAPPED(bp) do {} while (0)
1194 isbufbusy(struct buf *bp)
1196 if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) ||
1197 ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
1209 struct buf *bp;
1228 for (bp = &buf[nbuf]; --bp >= buf; )
1229 if (isbufbusy(bp))
1277 for (bp = &buf[nbuf]; --bp >= buf; ) {
1278 if (isbufbusy(bp)) {
1281 if (bp->b_dev == NULL) {
1283 bp->b_vp->v_mount, mnt_list);
1291 nbusy, bp, bp->b_vp, bp->b_flags,
1292 (intmax_t)bp->b_blkno,
1293 (intmax_t)bp->b_lblkno);
1294 BUF_LOCKPRINTINFO(bp);
1296 vn_printf(bp->b_vp,
1322 bpmap_qenter(struct buf *bp)
1325 BUF_CHECK_MAPPED(bp);
1328 * bp->b_data is relative to bp->b_offset, but
1329 * bp->b_offset may be offset into the first page.
1331 bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
1332 pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
1333 bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
1334 (vm_offset_t)(bp->b_offset & PAGE_MASK));
1343 binsfree(struct buf *bp, int qindex)
1348 BUF_ASSERT_XLOCKED(bp);
1356 if (bqisclean(bp->b_qindex))
1357 qindex = bp->b_qindex;
1366 if (bp->b_flags & B_REMFREE) {
1367 olock = bqlock(bp->b_qindex);
1369 bremfreel(bp);
1377 if (bp->b_qindex != QUEUE_NONE)
1380 bp->b_qindex = qindex;
1381 if (bp->b_flags & B_AGE)
1382 TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1384 TAILQ_INSERT_TAIL(&bufqueues[bp->b_qindex], bp, b_freelist);
1386 bq_len[bp->b_qindex]++;
1397 buf_free(struct buf *bp)
1400 if (bp->b_flags & B_REMFREE)
1401 bremfreef(bp);
1402 if (bp->b_vflags & BV_BKGRDINPROG)
1404 if (bp->b_rcred != NOCRED) {
1405 crfree(bp->b_rcred);
1406 bp->b_rcred = NOCRED;
1408 if (bp->b_wcred != NOCRED) {
1409 crfree(bp->b_wcred);
1410 bp->b_wcred = NOCRED;
1412 if (!LIST_EMPTY(&bp->b_dep))
1413 buf_deallocate(bp);
1414 bufkva_free(bp);
1415 BUF_UNLOCK(bp);
1416 uma_zfree(buf_zone, bp);
1432 struct buf *bp;
1437 bp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY]);
1438 if (bp == NULL)
1440 bremfreel(bp);
1441 store[i] = bp;
1470 struct buf *bp;
1472 bp = uma_zalloc(buf_zone, M_NOWAIT);
1473 if (bp == NULL) {
1485 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
1486 panic("getnewbuf_empty: Locked buf %p on free queue.", bp);
1488 KASSERT(bp->b_vp == NULL,
1489 ("bp: %p still has vnode %p.", bp, bp->b_vp));
1490 KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
1491 ("invalid buffer %p flags %#x", bp, bp->b_flags));
1492 KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
1493 ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
1494 KASSERT(bp->b_npages == 0,
1495 ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
1496 KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
1497 KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
1499 bp->b_flags = 0;
1500 bp->b_ioflags = 0;
1501 bp->b_xflags = 0;
1502 bp->b_vflags = 0;
1503 bp->b_vp = NULL;
1504 bp->b_blkno = bp->b_lblkno = 0;
1505 bp->b_offset = NOOFFSET;
1506 bp->b_iodone = 0;
1507 bp->b_error = 0;
1508 bp->b_resid = 0;
1509 bp->b_bcount = 0;
1510 bp->b_npages = 0;
1511 bp->b_dirtyoff = bp->b_dirtyend = 0;
1512 bp->b_bufobj = NULL;
1513 bp->b_pin_count = 0;
1514 bp->b_data = bp->b_kvabase = unmapped_buf;
1515 bp->b_fsprivate1 = NULL;
1516 bp->b_fsprivate2 = NULL;
1517 bp->b_fsprivate3 = NULL;
1518 LIST_INIT(&bp->b_dep);
1520 return (bp);
1533 struct buf *bp, *nbp;
1545 while ((bp = nbp) != NULL) {
1547 * Calculate next bp (we can only use it if we do not
1550 nbp = TAILQ_NEXT(bp, b_freelist);
1556 if (kva && bp->b_kvasize == 0)
1559 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
1565 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
1566 BUF_UNLOCK(bp);
1570 KASSERT(bp->b_qindex == qindex,
1571 ("getnewbuf: inconsistent queue %d bp %p", qindex, bp));
1576 bremfreel(bp);
1583 if ((bp->b_vflags & BV_BKGRDERR) != 0) {
1584 bqrelse(bp);
1589 bp->b_flags |= B_INVAL;
1590 brelse(bp);
1655 bremfree(struct buf *bp)
1658 CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1659 KASSERT((bp->b_flags & B_REMFREE) == 0,
1660 ("bremfree: buffer %p already marked for delayed removal.", bp));
1661 KASSERT(bp->b_qindex != QUEUE_NONE,
1662 ("bremfree: buffer %p not on a queue.", bp));
1663 BUF_ASSERT_XLOCKED(bp);
1665 bp->b_flags |= B_REMFREE;
1675 bremfreef(struct buf *bp)
1679 qlock = bqlock(bp->b_qindex);
1681 bremfreel(bp);
1692 bremfreel(struct buf *bp)
1696 bp, bp->b_vp, bp->b_flags);
1697 KASSERT(bp->b_qindex != QUEUE_NONE,
1698 ("bremfreel: buffer %p not on a queue.", bp));
1699 if (bp->b_qindex != QUEUE_EMPTY) {
1700 BUF_ASSERT_XLOCKED(bp);
1702 mtx_assert(bqlock(bp->b_qindex), MA_OWNED);
1704 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
1706 KASSERT(bq_len[bp->b_qindex] >= 1, ("queue %d underflow",
1707 bp->b_qindex));
1708 bq_len[bp->b_qindex]--;
1710 bp->b_qindex = QUEUE_NONE;
1711 bp->b_flags &= ~B_REMFREE;
1721 bufkva_free(struct buf *bp)
1725 if (bp->b_kvasize == 0) {
1726 KASSERT(bp->b_kvabase == unmapped_buf &&
1727 bp->b_data == unmapped_buf,
1728 ("Leaked KVA space on %p", bp));
1729 } else if (buf_mapped(bp))
1730 BUF_CHECK_MAPPED(bp);
1732 BUF_CHECK_UNMAPPED(bp);
1734 if (bp->b_kvasize == 0)
1737 vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
1738 atomic_subtract_long(&bufkvaspace, bp->b_kvasize);
1740 bp->b_data = bp->b_kvabase = unmapped_buf;
1741 bp->b_kvasize = 0;
1750 bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
1758 bufkva_free(bp);
1769 bp->b_kvabase = (caddr_t)addr;
1770 bp->b_kvasize = maxsize;
1771 atomic_add_long(&bufkvaspace, bp->b_kvasize);
1773 bp->b_data = unmapped_buf;
1774 BUF_CHECK_UNMAPPED(bp);
1776 bp->b_data = bp->b_kvabase;
1777 BUF_CHECK_MAPPED(bp);
1858 struct buf *bp;
1865 *bpp = bp = getblk(vp, blkno, size, 0, 0, flags);
1866 if (bp == NULL)
1870 if ((bp->b_flags & B_CACHE) == 0) {
1875 racct_add_buf(curproc, bp, 0);
1881 bp->b_iocmd = BIO_READ;
1882 bp->b_flags &= ~B_INVAL;
1883 bp->b_ioflags &= ~BIO_ERROR;
1884 if (bp->b_rcred == NOCRED && cred != NOCRED)
1885 bp->b_rcred = crhold(cred);
1886 vfs_busy_pages(bp, 0);
1887 bp->b_iooffset = dbtob(bp->b_blkno);
1888 bstrategy(bp);
1895 rv = bufwait(bp);
1897 brelse(bp);
1916 bufwrite(struct buf *bp)
1923 CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1924 if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
1925 bp->b_flags |= B_INVAL | B_RELBUF;
1926 bp->b_flags &= ~B_CACHE;
1927 brelse(bp);
1930 if (bp->b_flags & B_INVAL) {
1931 brelse(bp);
1935 if (bp->b_flags & B_BARRIER)
1938 oldflags = bp->b_flags;
1940 BUF_ASSERT_HELD(bp);
1942 if (bp->b_pin_count > 0)
1943 bunpin_wait(bp);
1945 KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
1946 ("FFS background buffer should not get here %p", bp));
1948 vp = bp->b_vp;
1960 bufobj_wref(bp->b_bufobj);
1961 bundirty(bp);
1963 bp->b_flags &= ~B_DONE;
1964 bp->b_ioflags &= ~BIO_ERROR;
1965 bp->b_flags |= B_CACHE;
1966 bp->b_iocmd = BIO_WRITE;
1968 vfs_busy_pages(bp, 1);
1973 bp->b_runningbufspace = bp->b_bufsize;
1974 space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
1980 racct_add_buf(curproc, bp, 1);
1987 BUF_KERNPROC(bp);
1988 bp->b_iooffset = dbtob(bp->b_blkno);
1989 bstrategy(bp);
1992 int rtval = bufwait(bp);
1993 brelse(bp);
2012 bufbdflush(struct bufobj *bo, struct buf *bp)
2017 (void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
2029 if (bp == nbp)
2062 bdwrite(struct buf *bp)
2068 CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2069 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2070 KASSERT((bp->b_flags & B_BARRIER) == 0,
2071 ("Barrier request in delayed write %p", bp));
2072 BUF_ASSERT_HELD(bp);
2074 if (bp->b_flags & B_INVAL) {
2075 brelse(bp);
2086 vp = bp->b_vp;
2087 bo = bp->b_bufobj;
2090 BO_BDFLUSH(bo, bp);
2095 bdirty(bp);
2100 bp->b_flags |= B_CACHE;
2111 if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
2112 VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
2126 vfs_clean_pages_dirty_buf(bp);
2127 bqrelse(bp);
2154 bdirty(struct buf *bp)
2158 bp, bp->b_vp, bp->b_flags);
2159 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2160 KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2161 ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
2162 BUF_ASSERT_HELD(bp);
2163 bp->b_flags &= ~(B_RELBUF);
2164 bp->b_iocmd = BIO_WRITE;
2166 if ((bp->b_flags & B_DELWRI) == 0) {
2167 bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
2168 reassignbuf(bp);
2185 bundirty(struct buf *bp)
2188 CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2189 KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2190 KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2191 ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
2192 BUF_ASSERT_HELD(bp);
2194 if (bp->b_flags & B_DELWRI) {
2195 bp->b_flags &= ~B_DELWRI;
2196 reassignbuf(bp);
2202 bp->b_flags &= ~B_DEFERRED;
2215 bawrite(struct buf *bp)
2218 bp->b_flags |= B_ASYNC;
2219 (void) bwrite(bp);
2232 babarrierwrite(struct buf *bp)
2235 bp->b_flags |= B_ASYNC | B_BARRIER;
2236 (void) bwrite(bp);
2249 bbarrierwrite(struct buf *bp)
2252 bp->b_flags |= B_BARRIER;
2253 return (bwrite(bp));
2298 brelse(struct buf *bp)
2303 * Many functions erroneously call brelse with a NULL bp under rare
2304 * error conditions. Simply return when called with a NULL bp.
2306 if (bp == NULL)
2309 bp, bp->b_vp, bp->b_flags);
2310 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2311 ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2312 KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
2315 if (BUF_LOCKRECURSED(bp)) {
2320 BUF_UNLOCK(bp);
2324 if (bp->b_flags & B_MANAGED) {
2325 bqrelse(bp);
2329 if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
2330 BO_LOCK(bp->b_bufobj);
2331 bp->b_vflags &= ~BV_BKGRDERR;
2332 BO_UNLOCK(bp->b_bufobj);
2333 bdirty(bp);
2335 if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
2336 (bp->b_error != ENXIO || !LIST_EMPTY(&bp->b_dep)) &&
2337 !(bp->b_flags & B_INVAL)) {
2350 bp->b_ioflags &= ~BIO_ERROR;
2351 bdirty(bp);
2352 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
2353 (bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
2359 bp->b_flags |= B_INVAL;
2360 if (!LIST_EMPTY(&bp->b_dep))
2361 buf_deallocate(bp);
2362 if (bp->b_flags & B_DELWRI)
2364 bp->b_flags &= ~(B_DELWRI | B_CACHE);
2365 if ((bp->b_flags & B_VMIO) == 0) {
2366 allocbuf(bp, 0);
2367 if (bp->b_vp)
2368 brelvp(bp);
2376 * because pages associated with a B_DELWRI bp are marked clean.
2381 if (bp->b_flags & B_DELWRI)
2382 bp->b_flags &= ~B_RELBUF;
2401 if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
2402 (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
2403 !(bp->b_vp->v_mount != NULL &&
2404 (bp->b_vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2405 !vn_isdisk(bp->b_vp, NULL) && (bp->b_flags & B_DELWRI))) {
2406 vfs_vmio_invalidate(bp);
2407 allocbuf(bp, 0);
2410 if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
2411 (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
2412 allocbuf(bp, 0);
2413 bp->b_flags &= ~B_NOREUSE;
2414 if (bp->b_vp != NULL)
2415 brelvp(bp);
2423 if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
2424 (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
2425 bp->b_flags |= B_INVAL;
2426 if (bp->b_flags & B_INVAL) {
2427 if (bp->b_flags & B_DELWRI)
2428 bundirty(bp);
2429 if (bp->b_vp)
2430 brelvp(bp);
2434 if (bp->b_bufsize == 0) {
2435 buf_free(bp);
2439 if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
2440 (bp->b_ioflags & BIO_ERROR)) {
2441 bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
2442 if (bp->b_vflags & BV_BKGRDINPROG)
2445 bp->b_flags |= B_AGE;
2447 } else if (bp->b_flags & B_DELWRI)
2452 binsfree(bp, qindex);
2454 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF | B_DIRECT);
2455 if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
2458 BUF_UNLOCK(bp);
2475 bqrelse(struct buf *bp)
2479 CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2480 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2481 ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2484 if (BUF_LOCKRECURSED(bp)) {
2486 BUF_UNLOCK(bp);
2489 bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
2491 if (bp->b_flags & B_MANAGED) {
2492 if (bp->b_flags & B_REMFREE)
2493 bremfreef(bp);
2498 if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
2500 BO_LOCK(bp->b_bufobj);
2501 bp->b_vflags &= ~BV_BKGRDERR;
2502 BO_UNLOCK(bp->b_bufobj);
2505 if ((bp->b_flags & B_DELWRI) == 0 &&
2506 (bp->b_xflags & BX_VNDIRTY))
2508 if ((bp->b_flags & B_NOREUSE) != 0) {
2509 brelse(bp);
2514 binsfree(bp, qindex);
2518 BUF_UNLOCK(bp);
2528 vfs_vmio_iodone(struct buf *bp)
2537 obj = bp->b_bufobj->bo_object;
2538 KASSERT(obj->paging_in_progress >= bp->b_npages,
2540 obj->paging_in_progress, bp->b_npages));
2542 vp = bp->b_vp;
2548 foff = bp->b_offset;
2549 KASSERT(bp->b_offset != NOOFFSET,
2550 ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
2553 iosize = bp->b_bcount - bp->b_resid;
2555 for (i = 0; i < bp->b_npages; i++) {
2563 m = bp->b_pages[i];
2569 bp->b_pages[i] = m;
2570 } else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
2579 vfs_page_set_valid(bp, foff, m);
2589 vm_object_pip_wakeupn(obj, bp->b_npages);
2591 if (bogus && buf_mapped(bp)) {
2592 BUF_CHECK_MAPPED(bp);
2593 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
2594 bp->b_pages, bp->b_npages);
2602 vfs_vmio_unwire(struct buf *bp, vm_page_t m)
2616 } else if ((bp->b_flags & B_DIRECT) != 0)
2627 if ((bp->b_flags & B_NOREUSE) != 0)
2641 vfs_vmio_invalidate(struct buf *bp)
2647 if (buf_mapped(bp)) {
2648 BUF_CHECK_MAPPED(bp);
2649 pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
2651 BUF_CHECK_UNMAPPED(bp);
2664 obj = bp->b_bufobj->bo_object;
2665 resid = bp->b_bufsize;
2666 poffset = bp->b_offset & PAGE_MASK;
2668 for (i = 0; i < bp->b_npages; i++) {
2669 m = bp->b_pages[i];
2672 bp->b_pages[i] = NULL;
2685 vfs_vmio_unwire(bp, m);
2690 bp->b_npages = 0;
2697 vfs_vmio_truncate(struct buf *bp, int desiredpages)
2703 if (bp->b_npages == desiredpages)
2706 if (buf_mapped(bp)) {
2707 BUF_CHECK_MAPPED(bp);
2708 pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
2709 (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
2711 BUF_CHECK_UNMAPPED(bp);
2712 obj = bp->b_bufobj->bo_object;
2715 for (i = desiredpages; i < bp->b_npages; i++) {
2716 m = bp->b_pages[i];
2718 bp->b_pages[i] = NULL;
2719 vfs_vmio_unwire(bp, m);
2723 bp->b_npages = desiredpages;
2730 vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
2746 obj = bp->b_bufobj->bo_object;
2748 if (bp->b_npages < desiredpages) {
2760 OFF_TO_IDX(bp->b_offset) + bp->b_npages,
2763 &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
2764 bp->b_npages = desiredpages;
2781 toff = bp->b_bcount;
2782 tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2783 while ((bp->b_flags & B_CACHE) && toff < size) {
2788 pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
2789 m = bp->b_pages[pi];
2790 vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
2799 if (buf_mapped(bp))
2800 bpmap_qenter(bp);
2802 BUF_CHECK_UNMAPPED(bp);
2853 vfs_bio_awrite(struct buf *bp)
2858 daddr_t lblkno = bp->b_lblkno;
2859 struct vnode *vp = bp->b_vp;
2867 gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
2875 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
2883 bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
2888 bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
2897 BUF_UNLOCK(bp);
2903 bremfree(bp);
2904 bp->b_flags |= B_ASYNC;
2910 nwritten = bp->b_bufsize;
2911 (void) bwrite(bp);
2922 getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
2932 if (maxsize != bp->b_kvasize &&
2933 bufkva_alloc(bp, maxsize, gbflags))
2957 struct buf *bp;
2960 bp = NULL;
2978 if ((bp = buf_alloc()) == NULL)
2980 if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
2981 return (bp);
2987 if (bp != NULL) {
2988 bp->b_flags |= B_INVAL;
2989 brelse(bp);
3121 struct buf *bp;
3130 bp = NULL;
3139 bp = TAILQ_NEXT(sentinel, b_freelist);
3140 if (bp != NULL) {
3142 TAILQ_INSERT_AFTER(&bufqueues[queue], bp, sentinel,
3155 if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
3156 bp->b_vp != lvp)) {
3160 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
3164 if (bp->b_pin_count > 0) {
3165 BUF_UNLOCK(bp);
3172 if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
3173 (bp->b_flags & B_DELWRI) == 0) {
3174 BUF_UNLOCK(bp);
3177 if (bp->b_flags & B_INVAL) {
3178 bremfreef(bp);
3179 brelse(bp);
3184 if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
3186 BUF_UNLOCK(bp);
3202 vp = bp->b_vp;
3204 BUF_UNLOCK(bp);
3218 bp, bp->b_vp, bp->b_flags);
3220 vfs_bio_awrite(bp);
3222 bremfree(bp);
3223 bwrite(bp);
3242 BUF_UNLOCK(bp);
3257 struct buf *bp;
3260 bp = gbincore(bo, blkno);
3262 return (bp);
3327 vfs_clean_pages_dirty_buf(struct buf *bp)
3333 if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
3336 foff = bp->b_offset;
3337 KASSERT(bp->b_offset != NOOFFSET,
3340 VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
3341 vfs_drain_busy_pages(bp);
3342 vfs_setdirty_locked_object(bp);
3343 for (i = 0; i < bp->b_npages; i++) {
3346 if (eoff > bp->b_offset + bp->b_bufsize)
3347 eoff = bp->b_offset + bp->b_bufsize;
3348 m = bp->b_pages[i];
3349 vfs_page_set_validclean(bp, foff, m);
3353 VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
3357 vfs_setdirty_locked_object(struct buf *bp)
3362 object = bp->b_bufobj->bo_object;
3377 for (i = 0; i < bp->b_npages; i++)
3378 vm_page_test_dirty(bp->b_pages[i]);
3385 for (i = 0; i < bp->b_npages; i++) {
3386 if (bp->b_pages[i]->dirty)
3389 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3391 for (i = bp->b_npages - 1; i >= 0; --i) {
3392 if (bp->b_pages[i]->dirty) {
3396 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3402 if (eoffset > bp->b_bcount)
3403 eoffset = bp->b_bcount;
3411 if (bp->b_dirtyoff > boffset)
3412 bp->b_dirtyoff = boffset;
3413 if (bp->b_dirtyend < eoffset)
3414 bp->b_dirtyend = eoffset;
3425 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
3430 need_mapping = bp->b_data == unmapped_buf &&
3432 need_kva = bp->b_kvabase == unmapped_buf &&
3433 bp->b_data == unmapped_buf &&
3438 BUF_CHECK_UNMAPPED(bp);
3440 if (need_mapping && bp->b_kvabase != unmapped_buf) {
3453 bsize = vn_isdisk(bp->b_vp, NULL) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
3459 while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
3465 panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
3468 bufspace_wait(bp->b_vp, gbflags, 0, 0);
3473 bp->b_data = bp->b_kvabase;
3474 BUF_CHECK_MAPPED(bp);
3475 bpmap_qenter(bp);
3519 struct buf *bp;
3537 bp = gbincore(bo, blkno);
3538 if (bp != NULL) {
3549 error = BUF_TIMELOCK(bp, lockflags,
3562 else if (BUF_LOCKRECURSED(bp))
3571 if (bp->b_flags & B_INVAL)
3572 bp->b_flags &= ~B_CACHE;
3573 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
3574 bp->b_flags |= B_CACHE;
3575 if (bp->b_flags & B_MANAGED)
3576 MPASS(bp->b_qindex == QUEUE_NONE);
3578 bremfree(bp);
3583 if (bp->b_bcount != size) {
3584 if ((bp->b_flags & B_VMIO) == 0 ||
3585 (size > bp->b_kvasize)) {
3586 if (bp->b_flags & B_DELWRI) {
3592 if (bp->b_pin_count > 0) {
3594 bqrelse(bp);
3597 bunpin_wait(bp);
3600 bp->b_flags |= B_NOCACHE;
3601 bwrite(bp);
3603 if (LIST_EMPTY(&bp->b_dep)) {
3604 bp->b_flags |= B_RELBUF;
3605 brelse(bp);
3607 bp->b_flags |= B_NOCACHE;
3608 bwrite(bp);
3620 bp_unmapped_get_kva(bp, blkno, size, flags);
3628 allocbuf(bp, size);
3630 KASSERT(bp->b_offset != NOOFFSET,
3660 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
3661 bp->b_flags |= B_NOCACHE;
3662 bwrite(bp);
3665 bp->b_flags &= ~B_DONE;
3695 bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
3696 if (bp == NULL) {
3734 bp->b_flags |= B_INVAL;
3735 brelse(bp);
3744 bp->b_blkno = bp->b_lblkno = blkno;
3745 bp->b_offset = offset;
3746 bgetvp(vp, bp);
3757 bp->b_flags |= B_VMIO;
3758 KASSERT(vp->v_object == bp->b_bufobj->bo_object,
3760 bp, vp->v_object, bp->b_bufobj->bo_object));
3762 bp->b_flags &= ~B_VMIO;
3763 KASSERT(bp->b_bufobj->bo_object == NULL,
3765 bp, bp->b_bufobj->bo_object));
3766 BUF_CHECK_MAPPED(bp);
3769 allocbuf(bp, size);
3771 bp->b_flags &= ~B_DONE;
3773 CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
3774 BUF_ASSERT_HELD(bp);
3776 KASSERT(bp->b_bufobj == bo,
3777 ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
3778 return (bp);
3788 struct buf *bp;
3792 while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
3797 allocbuf(bp, size);
3799 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
3800 BUF_ASSERT_HELD(bp);
3801 return (bp);
3808 vfs_nonvmio_truncate(struct buf *bp, int newbsize)
3811 if (bp->b_flags & B_MALLOC) {
3816 bufmallocadjust(bp, 0);
3817 free(bp->b_data, M_BIOBUF);
3818 bp->b_data = bp->b_kvabase;
3819 bp->b_flags &= ~B_MALLOC;
3823 vm_hold_free_pages(bp, newbsize);
3824 bufspace_adjust(bp, newbsize);
3831 vfs_nonvmio_extend(struct buf *bp, int newbsize)
3846 if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
3848 bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
3849 bp->b_flags |= B_MALLOC;
3850 bufmallocadjust(bp, newbsize);
3861 if (bp->b_flags & B_MALLOC) {
3862 origbuf = bp->b_data;
3863 origbufsize = bp->b_bufsize;
3864 bp->b_data = bp->b_kvabase;
3865 bufmallocadjust(bp, 0);
3866 bp->b_flags &= ~B_MALLOC;
3869 vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
3870 (vm_offset_t) bp->b_data + newbsize);
3872 bcopy(origbuf, bp->b_data, origbufsize);
3875 bufspace_adjust(bp, newbsize);
3893 allocbuf(struct buf *bp, int size)
3897 BUF_ASSERT_HELD(bp);
3899 if (bp->b_bcount == size)
3902 if (bp->b_kvasize != 0 && bp->b_kvasize < size)
3906 if ((bp->b_flags & B_VMIO) == 0) {
3907 if ((bp->b_flags & B_MALLOC) == 0)
3913 if (newbsize < bp->b_bufsize)
3914 vfs_nonvmio_truncate(bp, newbsize);
3915 else if (newbsize > bp->b_bufsize)
3916 vfs_nonvmio_extend(bp, newbsize);
3921 num_pages((bp->b_offset & PAGE_MASK) + newbsize);
3923 if (bp->b_flags & B_MALLOC)
3929 if (size == 0 || bp->b_bufsize == 0)
3930 bp->b_flags |= B_CACHE;
3932 if (newbsize < bp->b_bufsize)
3933 vfs_vmio_truncate(bp, desiredpages);
3935 else if (size > bp->b_bcount)
3936 vfs_vmio_extend(bp, desiredpages, size);
3937 bufspace_adjust(bp, newbsize);
3939 bp->b_bcount = size; /* requested buffer size. */
3948 biodone(struct bio *bp)
3962 TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
3965 if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
3966 bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
3967 bp->bio_flags |= BIO_UNMAPPED;
3968 start = trunc_page((vm_offset_t)bp->bio_data);
3969 end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
3970 bp->bio_data = unmapped_buf;
3975 done = bp->bio_done;
3977 mtxp = mtx_pool_find(mtxpool_sleep, bp);
3979 bp->bio_flags |= BIO_DONE;
3980 wakeup(bp);
3983 done(bp);
3990 biowait(struct bio *bp, const char *wchan)
3994 mtxp = mtx_pool_find(mtxpool_sleep, bp);
3996 while ((bp->bio_flags & BIO_DONE) == 0)
3997 msleep(bp, mtxp, PRIBIO, wchan, 0);
3999 if (bp->bio_error != 0)
4000 return (bp->bio_error);
4001 if (!(bp->bio_flags & BIO_ERROR))
4007 biofinish(struct bio *bp, struct devstat *stat, int error)
4011 bp->bio_error = error;
4012 bp->bio_flags |= BIO_ERROR;
4015 devstat_end_transaction_bio(stat, bp);
4016 biodone(bp);
4027 bufwait(struct buf *bp)
4029 if (bp->b_iocmd == BIO_READ)
4030 bwait(bp, PRIBIO, "biord");
4032 bwait(bp, PRIBIO, "biowr");
4033 if (bp->b_flags & B_EINTR) {
4034 bp->b_flags &= ~B_EINTR;
4037 if (bp->b_ioflags & BIO_ERROR) {
4038 return (bp->b_error ? bp->b_error : EIO);
4051 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
4052 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
4064 bufdone(struct buf *bp)
4069 CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
4072 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
4073 BUF_ASSERT_HELD(bp);
4075 runningbufwakeup(bp);
4076 if (bp->b_iocmd == BIO_WRITE)
4077 dropobj = bp->b_bufobj;
4079 if (bp->b_iodone != NULL) {
4080 biodone = bp->b_iodone;
4081 bp->b_iodone = NULL;
4082 (*biodone) (bp);
4088 bufdone_finish(bp);
4095 bufdone_finish(struct buf *bp)
4097 BUF_ASSERT_HELD(bp);
4099 if (!LIST_EMPTY(&bp->b_dep))
4100 buf_complete(bp);
4102 if (bp->b_flags & B_VMIO) {
4108 if (bp->b_iocmd == BIO_READ &&
4109 !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
4110 !(bp->b_ioflags & BIO_ERROR))
4111 bp->b_flags |= B_CACHE;
4112 vfs_vmio_iodone(bp);
4120 if (bp->b_flags & B_ASYNC) {
4121 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
4122 (bp->b_ioflags & BIO_ERROR))
4123 brelse(bp);
4125 bqrelse(bp);
4127 bdone(bp);
4136 vfs_unbusy_pages(struct buf *bp)
4142 runningbufwakeup(bp);
4143 if (!(bp->b_flags & B_VMIO))
4146 obj = bp->b_bufobj->bo_object;
4148 for (i = 0; i < bp->b_npages; i++) {
4149 m = bp->b_pages[i];
4151 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
4154 bp->b_pages[i] = m;
4155 if (buf_mapped(bp)) {
4156 BUF_CHECK_MAPPED(bp);
4157 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4158 bp->b_pages, bp->b_npages);
4160 BUF_CHECK_UNMAPPED(bp);
4164 vm_object_pip_wakeupn(obj, bp->b_npages);
4177 vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4188 if (eoff > bp->b_offset + bp->b_bcount)
4189 eoff = bp->b_offset + bp->b_bcount;
4206 vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4218 if (eoff > bp->b_offset + bp->b_bcount)
4219 eoff = bp->b_offset + bp->b_bcount;
4239 vfs_drain_busy_pages(struct buf *bp)
4244 VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
4246 for (i = 0; i < bp->b_npages; i++) {
4247 m = bp->b_pages[i];
4250 vm_page_sbusy(bp->b_pages[last_busied]);
4253 VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4255 VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4260 vm_page_sunbusy(bp->b_pages[i]);
4276 vfs_busy_pages(struct buf *bp, int clear_modify)
4284 if (!(bp->b_flags & B_VMIO))
4287 obj = bp->b_bufobj->bo_object;
4288 foff = bp->b_offset;
4289 KASSERT(bp->b_offset != NOOFFSET,
4292 vfs_drain_busy_pages(bp);
4293 if (bp->b_bufsize != 0)
4294 vfs_setdirty_locked_object(bp);
4296 for (i = 0; i < bp->b_npages; i++) {
4297 m = bp->b_pages[i];
4299 if ((bp->b_flags & B_CLUSTER) == 0) {
4320 vfs_page_set_validclean(bp, foff, m);
4322 (bp->b_flags & B_CACHE) == 0) {
4323 bp->b_pages[i] = bogus_page;
4329 if (bogus && buf_mapped(bp)) {
4330 BUF_CHECK_MAPPED(bp);
4331 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4332 bp->b_pages, bp->b_npages);
4345 vfs_bio_set_valid(struct buf *bp, int base, int size)
4350 if (!(bp->b_flags & B_VMIO))
4358 base += (bp->b_offset & PAGE_MASK);
4361 VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4362 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4363 m = bp->b_pages[i];
4371 VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4387 vfs_bio_clrbuf(struct buf *bp)
4391 if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
4392 clrbuf(bp);
4395 bp->b_flags &= ~B_INVAL;
4396 bp->b_ioflags &= ~BIO_ERROR;
4397 VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4398 if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
4399 (bp->b_offset & PAGE_MASK) == 0) {
4400 if (bp->b_pages[0] == bogus_page)
4402 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
4403 VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object);
4404 if ((bp->b_pages[0]->valid & mask) == mask)
4406 if ((bp->b_pages[0]->valid & mask) == 0) {
4407 pmap_zero_page_area(bp->b_pages[0], 0, bp->b_bufsize);
4408 bp->b_pages[0]->valid |= mask;
4412 sa = bp->b_offset & PAGE_MASK;
4414 for (i = 0; i < bp->b_npages; i++, sa = 0) {
4415 slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
4419 if (bp->b_pages[i] == bogus_page)
4423 VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object);
4424 if ((bp->b_pages[i]->valid & mask) == mask)
4426 if ((bp->b_pages[i]->valid & mask) == 0)
4427 pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
4430 if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
4431 pmap_zero_page_area(bp->b_pages[i],
4436 bp->b_pages[i]->valid |= mask;
4439 VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4440 bp->b_resid = 0;
4444 vfs_bio_bzero_buf(struct buf *bp, int base, int size)
4449 if (buf_mapped(bp)) {
4450 BUF_CHECK_MAPPED(bp);
4451 bzero(bp->b_data + base, size);
4453 BUF_CHECK_UNMAPPED(bp);
4455 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4456 m = bp->b_pages[i];
4474 b_io_dismiss(struct buf *bp, int ioflag, bool release)
4478 ("buf %p non-VMIO noreuse", bp));
4481 bp->b_flags |= B_DIRECT;
4483 bp->b_xflags |= BX_ALTDATA;
4484 if ((ioflag & (IO_VMIO | IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
4485 bp->b_flags |= B_RELBUF;
4487 bp->b_flags |= B_NOREUSE;
4489 brelse(bp);
4491 bqrelse(bp);
4495 vfs_bio_brelse(struct buf *bp, int ioflag)
4498 b_io_dismiss(bp, ioflag, true);
4502 vfs_bio_set_flags(struct buf *bp, int ioflag)
4505 b_io_dismiss(bp, ioflag, false);
4514 vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
4520 BUF_CHECK_MAPPED(bp);
4524 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
4536 bp->b_pages[index] = p;
4538 bp->b_npages = index;
4543 vm_hold_free_pages(struct buf *bp, int newbsize)
4549 BUF_CHECK_MAPPED(bp);
4551 from = round_page((vm_offset_t)bp->b_data + newbsize);
4552 newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
4553 if (bp->b_npages > newnpages)
4554 pmap_qremove(from, bp->b_npages - newnpages);
4555 for (index = newnpages; index < bp->b_npages; index++) {
4556 p = bp->b_pages[index];
4557 bp->b_pages[index] = NULL;
4561 atomic_subtract_int(&vm_cnt.v_wire_count, bp->b_npages - newnpages);
4562 bp->b_npages = newnpages;
4580 vmapbuf(struct buf *bp, void *uaddr, size_t len, int mapbuf)
4586 if (bp->b_iocmd == BIO_READ)
4589 (vm_offset_t)uaddr, len, prot, bp->b_pages,
4592 bp->b_bufsize = len;
4593 bp->b_npages = pidx;
4594 bp->b_offset = ((vm_offset_t)uaddr) & PAGE_MASK;
4596 pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
4597 bp->b_data = bp->b_kvabase + bp->b_offset;
4599 bp->b_data = unmapped_buf;
4610 vunmapbuf(struct buf *bp)
4614 npages = bp->b_npages;
4615 if (buf_mapped(bp))
4616 pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
4617 vm_page_unhold_pages(bp->b_pages, npages);
4619 bp->b_data = unmapped_buf;
4623 bdone(struct buf *bp)
4627 mtxp = mtx_pool_find(mtxpool_sleep, bp);
4629 bp->b_flags |= B_DONE;
4630 wakeup(bp);
4635 bwait(struct buf *bp, u_char pri, const char *wchan)
4639 mtxp = mtx_pool_find(mtxpool_sleep, bp);
4641 while ((bp->b_flags & B_DONE) == 0)
4642 msleep(bp, mtxp, pri, wchan, 0);
4654 bufstrategy(struct bufobj *bo, struct buf *bp)
4659 vp = bp->b_vp;
4662 ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
4663 i = VOP_STRATEGY(vp, bp);
4664 KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
4719 bpin(struct buf *bp)
4723 mtxp = mtx_pool_find(mtxpool_sleep, bp);
4725 bp->b_pin_count++;
4730 bunpin(struct buf *bp)
4734 mtxp = mtx_pool_find(mtxpool_sleep, bp);
4736 if (--bp->b_pin_count == 0)
4737 wakeup(bp);
4742 bunpin_wait(struct buf *bp)
4746 mtxp = mtx_pool_find(mtxpool_sleep, bp);
4748 while (bp->b_pin_count > 0)
4749 msleep(bp, mtxp, PRIBIO, "bwunpin", 0);
4757 bdata2bio(struct buf *bp, struct bio *bip)
4760 if (!buf_mapped(bp)) {
4762 bip->bio_ma = bp->b_pages;
4763 bip->bio_ma_n = bp->b_npages;
4765 bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
4768 PAGE_SIZE == bp->b_npages,
4769 ("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
4772 bip->bio_data = bp->b_data;
4807 struct buf *bp;
4884 br_flags, &bp);
4887 if (LIST_EMPTY(&bp->b_dep)) {
4902 bp->b_flags |= B_RELBUF;
4904 bp->b_flags &= ~B_NOCACHE;
4905 brelse(bp);
4907 bqrelse(bp);
4960 struct buf *bp = (struct buf *)addr;
4967 db_printf("buf at %p\n", bp);
4969 (u_int)bp->b_flags, PRINT_BUF_FLAGS, (u_int)bp->b_xflags,
4970 PRINT_BUF_XFLAGS, (u_int)bp->b_vflags, PRINT_BUF_VFLAGS);
4975 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
4976 bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
4977 (intmax_t)bp->b_lblkno, bp->b_dep.lh_first);
4979 bp->b_kvabase, bp->b_kvasize);
4980 if (bp->b_npages) {
4982 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
4983 for (i = 0; i < bp->b_npages; i++) {
4985 m = bp->b_pages[i];
4988 if ((i + 1) < bp->b_npages)
4994 BUF_LOCKPRINTINFO(bp);
4999 struct buf *bp;
5003 bp = &buf[i];
5004 if (BUF_ISLOCKED(bp)) {
5005 db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5016 struct buf *bp;
5024 TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
5025 db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5029 TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
5030 db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5037 struct buf *bp;
5046 bp = &buf[i];
5047 if (bp->b_qindex == QUEUE_EMPTY)