Lines Matching refs:hdr
601 #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */
606 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE)
607 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS)
608 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR)
609 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH)
610 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ)
611 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE)
612 #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
613 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE)
614 #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \
615 (hdr)->b_l2hdr != NULL)
616 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING)
617 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED)
618 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD)
653 #define HDR_LOCK(hdr) \
654 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
815 static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr,
847 buf_discard_identity(arc_buf_hdr_t *hdr)
849 hdr->b_dva.dva_word[0] = 0;
850 hdr->b_dva.dva_word[1] = 0;
851 hdr->b_birth = 0;
852 hdr->b_cksum0 = 0;
1391 /* adjust l2arc hdr stats */
1470 arc_buf_hdr_t *hdr;
1474 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1475 ASSERT(BUF_EMPTY(hdr));
1476 hdr->b_size = size;
1477 hdr->b_type = type;
1478 hdr->b_spa = spa_load_guid(spa);
1479 hdr->b_state = arc_anon;
1480 hdr->b_arc_access = 0;
1482 buf->b_hdr = hdr;
1487 hdr->b_buf = buf;
1489 hdr->b_datacnt = 1;
1490 hdr->b_flags = 0;
1491 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1492 (void) refcount_add(&hdr->b_refcnt, tag);
1522 arc_buf_hdr_t *hdr = buf->b_hdr;
1525 (void) refcount_add(&hdr->b_refcnt, tag);
1526 (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1528 atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1535 arc_buf_hdr_t *hdr;
1538 hdr = buf->b_hdr;
1539 (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1540 (void) refcount_remove(&hdr->b_refcnt, tag);
1544 atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1551 arc_buf_hdr_t *hdr = from->b_hdr;
1552 uint64_t size = hdr->b_size;
1554 ASSERT(hdr->b_state != arc_anon);
1557 buf->b_hdr = hdr;
1561 buf->b_next = hdr->b_buf;
1562 hdr->b_buf = buf;
1572 if (hdr->b_type == ARC_BUFC_DATA) {
1576 hdr->b_datacnt += 1;
1583 arc_buf_hdr_t *hdr;
1598 hdr = buf->b_hdr;
1599 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1602 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1603 add_reference(hdr, hash_lock, tag);
1604 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1605 arc_access(hdr, hash_lock);
1608 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1609 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1620 arc_buf_hdr_t *hdr = buf->b_hdr;
1622 if (HDR_L2_WRITING(hdr)) {
1626 df->l2df_size = hdr->b_size;
1633 free_func(buf->b_data, hdr->b_size);
1694 /* remove the buf from the hdr list */
1708 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1710 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1711 ASSERT3P(hdr->b_state, ==, arc_anon);
1712 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1713 l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1724 * The hdr may be removed from l2ad_buflist before we
1729 l2hdr = hdr->b_l2hdr;
1734 hdr->b_size, 0);
1735 list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1736 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1739 if (hdr->b_state == arc_l2c_only)
1741 hdr->b_l2hdr = NULL;
1748 if (!BUF_EMPTY(hdr)) {
1749 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1750 buf_discard_identity(hdr);
1752 while (hdr->b_buf) {
1753 arc_buf_t *buf = hdr->b_buf;
1759 arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1760 hdr->b_buf = buf->b_next;
1767 arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1770 if (hdr->b_freeze_cksum != NULL) {
1771 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1772 hdr->b_freeze_cksum = NULL;
1774 if (hdr->b_thawed) {
1775 kmem_free(hdr->b_thawed, 1);
1776 hdr->b_thawed = NULL;
1779 ASSERT(!list_link_active(&hdr->b_arc_node));
1780 ASSERT3P(hdr->b_hash_next, ==, NULL);
1781 ASSERT3P(hdr->b_acb, ==, NULL);
1782 kmem_cache_free(hdr_cache, hdr);
1788 arc_buf_hdr_t *hdr = buf->b_hdr;
1789 int hashed = hdr->b_state != arc_anon;
1795 kmutex_t *hash_lock = HDR_LOCK(hdr);
1798 hdr = buf->b_hdr;
1799 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1801 (void) remove_reference(hdr, hash_lock, tag);
1802 if (hdr->b_datacnt > 1) {
1805 ASSERT(buf == hdr->b_buf);
1807 hdr->b_flags |= ARC_BUF_AVAILABLE;
1810 } else if (HDR_IO_IN_PROGRESS(hdr)) {
1818 (void) remove_reference(hdr, NULL, tag);
1819 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1820 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1823 arc_hdr_destroy(hdr);
1825 if (remove_reference(hdr, NULL, tag) > 0)
1828 arc_hdr_destroy(hdr);
1835 arc_buf_hdr_t *hdr = buf->b_hdr;
1836 kmutex_t *hash_lock = HDR_LOCK(hdr);
1839 if (hdr->b_state == arc_anon) {
1840 ASSERT(hdr->b_datacnt == 1);
1846 hdr = buf->b_hdr;
1847 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1848 ASSERT(hdr->b_state != arc_anon);
1851 (void) remove_reference(hdr, hash_lock, tag);
1852 if (hdr->b_datacnt > 1) {
1856 ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1858 hdr->b_flags |= ARC_BUF_AVAILABLE;
1860 ASSERT(no_callback || hdr->b_datacnt > 1 ||
1861 refcount_is_zero(&hdr->b_refcnt));
1881 arc_buf_hdr_t *hdr;
1888 hdr = buf->b_hdr;
1889 if (hdr == NULL) {
1902 ASSERT3P(hdr, ==, &arc_eviction_hdr);
1907 if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
2761 arc_buf_hdr_t *hdr = buf->b_hdr;
2763 atomic_add_64(&hdr->b_state->arcs_size, size);
2764 if (list_link_active(&hdr->b_arc_node)) {
2765 ASSERT(refcount_is_zero(&hdr->b_refcnt));
2766 atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2772 if (arc_size < arc_c && hdr->b_state == arc_anon &&
2940 arc_buf_hdr_t *hdr, *found;
2948 hdr = buf->b_hdr;
2951 * The hdr was inserted into hash-table and removed from lists
2958 found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2961 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2962 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2963 (found == hdr && HDR_L2_READING(hdr)));
2965 hdr->b_flags &= ~ARC_L2_EVICTED;
2966 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2967 hdr->b_flags &= ~ARC_L2CACHE;
2970 callback_list = hdr->b_acb;
2978 func(buf->b_data, hdr->b_size);
2986 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2993 arc_access(hdr, hash_lock);
3008 hdr->b_acb = NULL;
3009 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3010 ASSERT(!HDR_BUF_AVAILABLE(hdr));
3013 ASSERT(hdr->b_datacnt == 1);
3014 hdr->b_flags |= ARC_BUF_AVAILABLE;
3017 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
3020 hdr->b_flags |= ARC_IO_ERROR;
3021 if (hdr->b_state != arc_anon)
3022 arc_change_state(arc_anon, hdr, hash_lock);
3023 if (HDR_IN_HASH_TABLE(hdr))
3024 buf_hash_remove(hdr);
3025 freeable = refcount_is_zero(&hdr->b_refcnt);
3030 * that the hdr (and hence the cv) might be freed before we get to
3033 cv_broadcast(&hdr->b_cv);
3044 ASSERT3P(hdr->b_state, ==, arc_anon);
3045 freeable = refcount_is_zero(&hdr->b_refcnt);
3063 arc_hdr_destroy(hdr);
3089 arc_buf_hdr_t *hdr;
3096 hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
3098 if (hdr && hdr->b_datacnt > 0) {
3102 if (HDR_IO_IN_PROGRESS(hdr)) {
3105 cv_wait(&hdr->b_cv, hash_lock);
3123 acb->acb_next = hdr->b_acb;
3124 hdr->b_acb = acb;
3125 add_reference(hdr, hash_lock, private);
3133 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3136 add_reference(hdr, hash_lock, private);
3142 buf = hdr->b_buf;
3145 if (HDR_BUF_AVAILABLE(hdr)) {
3147 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3153 refcount_count(&hdr->b_refcnt) == 0) {
3154 hdr->b_flags |= ARC_PREFETCH;
3156 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
3157 arc_access(hdr, hash_lock);
3159 hdr->b_flags |= ARC_L2CACHE;
3161 hdr->b_flags |= ARC_L2COMPRESS;
3164 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
3165 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
3179 if (hdr == NULL) {
3184 hdr = buf->b_hdr;
3185 hdr->b_dva = *BP_IDENTITY(bp);
3186 hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
3187 hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
3188 exists = buf_hash_insert(hdr, &hash_lock);
3192 buf_discard_identity(hdr);
3198 (void) remove_reference(hdr, hash_lock,
3200 hdr->b_flags |= ARC_PREFETCH;
3203 hdr->b_flags |= ARC_L2CACHE;
3205 hdr->b_flags |= ARC_L2COMPRESS;
3207 hdr->b_flags |= ARC_INDIRECT;
3210 ASSERT(GHOST_STATE(hdr->b_state));
3211 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3212 ASSERT0(refcount_count(&hdr->b_refcnt));
3213 ASSERT(hdr->b_buf == NULL);
3217 hdr->b_flags |= ARC_PREFETCH;
3219 add_reference(hdr, hash_lock, private);
3221 hdr->b_flags |= ARC_L2CACHE;
3223 hdr->b_flags |= ARC_L2COMPRESS;
3225 buf->b_hdr = hdr;
3230 hdr->b_buf = buf;
3231 ASSERT(hdr->b_datacnt == 0);
3232 hdr->b_datacnt = 1;
3234 arc_access(hdr, hash_lock);
3237 ASSERT(!GHOST_STATE(hdr->b_state));
3243 ASSERT(hdr->b_acb == NULL);
3244 hdr->b_acb = acb;
3245 hdr->b_flags |= ARC_IO_IN_PROGRESS;
3247 if (hdr->b_l2hdr != NULL &&
3248 (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
3249 devw = hdr->b_l2hdr->b_dev->l2ad_writing;
3250 addr = hdr->b_l2hdr->b_daddr;
3251 b_compress = hdr->b_l2hdr->b_compress;
3252 b_asize = hdr->b_l2hdr->b_asize;
3267 ASSERT3U(hdr->b_size, ==, size);
3268 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
3271 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
3272 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
3288 if (hdr->b_l2hdr != NULL &&
3289 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
3290 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
3293 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
3348 arc_buf_hdr_t *, hdr);
3350 if (HDR_L2_WRITING(hdr))
3359 arc_buf_hdr_t *, hdr);
3395 arc_buf_hdr_t *hdr;
3399 hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
3401 if (hdr == NULL)
3403 if (HDR_BUF_AVAILABLE(hdr)) {
3404 arc_buf_t *buf = hdr->b_buf;
3405 add_reference(hdr, hash_lock, FTAG);
3406 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3425 arc_buf_hdr_t *hdr;
3432 hdr = buf->b_hdr;
3433 if (hdr == NULL) {
3451 hash_lock = HDR_LOCK(hdr);
3453 hdr = buf->b_hdr;
3454 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3456 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
3457 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3460 * Pull this buffer off of the hdr
3462 bufp = &hdr->b_buf;
3470 if (hdr->b_datacnt == 0) {
3471 arc_state_t *old_state = hdr->b_state;
3474 ASSERT(hdr->b_buf == NULL);
3475 ASSERT(refcount_is_zero(&hdr->b_refcnt));
3480 get_buf_info(hdr, old_state, &list, &lock);
3481 get_buf_info(hdr, evicted_state, &evicted_list, &evicted_lock);
3485 arc_change_state(evicted_state, hdr, hash_lock);
3486 ASSERT(HDR_IN_HASH_TABLE(hdr));
3487 hdr->b_flags |= ARC_IN_HASH_TABLE;
3488 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3509 * a new hdr for the buffer.
3514 arc_buf_hdr_t *hdr;
3526 hdr = buf->b_hdr;
3529 ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3531 if (hdr->b_state == arc_anon) {
3535 hash_lock = HDR_LOCK(hdr);
3537 hdr = buf->b_hdr;
3538 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3541 l2hdr = hdr->b_l2hdr;
3544 hdr->b_l2hdr = NULL;
3545 list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3547 buf_size = hdr->b_size;
3552 if (hdr->b_datacnt > 1) {
3555 uint64_t blksz = hdr->b_size;
3556 uint64_t spa = hdr->b_spa;
3557 arc_buf_contents_t type = hdr->b_type;
3558 uint32_t flags = hdr->b_flags;
3560 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3562 * Pull the data off of this hdr and attach it to
3563 * a new anonymous hdr.
3565 (void) remove_reference(hdr, hash_lock, tag);
3566 bufp = &hdr->b_buf;
3572 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3573 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3574 if (refcount_is_zero(&hdr->b_refcnt)) {
3575 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3576 ASSERT3U(*size, >=, hdr->b_size);
3577 atomic_add_64(size, -hdr->b_size);
3584 if (hdr->b_type == ARC_BUFC_DATA) {
3587 -hdr->b_size);
3589 hdr->b_datacnt -= 1;
3614 ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3615 ASSERT(!list_link_active(&hdr->b_arc_node));
3616 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3617 if (hdr->b_state != arc_anon)
3618 arc_change_state(arc_anon, hdr, hash_lock);
3619 hdr->b_arc_access = 0;
3623 buf_discard_identity(hdr);
3632 hdr->b_size, 0);
3679 arc_buf_hdr_t *hdr = buf->b_hdr;
3690 if (HDR_IO_IN_PROGRESS(hdr)) {
3691 mutex_enter(&hdr->b_freeze_lock);
3692 if (hdr->b_freeze_cksum != NULL) {
3693 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3694 hdr->b_freeze_cksum = NULL;
3696 mutex_exit(&hdr->b_freeze_lock);
3699 hdr->b_flags |= ARC_IO_IN_PROGRESS;
3707 arc_buf_hdr_t *hdr = buf->b_hdr;
3709 ASSERT(hdr->b_acb == NULL);
3712 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3713 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3714 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3716 ASSERT(BUF_EMPTY(hdr));
3725 if (!BUF_EMPTY(hdr)) {
3733 exists = buf_hash_insert(hdr, &hash_lock);
3742 panic("bad overwrite, hdr=%p exists=%p",
3743 (void *)hdr, (void *)exists);
3748 exists = buf_hash_insert(hdr, &hash_lock);
3754 panic("bad nopwrite, hdr=%p exists=%p",
3755 (void *)hdr, (void *)exists);
3758 ASSERT(hdr->b_datacnt == 1);
3759 ASSERT(hdr->b_state == arc_anon);
3764 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3766 if (!exists && hdr->b_state == arc_anon)
3767 arc_access(hdr, hash_lock);
3770 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3773 ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3785 arc_buf_hdr_t *hdr = buf->b_hdr;
3791 ASSERT(!HDR_IO_ERROR(hdr));
3792 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3793 ASSERT(hdr->b_acb == NULL);
3795 hdr->b_flags |= ARC_L2CACHE;
3797 hdr->b_flags |= ARC_L2COMPRESS;
3804 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
4607 arc_buf_hdr_t *hdr;
4624 hdr = buf->b_hdr;
4625 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4631 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
4638 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
5195 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
5204 zio->io_orig_size = zio->io_size = hdr->b_size;
5214 ASSERT(hdr->b_buf != NULL);
5215 bzero(hdr->b_buf->b_data, hdr->b_size);
5216 zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
5236 hdr->b_size) != 0)
5242 zio->io_orig_size = zio->io_size = hdr->b_size;