Lines Matching refs:b_l1hdr
1140 l1arc_buf_hdr_t b_l1hdr;
1303 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
1678 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1679 refcount_create(&hdr->b_l1hdr.b_refcnt);
1680 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1681 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1723 cv_destroy(&hdr->b_l1hdr.b_cv);
1724 refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1725 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1726 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1840 buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1841 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1842 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1863 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1864 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1865 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1866 hdr->b_l1hdr.b_freeze_cksum = NULL;
1868 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1877 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1900 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1907 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1908 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1909 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1914 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1916 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2005 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
2006 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
2008 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2011 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2016 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
2019 hdr->b_l1hdr.b_freeze_cksum);
2020 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2111 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2121 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2129 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
2132 if (hdr->b_l1hdr.b_thawed != NULL)
2133 kmem_free(hdr->b_l1hdr.b_thawed, 1);
2134 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
2138 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
2155 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
2164 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
2165 hdr->b_l1hdr.b_state == arc_anon);
2238 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
2256 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
2279 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2287 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2335 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
2339 hdr->b_l1hdr.b_pabd, buf->b_data,
2406 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2407 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2408 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2415 if (hdr->b_l1hdr.b_pabd != NULL) {
2419 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2441 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2442 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2443 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2450 if (hdr->b_l1hdr.b_pabd != NULL) {
2454 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2474 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2475 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2476 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2479 arc_state_t *state = hdr->b_l1hdr.b_state;
2481 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2503 arc_state_t *state = hdr->b_l1hdr.b_state;
2513 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2516 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2544 old_state = hdr->b_l1hdr.b_state;
2545 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2546 bufcnt = hdr->b_l1hdr.b_bufcnt;
2547 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL);
2572 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2590 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2617 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2626 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2646 if (hdr->b_l1hdr.b_pabd != NULL) {
2659 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2679 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2699 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2706 hdr->b_l1hdr.b_state = new_state;
2816 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2842 buf->b_next = hdr->b_l1hdr.b_buf;
2875 abd_is_linear(hdr->b_l1hdr.b_pabd);
2879 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2889 hdr->b_l1hdr.b_buf = buf;
2890 hdr->b_l1hdr.b_bufcnt += 1;
2954 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2955 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2968 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2969 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2990 arc_state_t *state = hdr->b_l1hdr.b_state;
2995 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2996 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3010 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
3021 arc_state_t *state = hdr->b_l1hdr.b_state;
3024 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3033 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
3034 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
3052 arc_state_t *state = hdr->b_l1hdr.b_state;
3055 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3064 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3065 abd_put(hdr->b_l1hdr.b_pabd);
3066 hdr->b_l1hdr.b_pabd = NULL;
3089 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3112 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
3113 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
3154 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3155 hdr->b_l1hdr.b_bufcnt -= 1;
3178 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3223 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3224 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, do_adapt);
3225 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3226 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3236 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3248 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
3251 hdr->b_l1hdr.b_pabd = NULL;
3252 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3268 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3269 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
3278 hdr->b_l1hdr.b_state = arc_anon;
3279 hdr->b_l1hdr.b_arc_access = 0;
3280 hdr->b_l1hdr.b_bufcnt = 0;
3281 hdr->b_l1hdr.b_buf = NULL;
3289 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3326 nhdr->b_l1hdr.b_state = arc_l2c_only;
3329 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3331 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3332 ASSERT0(hdr->b_l1hdr.b_bufcnt);
3333 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3342 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3347 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3351 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3354 if (hdr->b_l1hdr.b_thawed != NULL) {
3355 kmem_free(hdr->b_l1hdr.b_thawed, 1);
3356 hdr->b_l1hdr.b_thawed = NULL;
3438 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3447 ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd));
3480 ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3481 hdr->b_l1hdr.b_bufcnt > 0);
3482 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3483 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3519 while (hdr->b_l1hdr.b_buf != NULL)
3520 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3523 if (hdr->b_l1hdr.b_thawed != NULL) {
3524 kmem_free(hdr->b_l1hdr.b_thawed, 1);
3525 hdr->b_l1hdr.b_thawed = NULL;
3529 if (hdr->b_l1hdr.b_pabd != NULL) {
3536 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3537 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3550 if (hdr->b_l1hdr.b_state == arc_anon) {
3551 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3560 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3562 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3593 state = hdr->b_l1hdr.b_state;
3596 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3615 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3641 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) {
3646 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3647 while (hdr->b_l1hdr.b_buf) {
3648 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3671 if (hdr->b_l1hdr.b_bufcnt == 0) {
4078 if (data_hdr->b_l1hdr.b_arc_access <
4079 meta_hdr->b_l1hdr.b_arc_access) {
4889 arc_state_t *state = hdr->b_l1hdr.b_state;
4956 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4957 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4967 hdr->b_l1hdr.b_state == arc_anon &&
5002 arc_state_t *state = hdr->b_l1hdr.b_state;
5006 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5007 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5036 if (hdr->b_l1hdr.b_state == arc_anon) {
5043 ASSERT0(hdr->b_l1hdr.b_arc_access);
5044 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5048 } else if (hdr->b_l1hdr.b_state == arc_mru) {
5060 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5063 &hdr->b_l1hdr.b_arc_node));
5070 hdr->b_l1hdr.b_arc_access = now;
5079 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
5085 hdr->b_l1hdr.b_arc_access = now;
5090 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5100 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) {
5111 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5115 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
5127 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5128 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5144 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5149 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5154 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
5177 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5186 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5195 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5196 hdr->b_l1hdr.b_state == arc_mfu);
5291 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5293 hdr->b_l1hdr.b_byteswap =
5297 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5305 callback_list = hdr->b_l1hdr.b_acb;
5308 if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5363 hdr->b_l1hdr.b_acb = NULL;
5367 ASSERT0(hdr->b_l1hdr.b_bufcnt);
5368 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5371 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5378 if (hdr->b_l1hdr.b_state != arc_anon)
5382 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5390 cv_broadcast(&hdr->b_l1hdr.b_cv);
5401 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5402 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5476 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) {
5481 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5502 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
5522 acb->acb_next = hdr->b_l1hdr.b_acb;
5523 hdr->b_l1hdr.b_acb = acb;
5531 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5532 hdr->b_l1hdr.b_state == arc_mfu);
5568 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
5623 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5624 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
5626 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5627 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5628 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5640 arc_adapt(arc_hdr_size(hdr), hdr->b_l1hdr.b_state);
5644 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5668 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5675 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5676 hdr->b_l1hdr.b_acb = acb;
5759 abd = hdr->b_l1hdr.b_pabd;
5821 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size,
5876 refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
5912 if (hdr->b_l1hdr.b_state == arc_anon) {
5918 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5919 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
5920 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
5922 hdr->b_l1hdr.b_arc_access = 0;
5942 arc_state_t *state = hdr->b_l1hdr.b_state;
5947 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
5971 if (hdr->b_l1hdr.b_bufcnt > 1) {
5980 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
5984 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6019 abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6036 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6042 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6048 hdr->b_l1hdr.b_bufcnt -= 1;
6061 ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6062 ASSERT0(nhdr->b_l1hdr.b_bufcnt);
6063 ASSERT0(refcount_count(&nhdr->b_l1hdr.b_refcnt));
6067 nhdr->b_l1hdr.b_buf = buf;
6068 nhdr->b_l1hdr.b_bufcnt = 1;
6069 (void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6077 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6079 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6082 hdr->b_l1hdr.b_arc_access = 0;
6097 buf->b_hdr->b_l1hdr.b_state == arc_anon);
6109 referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6124 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6125 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
6137 if (hdr->b_l1hdr.b_pabd != NULL) {
6145 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6193 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6197 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6203 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
6239 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6280 &exists->b_l1hdr.b_refcnt));
6294 ASSERT(hdr->b_l1hdr.b_bufcnt == 1);
6295 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6302 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6309 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
6332 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6333 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
6359 if (hdr->b_l1hdr.b_pabd != NULL) {
6375 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6635 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6639 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6643 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6647 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6651 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6655 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6659 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6663 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6667 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6671 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
7444 abd_copy(hdr->b_l1hdr.b_pabd, cb->l2rcb_abd,
7461 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
7469 ASSERT3P(zio->io_abd, ==, hdr->b_l1hdr.b_pabd);
7502 hdr->b_l1hdr.b_pabd, zio->io_size, arc_read_done,
7505 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
7648 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
7767 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
7826 to_write = hdr->b_l1hdr.b_pabd;
7830 abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize);