Deleted Added
sdiff udiff text old ( 242858 ) new ( 243674 )
full compact
arc.c (242858) arc.c (243674)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 121 unchanged lines hidden (view full) ---

130#endif
131#include <sys/callb.h>
132#include <sys/kstat.h>
133#include <zfs_fletcher.h>
134#include <sys/sdt.h>
135
136#include <vm/vm_pageout.h>
137
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 121 unchanged lines hidden (view full) ---

130#endif
131#include <sys/callb.h>
132#include <sys/kstat.h>
133#include <zfs_fletcher.h>
134#include <sys/sdt.h>
135
136#include <vm/vm_pageout.h>
137
138#ifdef illumos
139#ifndef _KERNEL
140/* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
141boolean_t arc_watch = B_FALSE;
142int arc_procfd;
143#endif
144#endif /* illumos */
145
138static kmutex_t arc_reclaim_thr_lock;
139static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */
140static uint8_t arc_thread_exit;
141
142extern int zfs_write_limit_shift;
143extern uint64_t zfs_write_limit_max;
144extern kmutex_t zfs_write_limit_lock;
145

--- 383 unchanged lines hidden (view full) ---

529
530static arc_buf_t *arc_eviction_list;
531static kmutex_t arc_eviction_mtx;
532static arc_buf_hdr_t arc_eviction_hdr;
533static void arc_get_data_buf(arc_buf_t *buf);
534static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
535static int arc_evict_needed(arc_buf_contents_t type);
536static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
146static kmutex_t arc_reclaim_thr_lock;
147static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */
148static uint8_t arc_thread_exit;
149
150extern int zfs_write_limit_shift;
151extern uint64_t zfs_write_limit_max;
152extern kmutex_t zfs_write_limit_lock;
153

--- 383 unchanged lines hidden (view full) ---

537
538static arc_buf_t *arc_eviction_list;
539static kmutex_t arc_eviction_mtx;
540static arc_buf_hdr_t arc_eviction_hdr;
541static void arc_get_data_buf(arc_buf_t *buf);
542static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
543static int arc_evict_needed(arc_buf_contents_t type);
544static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
545#ifdef illumos
546static void arc_buf_watch(arc_buf_t *buf);
547#endif /* illumos */
537
538static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
539
540#define GHOST_STATE(state) \
541 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
542 (state) == arc_l2c_only)
543
544/*

--- 519 unchanged lines hidden (view full) ---

1064 if (buf->b_hdr->b_freeze_cksum != NULL) {
1065 mutex_exit(&buf->b_hdr->b_freeze_lock);
1066 return;
1067 }
1068 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1069 fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1070 buf->b_hdr->b_freeze_cksum);
1071 mutex_exit(&buf->b_hdr->b_freeze_lock);
548
549static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
550
551#define GHOST_STATE(state) \
552 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
553 (state) == arc_l2c_only)
554
555/*

--- 519 unchanged lines hidden (view full) ---

1075 if (buf->b_hdr->b_freeze_cksum != NULL) {
1076 mutex_exit(&buf->b_hdr->b_freeze_lock);
1077 return;
1078 }
1079 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1080 fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1081 buf->b_hdr->b_freeze_cksum);
1082 mutex_exit(&buf->b_hdr->b_freeze_lock);
1083#ifdef illumos
1084 arc_buf_watch(buf);
1085#endif /* illumos */
1072}
1073
1086}
1087
1088#ifdef illumos
1089#ifndef _KERNEL
1090typedef struct procctl {
1091 long cmd;
1092 prwatch_t prwatch;
1093} procctl_t;
1094#endif
1095
1096/* ARGSUSED */
1097static void
1098arc_buf_unwatch(arc_buf_t *buf)
1099{
1100#ifndef _KERNEL
1101 if (arc_watch) {
1102 int result;
1103 procctl_t ctl;
1104 ctl.cmd = PCWATCH;
1105 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1106 ctl.prwatch.pr_size = 0;
1107 ctl.prwatch.pr_wflags = 0;
1108 result = write(arc_procfd, &ctl, sizeof (ctl));
1109 ASSERT3U(result, ==, sizeof (ctl));
1110 }
1111#endif
1112}
1113
1114/* ARGSUSED */
1115static void
1116arc_buf_watch(arc_buf_t *buf)
1117{
1118#ifndef _KERNEL
1119 if (arc_watch) {
1120 int result;
1121 procctl_t ctl;
1122 ctl.cmd = PCWATCH;
1123 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1124 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1125 ctl.prwatch.pr_wflags = WA_WRITE;
1126 result = write(arc_procfd, &ctl, sizeof (ctl));
1127 ASSERT3U(result, ==, sizeof (ctl));
1128 }
1129#endif
1130}
1131#endif /* illumos */
1132
1074void
1075arc_buf_thaw(arc_buf_t *buf)
1076{
1077 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1078 if (buf->b_hdr->b_state != arc_anon)
1079 panic("modifying non-anon buffer!");
1080 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1081 panic("modifying buffer while i/o in progress!");

--- 8 unchanged lines hidden (view full) ---

1090
1091 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1092 if (buf->b_hdr->b_thawed)
1093 kmem_free(buf->b_hdr->b_thawed, 1);
1094 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1095 }
1096
1097 mutex_exit(&buf->b_hdr->b_freeze_lock);
1133void
1134arc_buf_thaw(arc_buf_t *buf)
1135{
1136 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1137 if (buf->b_hdr->b_state != arc_anon)
1138 panic("modifying non-anon buffer!");
1139 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1140 panic("modifying buffer while i/o in progress!");

--- 8 unchanged lines hidden (view full) ---

1149
1150 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1151 if (buf->b_hdr->b_thawed)
1152 kmem_free(buf->b_hdr->b_thawed, 1);
1153 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1154 }
1155
1156 mutex_exit(&buf->b_hdr->b_freeze_lock);
1157
1158#ifdef illumos
1159 arc_buf_unwatch(buf);
1160#endif /* illumos */
1098}
1099
1100void
1101arc_buf_freeze(arc_buf_t *buf)
1102{
1103 kmutex_t *hash_lock;
1104
1105 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1106 return;
1107
1108 hash_lock = HDR_LOCK(buf->b_hdr);
1109 mutex_enter(hash_lock);
1110
1111 ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1112 buf->b_hdr->b_state == arc_anon);
1113 arc_cksum_compute(buf, B_FALSE);
1114 mutex_exit(hash_lock);
1161}
1162
1163void
1164arc_buf_freeze(arc_buf_t *buf)
1165{
1166 kmutex_t *hash_lock;
1167
1168 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1169 return;
1170
1171 hash_lock = HDR_LOCK(buf->b_hdr);
1172 mutex_enter(hash_lock);
1173
1174 ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1175 buf->b_hdr->b_state == arc_anon);
1176 arc_cksum_compute(buf, B_FALSE);
1177 mutex_exit(hash_lock);
1178
1115}
1116
1117static void
1118get_buf_info(arc_buf_hdr_t *ab, arc_state_t *state, list_t **list, kmutex_t **lock)
1119{
1120 uint64_t buf_hashid = buf_hash(ab->b_spa, &ab->b_dva, ab->b_birth);
1121
1122 if (ab->b_type == ARC_BUFC_METADATA)

--- 21 unchanged lines hidden (view full) ---

1144 kmutex_t *lock;
1145
1146 get_buf_info(ab, ab->b_state, &list, &lock);
1147 ASSERT(!MUTEX_HELD(lock));
1148 mutex_enter(lock);
1149 ASSERT(list_link_active(&ab->b_arc_node));
1150 list_remove(list, ab);
1151 if (GHOST_STATE(ab->b_state)) {
1179}
1180
1181static void
1182get_buf_info(arc_buf_hdr_t *ab, arc_state_t *state, list_t **list, kmutex_t **lock)
1183{
1184 uint64_t buf_hashid = buf_hash(ab->b_spa, &ab->b_dva, ab->b_birth);
1185
1186 if (ab->b_type == ARC_BUFC_METADATA)

--- 21 unchanged lines hidden (view full) ---

1208 kmutex_t *lock;
1209
1210 get_buf_info(ab, ab->b_state, &list, &lock);
1211 ASSERT(!MUTEX_HELD(lock));
1212 mutex_enter(lock);
1213 ASSERT(list_link_active(&ab->b_arc_node));
1214 list_remove(list, ab);
1215 if (GHOST_STATE(ab->b_state)) {
1152 ASSERT3U(ab->b_datacnt, ==, 0);
1216 ASSERT0(ab->b_datacnt);
1153 ASSERT3P(ab->b_buf, ==, NULL);
1154 delta = ab->b_size;
1155 }
1156 ASSERT(delta > 0);
1157 ASSERT3U(*size, >=, delta);
1158 atomic_add_64(size, -delta);
1159 mutex_exit(lock);
1160 /* remove the prefetch flag if we get a reference */

--- 330 unchanged lines hidden (view full) ---

1491 data, metadata, hits);
1492}
1493
1494/*
1495 * Free the arc data buffer. If it is an l2arc write in progress,
1496 * the buffer is placed on l2arc_free_on_write to be freed later.
1497 */
1498static void
1217 ASSERT3P(ab->b_buf, ==, NULL);
1218 delta = ab->b_size;
1219 }
1220 ASSERT(delta > 0);
1221 ASSERT3U(*size, >=, delta);
1222 atomic_add_64(size, -delta);
1223 mutex_exit(lock);
1224 /* remove the prefetch flag if we get a reference */

--- 330 unchanged lines hidden (view full) ---

1555 data, metadata, hits);
1556}
1557
1558/*
1559 * Free the arc data buffer. If it is an l2arc write in progress,
1560 * the buffer is placed on l2arc_free_on_write to be freed later.
1561 */
1562static void
1499arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t),
1500 void *data, size_t size)
1563arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1501{
1564{
1565 arc_buf_hdr_t *hdr = buf->b_hdr;
1566
1502 if (HDR_L2_WRITING(hdr)) {
1503 l2arc_data_free_t *df;
1504 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1567 if (HDR_L2_WRITING(hdr)) {
1568 l2arc_data_free_t *df;
1569 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1505 df->l2df_data = data;
1506 df->l2df_size = size;
1570 df->l2df_data = buf->b_data;
1571 df->l2df_size = hdr->b_size;
1507 df->l2df_func = free_func;
1508 mutex_enter(&l2arc_free_on_write_mtx);
1509 list_insert_head(l2arc_free_on_write, df);
1510 mutex_exit(&l2arc_free_on_write_mtx);
1511 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1512 } else {
1572 df->l2df_func = free_func;
1573 mutex_enter(&l2arc_free_on_write_mtx);
1574 list_insert_head(l2arc_free_on_write, df);
1575 mutex_exit(&l2arc_free_on_write_mtx);
1576 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1577 } else {
1513 free_func(data, size);
1578 free_func(buf->b_data, hdr->b_size);
1514 }
1515}
1516
1517static void
1518arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1519{
1520 arc_buf_t **bufp;
1521
1522 /* free up data associated with the buf */
1523 if (buf->b_data) {
1524 arc_state_t *state = buf->b_hdr->b_state;
1525 uint64_t size = buf->b_hdr->b_size;
1526 arc_buf_contents_t type = buf->b_hdr->b_type;
1527
1528 arc_cksum_verify(buf);
1579 }
1580}
1581
1582static void
1583arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1584{
1585 arc_buf_t **bufp;
1586
1587 /* free up data associated with the buf */
1588 if (buf->b_data) {
1589 arc_state_t *state = buf->b_hdr->b_state;
1590 uint64_t size = buf->b_hdr->b_size;
1591 arc_buf_contents_t type = buf->b_hdr->b_type;
1592
1593 arc_cksum_verify(buf);
1594#ifdef illumos
1595 arc_buf_unwatch(buf);
1596#endif /* illumos */
1529
1530 if (!recycle) {
1531 if (type == ARC_BUFC_METADATA) {
1597
1598 if (!recycle) {
1599 if (type == ARC_BUFC_METADATA) {
1532 arc_buf_data_free(buf->b_hdr, zio_buf_free,
1533 buf->b_data, size);
1600 arc_buf_data_free(buf, zio_buf_free);
1534 arc_space_return(size, ARC_SPACE_DATA);
1535 } else {
1536 ASSERT(type == ARC_BUFC_DATA);
1601 arc_space_return(size, ARC_SPACE_DATA);
1602 } else {
1603 ASSERT(type == ARC_BUFC_DATA);
1537 arc_buf_data_free(buf->b_hdr,
1538 zio_data_buf_free, buf->b_data, size);
1604 arc_buf_data_free(buf, zio_data_buf_free);
1539 ARCSTAT_INCR(arcstat_data_size, -size);
1540 atomic_add_64(&arc_size, -size);
1541 }
1542 }
1543 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1544 uint64_t *cnt = &state->arcs_lsize[type];
1545
1546 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));

--- 260 unchanged lines hidden (view full) ---

1807 }
1808 /* "lookahead" for better eviction candidate */
1809 if (recycle && ab->b_size != bytes &&
1810 ab_prev && ab_prev->b_size == bytes)
1811 continue;
1812 hash_lock = HDR_LOCK(ab);
1813 have_lock = MUTEX_HELD(hash_lock);
1814 if (have_lock || mutex_tryenter(hash_lock)) {
1605 ARCSTAT_INCR(arcstat_data_size, -size);
1606 atomic_add_64(&arc_size, -size);
1607 }
1608 }
1609 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1610 uint64_t *cnt = &state->arcs_lsize[type];
1611
1612 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));

--- 260 unchanged lines hidden (view full) ---

1873 }
1874 /* "lookahead" for better eviction candidate */
1875 if (recycle && ab->b_size != bytes &&
1876 ab_prev && ab_prev->b_size == bytes)
1877 continue;
1878 hash_lock = HDR_LOCK(ab);
1879 have_lock = MUTEX_HELD(hash_lock);
1880 if (have_lock || mutex_tryenter(hash_lock)) {
1815 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0);
1881 ASSERT0(refcount_count(&ab->b_refcnt));
1816 ASSERT(ab->b_datacnt > 0);
1817 while (ab->b_buf) {
1818 arc_buf_t *buf = ab->b_buf;
1819 if (!mutex_tryenter(&buf->b_evict_lock)) {
1820 missed += 1;
1821 break;
1822 }
1823 if (buf->b_data) {

--- 883 unchanged lines hidden (view full) ---

2707 * MFU state.
2708 */
2709
2710 if (buf->b_flags & ARC_PREFETCH) {
2711 /*
2712 * This is a prefetch access...
2713 * move this block back to the MRU state.
2714 */
1882 ASSERT(ab->b_datacnt > 0);
1883 while (ab->b_buf) {
1884 arc_buf_t *buf = ab->b_buf;
1885 if (!mutex_tryenter(&buf->b_evict_lock)) {
1886 missed += 1;
1887 break;
1888 }
1889 if (buf->b_data) {

--- 883 unchanged lines hidden (view full) ---

2773 * MFU state.
2774 */
2775
2776 if (buf->b_flags & ARC_PREFETCH) {
2777 /*
2778 * This is a prefetch access...
2779 * move this block back to the MRU state.
2780 */
2715 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0);
2781 ASSERT0(refcount_count(&buf->b_refcnt));
2716 new_state = arc_mru;
2717 }
2718
2719 buf->b_arc_access = ddi_get_lbolt();
2720 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2721 arc_change_state(new_state, buf, hash_lock);
2722
2723 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);

--- 65 unchanged lines hidden (view full) ---

2789 hdr->b_flags &= ~ARC_L2_EVICTED;
2790 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2791 hdr->b_flags &= ~ARC_L2CACHE;
2792
2793 /* byteswap if necessary */
2794 callback_list = hdr->b_acb;
2795 ASSERT(callback_list != NULL);
2796 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2782 new_state = arc_mru;
2783 }
2784
2785 buf->b_arc_access = ddi_get_lbolt();
2786 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2787 arc_change_state(new_state, buf, hash_lock);
2788
2789 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);

--- 65 unchanged lines hidden (view full) ---

2855 hdr->b_flags &= ~ARC_L2_EVICTED;
2856 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2857 hdr->b_flags &= ~ARC_L2CACHE;
2858
2859 /* byteswap if necessary */
2860 callback_list = hdr->b_acb;
2861 ASSERT(callback_list != NULL);
2862 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2863 dmu_object_byteswap_t bswap =
2864 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2797 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2798 byteswap_uint64_array :
2865 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2866 byteswap_uint64_array :
2799 dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap;
2867 dmu_ot_byteswap[bswap].ob_func;
2800 func(buf->b_data, hdr->b_size);
2801 }
2802
2803 arc_cksum_compute(buf, B_FALSE);
2868 func(buf->b_data, hdr->b_size);
2869 }
2870
2871 arc_cksum_compute(buf, B_FALSE);
2872#ifdef illumos
2873 arc_buf_watch(buf);
2874#endif /* illumos */
2804
2805 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2806 /*
2807 * Only call arc_access on anonymous buffers. This is because
2808 * if we've issued an I/O for an evicted buffer, we've already
2809 * called arc_access (to prevent any simultaneous readers from
2810 * getting confused).
2811 */

--- 234 unchanged lines hidden (view full) ---

3046 if (*arc_flags & ARC_L2CACHE)
3047 hdr->b_flags |= ARC_L2CACHE;
3048 if (BP_GET_LEVEL(bp) > 0)
3049 hdr->b_flags |= ARC_INDIRECT;
3050 } else {
3051 /* this block is in the ghost cache */
3052 ASSERT(GHOST_STATE(hdr->b_state));
3053 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2875
2876 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2877 /*
2878 * Only call arc_access on anonymous buffers. This is because
2879 * if we've issued an I/O for an evicted buffer, we've already
2880 * called arc_access (to prevent any simultaneous readers from
2881 * getting confused).
2882 */

--- 234 unchanged lines hidden (view full) ---

3117 if (*arc_flags & ARC_L2CACHE)
3118 hdr->b_flags |= ARC_L2CACHE;
3119 if (BP_GET_LEVEL(bp) > 0)
3120 hdr->b_flags |= ARC_INDIRECT;
3121 } else {
3122 /* this block is in the ghost cache */
3123 ASSERT(GHOST_STATE(hdr->b_state));
3124 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3054 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0);
3125 ASSERT0(refcount_count(&hdr->b_refcnt));
3055 ASSERT(hdr->b_buf == NULL);
3056
3057 /* if this is a prefetch, we don't have a reference */
3058 if (*arc_flags & ARC_PREFETCH)
3059 hdr->b_flags |= ARC_PREFETCH;
3060 else
3061 add_reference(hdr, hash_lock, private);
3062 if (*arc_flags & ARC_L2CACHE)

--- 297 unchanged lines hidden (view full) ---

3360 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3361 if (refcount_is_zero(&hdr->b_refcnt)) {
3362 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3363 ASSERT3U(*size, >=, hdr->b_size);
3364 atomic_add_64(size, -hdr->b_size);
3365 }
3366 hdr->b_datacnt -= 1;
3367 arc_cksum_verify(buf);
3126 ASSERT(hdr->b_buf == NULL);
3127
3128 /* if this is a prefetch, we don't have a reference */
3129 if (*arc_flags & ARC_PREFETCH)
3130 hdr->b_flags |= ARC_PREFETCH;
3131 else
3132 add_reference(hdr, hash_lock, private);
3133 if (*arc_flags & ARC_L2CACHE)

--- 297 unchanged lines hidden (view full) ---

3431 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3432 if (refcount_is_zero(&hdr->b_refcnt)) {
3433 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3434 ASSERT3U(*size, >=, hdr->b_size);
3435 atomic_add_64(size, -hdr->b_size);
3436 }
3437 hdr->b_datacnt -= 1;
3438 arc_cksum_verify(buf);
3439#ifdef illumos
3440 arc_buf_unwatch(buf);
3441#endif /* illumos */
3368
3369 mutex_exit(hash_lock);
3370
3371 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3372 nhdr->b_size = blksz;
3373 nhdr->b_spa = spa;
3374 nhdr->b_type = type;
3375 nhdr->b_buf = buf;

--- 1364 unchanged lines hidden (view full) ---

4740 mutex_exit(list_lock);
4741
4742 if (full == B_TRUE)
4743 break;
4744 }
4745 mutex_exit(&l2arc_buflist_mtx);
4746
4747 if (pio == NULL) {
3442
3443 mutex_exit(hash_lock);
3444
3445 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3446 nhdr->b_size = blksz;
3447 nhdr->b_spa = spa;
3448 nhdr->b_type = type;
3449 nhdr->b_buf = buf;

--- 1364 unchanged lines hidden (view full) ---

4814 mutex_exit(list_lock);
4815
4816 if (full == B_TRUE)
4817 break;
4818 }
4819 mutex_exit(&l2arc_buflist_mtx);
4820
4821 if (pio == NULL) {
4748 ASSERT3U(write_sz, ==, 0);
4822 ASSERT0(write_sz);
4749 kmem_cache_free(hdr_cache, head);
4750 return (0);
4751 }
4752
4753 ASSERT3U(write_sz, <=, target_sz);
4754 ARCSTAT_BUMP(arcstat_l2_writes_sent);
4755 ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4756 ARCSTAT_INCR(arcstat_l2_size, write_sz);

--- 284 unchanged lines hidden --- 		4823 kmem_cache_free(hdr_cache, head);
4824 return (0);
4825 }
4826
4827 ASSERT3U(write_sz, <=, target_sz);
4828 ARCSTAT_BUMP(arcstat_l2_writes_sent);
4829 ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4830 ARCSTAT_INCR(arcstat_l2_size, write_sz);

--- 284 unchanged lines hidden ---