1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 121 unchanged lines hidden (view full) ---

130#endif
131#include <sys/callb.h>
132#include <sys/kstat.h>
133#include <zfs_fletcher.h>
134#include <sys/sdt.h>
135
136#include <vm/vm_pageout.h>
137
138static kmutex_t arc_reclaim_thr_lock;
139static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */
140static uint8_t arc_thread_exit;
141
142extern int zfs_write_limit_shift;
143extern uint64_t zfs_write_limit_max;
144extern kmutex_t zfs_write_limit_lock;
145

--- 383 unchanged lines hidden (view full) ---

529
530static arc_buf_t *arc_eviction_list;
531static kmutex_t arc_eviction_mtx;
532static arc_buf_hdr_t arc_eviction_hdr;
533static void arc_get_data_buf(arc_buf_t *buf);
534static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
535static int arc_evict_needed(arc_buf_contents_t type);
536static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
537
538static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
539
540#define GHOST_STATE(state) \
541 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
542 (state) == arc_l2c_only)
543
544/*

--- 519 unchanged lines hidden (view full) ---

1064 if (buf->b_hdr->b_freeze_cksum != NULL) {
1065 mutex_exit(&buf->b_hdr->b_freeze_lock);
1066 return;
1067 }
1068 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1069 fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1070 buf->b_hdr->b_freeze_cksum);
1071 mutex_exit(&buf->b_hdr->b_freeze_lock);
1072}
1073
1074void
1075arc_buf_thaw(arc_buf_t *buf)
1076{
1077 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1078 if (buf->b_hdr->b_state != arc_anon)
1079 panic("modifying non-anon buffer!");
1080 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1081 panic("modifying buffer while i/o in progress!");

--- 8 unchanged lines hidden (view full) ---

1090
1091 if (zfs_flags & ZFS_DEBUG_MODIFY) {
1092 if (buf->b_hdr->b_thawed)
1093 kmem_free(buf->b_hdr->b_thawed, 1);
1094 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1095 }
1096
1097 mutex_exit(&buf->b_hdr->b_freeze_lock);
1098}
1099
1100void
1101arc_buf_freeze(arc_buf_t *buf)
1102{
1103 kmutex_t *hash_lock;
1104
1105 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1106 return;
1107
1108 hash_lock = HDR_LOCK(buf->b_hdr);
1109 mutex_enter(hash_lock);
1110
1111 ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1112 buf->b_hdr->b_state == arc_anon);
1113 arc_cksum_compute(buf, B_FALSE);
1114 mutex_exit(hash_lock);
1115}
1116
1117static void
1118get_buf_info(arc_buf_hdr_t *ab, arc_state_t *state, list_t **list, kmutex_t **lock)
1119{
1120 uint64_t buf_hashid = buf_hash(ab->b_spa, &ab->b_dva, ab->b_birth);
1121
1122 if (ab->b_type == ARC_BUFC_METADATA)

--- 21 unchanged lines hidden (view full) ---

1144 kmutex_t *lock;
1145
1146 get_buf_info(ab, ab->b_state, &list, &lock);
1147 ASSERT(!MUTEX_HELD(lock));
1148 mutex_enter(lock);
1149 ASSERT(list_link_active(&ab->b_arc_node));
1150 list_remove(list, ab);
1151 if (GHOST_STATE(ab->b_state)) {
1152 ASSERT3U(ab->b_datacnt, ==, 0);
1153 ASSERT3P(ab->b_buf, ==, NULL);
1154 delta = ab->b_size;
1155 }
1156 ASSERT(delta > 0);
1157 ASSERT3U(*size, >=, delta);
1158 atomic_add_64(size, -delta);
1159 mutex_exit(lock);
1160 /* remove the prefetch flag if we get a reference */

--- 330 unchanged lines hidden (view full) ---

1491 data, metadata, hits);
1492}
1493
1494/*
1495 * Free the arc data buffer. If it is an l2arc write in progress,
1496 * the buffer is placed on l2arc_free_on_write to be freed later.
1497 */
1498static void
1499arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t),
1500 void *data, size_t size)
1501{
1502 if (HDR_L2_WRITING(hdr)) {
1503 l2arc_data_free_t *df;
1504 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1505 df->l2df_data = data;
1506 df->l2df_size = size;
1507 df->l2df_func = free_func;
1508 mutex_enter(&l2arc_free_on_write_mtx);
1509 list_insert_head(l2arc_free_on_write, df);
1510 mutex_exit(&l2arc_free_on_write_mtx);
1511 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1512 } else {
1513 free_func(data, size);
1514 }
1515}
1516
1517static void
1518arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1519{
1520 arc_buf_t **bufp;
1521
1522 /* free up data associated with the buf */
1523 if (buf->b_data) {
1524 arc_state_t *state = buf->b_hdr->b_state;
1525 uint64_t size = buf->b_hdr->b_size;
1526 arc_buf_contents_t type = buf->b_hdr->b_type;
1527
1528 arc_cksum_verify(buf);
1529
1530 if (!recycle) {
1531 if (type == ARC_BUFC_METADATA) {
1532 arc_buf_data_free(buf->b_hdr, zio_buf_free,
1533 buf->b_data, size);
1534 arc_space_return(size, ARC_SPACE_DATA);
1535 } else {
1536 ASSERT(type == ARC_BUFC_DATA);
1537 arc_buf_data_free(buf->b_hdr,
1538 zio_data_buf_free, buf->b_data, size);
1539 ARCSTAT_INCR(arcstat_data_size, -size);
1540 atomic_add_64(&arc_size, -size);
1541 }
1542 }
1543 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1544 uint64_t *cnt = &state->arcs_lsize[type];
1545
1546 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));

--- 260 unchanged lines hidden (view full) ---

1807 }
1808 /* "lookahead" for better eviction candidate */
1809 if (recycle && ab->b_size != bytes &&
1810 ab_prev && ab_prev->b_size == bytes)
1811 continue;
1812 hash_lock = HDR_LOCK(ab);
1813 have_lock = MUTEX_HELD(hash_lock);
1814 if (have_lock || mutex_tryenter(hash_lock)) {
1815 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0);
1816 ASSERT(ab->b_datacnt > 0);
1817 while (ab->b_buf) {
1818 arc_buf_t *buf = ab->b_buf;
1819 if (!mutex_tryenter(&buf->b_evict_lock)) {
1820 missed += 1;
1821 break;
1822 }
1823 if (buf->b_data) {

--- 883 unchanged lines hidden (view full) ---

2707 * MFU state.
2708 */
2709
2710 if (buf->b_flags & ARC_PREFETCH) {
2711 /*
2712 * This is a prefetch access...
2713 * move this block back to the MRU state.
2714 */
2715 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0);
2716 new_state = arc_mru;
2717 }
2718
2719 buf->b_arc_access = ddi_get_lbolt();
2720 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2721 arc_change_state(new_state, buf, hash_lock);
2722
2723 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);

--- 65 unchanged lines hidden (view full) ---

2789 hdr->b_flags &= ~ARC_L2_EVICTED;
2790 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2791 hdr->b_flags &= ~ARC_L2CACHE;
2792
2793 /* byteswap if necessary */
2794 callback_list = hdr->b_acb;
2795 ASSERT(callback_list != NULL);
2796 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2797 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2798 byteswap_uint64_array :
2799 dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap;
2800 func(buf->b_data, hdr->b_size);
2801 }
2802
2803 arc_cksum_compute(buf, B_FALSE);
2804
2805 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2806 /*
2807 * Only call arc_access on anonymous buffers. This is because
2808 * if we've issued an I/O for an evicted buffer, we've already
2809 * called arc_access (to prevent any simultaneous readers from
2810 * getting confused).
2811 */

--- 234 unchanged lines hidden (view full) ---

3046 if (*arc_flags & ARC_L2CACHE)
3047 hdr->b_flags |= ARC_L2CACHE;
3048 if (BP_GET_LEVEL(bp) > 0)
3049 hdr->b_flags |= ARC_INDIRECT;
3050 } else {
3051 /* this block is in the ghost cache */
3052 ASSERT(GHOST_STATE(hdr->b_state));
3053 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3054 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0);
3055 ASSERT(hdr->b_buf == NULL);
3056
3057 /* if this is a prefetch, we don't have a reference */
3058 if (*arc_flags & ARC_PREFETCH)
3059 hdr->b_flags |= ARC_PREFETCH;
3060 else
3061 add_reference(hdr, hash_lock, private);
3062 if (*arc_flags & ARC_L2CACHE)

--- 297 unchanged lines hidden (view full) ---

3360 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3361 if (refcount_is_zero(&hdr->b_refcnt)) {
3362 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3363 ASSERT3U(*size, >=, hdr->b_size);
3364 atomic_add_64(size, -hdr->b_size);
3365 }
3366 hdr->b_datacnt -= 1;
3367 arc_cksum_verify(buf);
3368
3369 mutex_exit(hash_lock);
3370
3371 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3372 nhdr->b_size = blksz;
3373 nhdr->b_spa = spa;
3374 nhdr->b_type = type;
3375 nhdr->b_buf = buf;

--- 1364 unchanged lines hidden (view full) ---

4740 mutex_exit(list_lock);
4741
4742 if (full == B_TRUE)
4743 break;
4744 }
4745 mutex_exit(&l2arc_buflist_mtx);
4746
4747 if (pio == NULL) {
4748 ASSERT3U(write_sz, ==, 0);
4749 kmem_cache_free(hdr_cache, head);
4750 return (0);
4751 }
4752
4753 ASSERT3U(write_sz, <=, target_sz);
4754 ARCSTAT_BUMP(arcstat_l2_writes_sent);
4755 ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4756 ARCSTAT_INCR(arcstat_l2_size, write_sz);

--- 284 unchanged lines hidden ---