Deleted Added
sdiff udiff text old ( 241773 ) new ( 242845 )
full compact
arc.c (241773) arc.c (242845)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 177 unchanged lines hidden (view full) ---

186 * These tunables are for performance analysis.
187 */
188uint64_t zfs_arc_max;
189uint64_t zfs_arc_min;
190uint64_t zfs_arc_meta_limit = 0;
191int zfs_arc_grow_retry = 0;
192int zfs_arc_shrink_shift = 0;
193int zfs_arc_p_min_shift = 0;
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 177 unchanged lines hidden (view full) ---

186 * These tunables are for performance analysis.
187 */
188uint64_t zfs_arc_max;
189uint64_t zfs_arc_min;
190uint64_t zfs_arc_meta_limit = 0;
191int zfs_arc_grow_retry = 0;
192int zfs_arc_shrink_shift = 0;
193int zfs_arc_p_min_shift = 0;
194int zfs_disable_dup_eviction = 0;
194
195TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);
196TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);
197TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
198SYSCTL_DECL(_vfs_zfs);
199SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_max, CTLFLAG_RDTUN, &zfs_arc_max, 0,
200 "Maximum ARC size");
201SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_RDTUN, &zfs_arc_min, 0,

--- 114 unchanged lines hidden (view full) ---

316 kstat_named_t arcstat_l2_evict_lock_retry;
317 kstat_named_t arcstat_l2_evict_reading;
318 kstat_named_t arcstat_l2_free_on_write;
319 kstat_named_t arcstat_l2_abort_lowmem;
320 kstat_named_t arcstat_l2_cksum_bad;
321 kstat_named_t arcstat_l2_io_error;
322 kstat_named_t arcstat_l2_size;
323 kstat_named_t arcstat_l2_hdr_size;
195
196TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);
197TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);
198TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
199SYSCTL_DECL(_vfs_zfs);
200SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_max, CTLFLAG_RDTUN, &zfs_arc_max, 0,
201 "Maximum ARC size");
202SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_RDTUN, &zfs_arc_min, 0,

--- 114 unchanged lines hidden (view full) ---

317 kstat_named_t arcstat_l2_evict_lock_retry;
318 kstat_named_t arcstat_l2_evict_reading;
319 kstat_named_t arcstat_l2_free_on_write;
320 kstat_named_t arcstat_l2_abort_lowmem;
321 kstat_named_t arcstat_l2_cksum_bad;
322 kstat_named_t arcstat_l2_io_error;
323 kstat_named_t arcstat_l2_size;
324 kstat_named_t arcstat_l2_hdr_size;
324 kstat_named_t arcstat_memory_throttle_count;
325 kstat_named_t arcstat_l2_write_trylock_fail;
326 kstat_named_t arcstat_l2_write_passed_headroom;
327 kstat_named_t arcstat_l2_write_spa_mismatch;
328 kstat_named_t arcstat_l2_write_in_l2;
329 kstat_named_t arcstat_l2_write_hdr_io_in_progress;
330 kstat_named_t arcstat_l2_write_not_cacheable;
331 kstat_named_t arcstat_l2_write_full;
332 kstat_named_t arcstat_l2_write_buffer_iter;
333 kstat_named_t arcstat_l2_write_pios;
334 kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
335 kstat_named_t arcstat_l2_write_buffer_list_iter;
336 kstat_named_t arcstat_l2_write_buffer_list_null_iter;
325 kstat_named_t arcstat_l2_write_trylock_fail;
326 kstat_named_t arcstat_l2_write_passed_headroom;
327 kstat_named_t arcstat_l2_write_spa_mismatch;
328 kstat_named_t arcstat_l2_write_in_l2;
329 kstat_named_t arcstat_l2_write_hdr_io_in_progress;
330 kstat_named_t arcstat_l2_write_not_cacheable;
331 kstat_named_t arcstat_l2_write_full;
332 kstat_named_t arcstat_l2_write_buffer_iter;
333 kstat_named_t arcstat_l2_write_pios;
334 kstat_named_t arcstat_l2_write_buffer_bytes_scanned;
335 kstat_named_t arcstat_l2_write_buffer_list_iter;
336 kstat_named_t arcstat_l2_write_buffer_list_null_iter;
337 kstat_named_t arcstat_memory_throttle_count;
338 kstat_named_t arcstat_duplicate_buffers;
339 kstat_named_t arcstat_duplicate_buffers_size;
340 kstat_named_t arcstat_duplicate_reads;
337} arc_stats_t;
338
339static arc_stats_t arc_stats = {
340 { "hits", KSTAT_DATA_UINT64 },
341 { "misses", KSTAT_DATA_UINT64 },
342 { "demand_data_hits", KSTAT_DATA_UINT64 },
343 { "demand_data_misses", KSTAT_DATA_UINT64 },
344 { "demand_metadata_hits", KSTAT_DATA_UINT64 },

--- 41 unchanged lines hidden (view full) ---

386 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
387 { "l2_evict_reading", KSTAT_DATA_UINT64 },
388 { "l2_free_on_write", KSTAT_DATA_UINT64 },
389 { "l2_abort_lowmem", KSTAT_DATA_UINT64 },
390 { "l2_cksum_bad", KSTAT_DATA_UINT64 },
391 { "l2_io_error", KSTAT_DATA_UINT64 },
392 { "l2_size", KSTAT_DATA_UINT64 },
393 { "l2_hdr_size", KSTAT_DATA_UINT64 },
341} arc_stats_t;
342
343static arc_stats_t arc_stats = {
344 { "hits", KSTAT_DATA_UINT64 },
345 { "misses", KSTAT_DATA_UINT64 },
346 { "demand_data_hits", KSTAT_DATA_UINT64 },
347 { "demand_data_misses", KSTAT_DATA_UINT64 },
348 { "demand_metadata_hits", KSTAT_DATA_UINT64 },

--- 41 unchanged lines hidden (view full) ---

390 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
391 { "l2_evict_reading", KSTAT_DATA_UINT64 },
392 { "l2_free_on_write", KSTAT_DATA_UINT64 },
393 { "l2_abort_lowmem", KSTAT_DATA_UINT64 },
394 { "l2_cksum_bad", KSTAT_DATA_UINT64 },
395 { "l2_io_error", KSTAT_DATA_UINT64 },
396 { "l2_size", KSTAT_DATA_UINT64 },
397 { "l2_hdr_size", KSTAT_DATA_UINT64 },
394 { "memory_throttle_count", KSTAT_DATA_UINT64 },
395 { "l2_write_trylock_fail", KSTAT_DATA_UINT64 },
396 { "l2_write_passed_headroom", KSTAT_DATA_UINT64 },
397 { "l2_write_spa_mismatch", KSTAT_DATA_UINT64 },
398 { "l2_write_in_l2", KSTAT_DATA_UINT64 },
399 { "l2_write_io_in_progress", KSTAT_DATA_UINT64 },
400 { "l2_write_not_cacheable", KSTAT_DATA_UINT64 },
401 { "l2_write_full", KSTAT_DATA_UINT64 },
402 { "l2_write_buffer_iter", KSTAT_DATA_UINT64 },
403 { "l2_write_pios", KSTAT_DATA_UINT64 },
404 { "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
405 { "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 },
398 { "l2_write_trylock_fail", KSTAT_DATA_UINT64 },
399 { "l2_write_passed_headroom", KSTAT_DATA_UINT64 },
400 { "l2_write_spa_mismatch", KSTAT_DATA_UINT64 },
401 { "l2_write_in_l2", KSTAT_DATA_UINT64 },
402 { "l2_write_io_in_progress", KSTAT_DATA_UINT64 },
403 { "l2_write_not_cacheable", KSTAT_DATA_UINT64 },
404 { "l2_write_full", KSTAT_DATA_UINT64 },
405 { "l2_write_buffer_iter", KSTAT_DATA_UINT64 },
406 { "l2_write_pios", KSTAT_DATA_UINT64 },
407 { "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 },
408 { "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 },
406 { "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 }
409 { "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 },
410 { "memory_throttle_count", KSTAT_DATA_UINT64 },
411 { "duplicate_buffers", KSTAT_DATA_UINT64 },
412 { "duplicate_buffers_size", KSTAT_DATA_UINT64 },
413 { "duplicate_reads", KSTAT_DATA_UINT64 }
407};
408
409#define ARCSTAT(stat) (arc_stats.stat.value.ui64)
410
411#define ARCSTAT_INCR(stat, val) \
412 atomic_add_64(&arc_stats.stat.value.ui64, (val));
413
414#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1)

--- 1098 unchanged lines hidden (view full) ---

1513 buf->b_hdr = hdr;
1514 buf->b_data = NULL;
1515 buf->b_efunc = NULL;
1516 buf->b_private = NULL;
1517 buf->b_next = hdr->b_buf;
1518 hdr->b_buf = buf;
1519 arc_get_data_buf(buf);
1520 bcopy(from->b_data, buf->b_data, size);
414};
415
416#define ARCSTAT(stat) (arc_stats.stat.value.ui64)
417
418#define ARCSTAT_INCR(stat, val) \
419 atomic_add_64(&arc_stats.stat.value.ui64, (val));
420
421#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1)

--- 1098 unchanged lines hidden (view full) ---

1520 buf->b_hdr = hdr;
1521 buf->b_data = NULL;
1522 buf->b_efunc = NULL;
1523 buf->b_private = NULL;
1524 buf->b_next = hdr->b_buf;
1525 hdr->b_buf = buf;
1526 arc_get_data_buf(buf);
1527 bcopy(from->b_data, buf->b_data, size);
1528
1529 /*
1530 * This buffer already exists in the arc so create a duplicate
1531 * copy for the caller. If the buffer is associated with user data
1532 * then track the size and number of duplicates. These stats will be
1533 * updated as duplicate buffers are created and destroyed.
1534 */
1535 if (hdr->b_type == ARC_BUFC_DATA) {
1536 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1537 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1538 }
1521 hdr->b_datacnt += 1;
1522 return (buf);
1523}
1524
1525void
1526arc_buf_add_ref(arc_buf_t *buf, void* tag)
1527{
1528 arc_buf_hdr_t *hdr;

--- 84 unchanged lines hidden (view full) ---

1613 ASSERT(state != arc_anon);
1614
1615 ASSERT3U(*cnt, >=, size);
1616 atomic_add_64(cnt, -size);
1617 }
1618 ASSERT3U(state->arcs_size, >=, size);
1619 atomic_add_64(&state->arcs_size, -size);
1620 buf->b_data = NULL;
1539 hdr->b_datacnt += 1;
1540 return (buf);
1541}
1542
1543void
1544arc_buf_add_ref(arc_buf_t *buf, void* tag)
1545{
1546 arc_buf_hdr_t *hdr;

--- 84 unchanged lines hidden (view full) ---

1631 ASSERT(state != arc_anon);
1632
1633 ASSERT3U(*cnt, >=, size);
1634 atomic_add_64(cnt, -size);
1635 }
1636 ASSERT3U(state->arcs_size, >=, size);
1637 atomic_add_64(&state->arcs_size, -size);
1638 buf->b_data = NULL;
1639
1640 /*
1641 * If we're destroying a duplicate buffer make sure
1642 * that the appropriate statistics are updated.
1643 */
1644 if (buf->b_hdr->b_datacnt > 1 &&
1645 buf->b_hdr->b_type == ARC_BUFC_DATA) {
1646 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
1647 ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
1648 }
1621 ASSERT(buf->b_hdr->b_datacnt > 0);
1622 buf->b_hdr->b_datacnt -= 1;
1623 }
1624
1625 /* only remove the buf if requested */
1626 if (!all)
1627 return;
1628

--- 168 unchanged lines hidden (view full) ---

1797
1798int
1799arc_buf_size(arc_buf_t *buf)
1800{
1801 return (buf->b_hdr->b_size);
1802}
1803
1804/*
1649 ASSERT(buf->b_hdr->b_datacnt > 0);
1650 buf->b_hdr->b_datacnt -= 1;
1651 }
1652
1653 /* only remove the buf if requested */
1654 if (!all)
1655 return;
1656

--- 168 unchanged lines hidden (view full) ---

1825
1826int
1827arc_buf_size(arc_buf_t *buf)
1828{
1829 return (buf->b_hdr->b_size);
1830}
1831
1832/*
1833 * Called from the DMU to determine if the current buffer should be
1834 * evicted. In order to ensure proper locking, the eviction must be initiated
1835 * from the DMU. Return true if the buffer is associated with user data and
1836 * duplicate buffers still exist.
1837 */
1838boolean_t
1839arc_buf_eviction_needed(arc_buf_t *buf)
1840{
1841 arc_buf_hdr_t *hdr;
1842 boolean_t evict_needed = B_FALSE;
1843
1844 if (zfs_disable_dup_eviction)
1845 return (B_FALSE);
1846
1847 mutex_enter(&buf->b_evict_lock);
1848 hdr = buf->b_hdr;
1849 if (hdr == NULL) {
1850 /*
1851 * We are in arc_do_user_evicts(); let that function
1852 * perform the eviction.
1853 */
1854 ASSERT(buf->b_data == NULL);
1855 mutex_exit(&buf->b_evict_lock);
1856 return (B_FALSE);
1857 } else if (buf->b_data == NULL) {
1858 /*
1859 * We have already been added to the arc eviction list;
1860 * recommend eviction.
1861 */
1862 ASSERT3P(hdr, ==, &arc_eviction_hdr);
1863 mutex_exit(&buf->b_evict_lock);
1864 return (B_TRUE);
1865 }
1866
1867 if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
1868 evict_needed = B_TRUE;
1869
1870 mutex_exit(&buf->b_evict_lock);
1871 return (evict_needed);
1872}
1873
1874/*
1805 * Evict buffers from list until we've removed the specified number of
1806 * bytes. Move the removed buffers to the appropriate evict state.
1807 * If the recycle flag is set, then attempt to "recycle" a buffer:
1808 * - look for a buffer to evict that is `bytes' long.
1809 * - return the data block from this buffer rather than freeing it.
1810 * This flag is used by callers that are trying to make space for a
1811 * new buffer in a full arc cache.
1812 *

--- 1069 unchanged lines hidden (view full) ---

2882 */
2883 arc_access(hdr, hash_lock);
2884 }
2885
2886 /* create copies of the data buffer for the callers */
2887 abuf = buf;
2888 for (acb = callback_list; acb; acb = acb->acb_next) {
2889 if (acb->acb_done) {
1875 * Evict buffers from list until we've removed the specified number of
1876 * bytes. Move the removed buffers to the appropriate evict state.
1877 * If the recycle flag is set, then attempt to "recycle" a buffer:
1878 * - look for a buffer to evict that is `bytes' long.
1879 * - return the data block from this buffer rather than freeing it.
1880 * This flag is used by callers that are trying to make space for a
1881 * new buffer in a full arc cache.
1882 *

--- 1069 unchanged lines hidden (view full) ---

2952 */
2953 arc_access(hdr, hash_lock);
2954 }
2955
2956 /* create copies of the data buffer for the callers */
2957 abuf = buf;
2958 for (acb = callback_list; acb; acb = acb->acb_next) {
2959 if (acb->acb_done) {
2890 if (abuf == NULL)
2960 if (abuf == NULL) {
2961 ARCSTAT_BUMP(arcstat_duplicate_reads);
2891 abuf = arc_buf_clone(buf);
2962 abuf = arc_buf_clone(buf);
2963 }
2892 acb->acb_buf = abuf;
2893 abuf = NULL;
2894 }
2895 }
2896 hdr->b_acb = NULL;
2897 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2898 ASSERT(!HDR_BUF_AVAILABLE(hdr));
2899 if (abuf == buf) {

--- 529 unchanged lines hidden (view full) ---

3429
3430 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3431 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3432 if (refcount_is_zero(&hdr->b_refcnt)) {
3433 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3434 ASSERT3U(*size, >=, hdr->b_size);
3435 atomic_add_64(size, -hdr->b_size);
3436 }
2964 acb->acb_buf = abuf;
2965 abuf = NULL;
2966 }
2967 }
2968 hdr->b_acb = NULL;
2969 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2970 ASSERT(!HDR_BUF_AVAILABLE(hdr));
2971 if (abuf == buf) {

--- 529 unchanged lines hidden (view full) ---

3501
3502 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3503 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3504 if (refcount_is_zero(&hdr->b_refcnt)) {
3505 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3506 ASSERT3U(*size, >=, hdr->b_size);
3507 atomic_add_64(size, -hdr->b_size);
3508 }
3509
3510 /*
3511 * We're releasing a duplicate user data buffer, update
3512 * our statistics accordingly.
3513 */
3514 if (hdr->b_type == ARC_BUFC_DATA) {
3515 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
3516 ARCSTAT_INCR(arcstat_duplicate_buffers_size,
3517 -hdr->b_size);
3518 }
3437 hdr->b_datacnt -= 1;
3438 arc_cksum_verify(buf);
3439#ifdef illumos
3440 arc_buf_unwatch(buf);
3441#endif /* illumos */
3442
3443 mutex_exit(hash_lock);
3444

--- 1670 unchanged lines hidden --- 		3519 hdr->b_datacnt -= 1;
3520 arc_cksum_verify(buf);
3521#ifdef illumos
3522 arc_buf_unwatch(buf);
3523#endif /* illumos */
3524
3525 mutex_exit(hash_lock);
3526

--- 1670 unchanged lines hidden ---