arc.c (241773) | arc.c (242845) |
---|---|
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE --- 177 unchanged lines hidden (view full) --- 186 * These tunables are for performance analysis. 187 */ 188uint64_t zfs_arc_max; 189uint64_t zfs_arc_min; 190uint64_t zfs_arc_meta_limit = 0; 191int zfs_arc_grow_retry = 0; 192int zfs_arc_shrink_shift = 0; 193int zfs_arc_p_min_shift = 0; | 1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE --- 177 unchanged lines hidden (view full) --- 186 * These tunables are for performance analysis. 187 */ 188uint64_t zfs_arc_max; 189uint64_t zfs_arc_min; 190uint64_t zfs_arc_meta_limit = 0; 191int zfs_arc_grow_retry = 0; 192int zfs_arc_shrink_shift = 0; 193int zfs_arc_p_min_shift = 0; |
194int zfs_disable_dup_eviction = 0; |
|
194 195TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max); 196TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min); 197TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit); 198SYSCTL_DECL(_vfs_zfs); 199SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_max, CTLFLAG_RDTUN, &zfs_arc_max, 0, 200 "Maximum ARC size"); 201SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_RDTUN, &zfs_arc_min, 0, --- 114 unchanged lines hidden (view full) --- 316 kstat_named_t arcstat_l2_evict_lock_retry; 317 kstat_named_t arcstat_l2_evict_reading; 318 kstat_named_t arcstat_l2_free_on_write; 319 kstat_named_t arcstat_l2_abort_lowmem; 320 kstat_named_t arcstat_l2_cksum_bad; 321 kstat_named_t arcstat_l2_io_error; 322 kstat_named_t arcstat_l2_size; 323 kstat_named_t arcstat_l2_hdr_size; | 195 196TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max); 197TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min); 198TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit); 199SYSCTL_DECL(_vfs_zfs); 200SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_max, CTLFLAG_RDTUN, &zfs_arc_max, 0, 201 "Maximum ARC size"); 202SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_RDTUN, &zfs_arc_min, 0, --- 114 unchanged lines hidden (view full) --- 317 kstat_named_t arcstat_l2_evict_lock_retry; 318 kstat_named_t arcstat_l2_evict_reading; 319 kstat_named_t arcstat_l2_free_on_write; 320 kstat_named_t arcstat_l2_abort_lowmem; 321 kstat_named_t arcstat_l2_cksum_bad; 322 kstat_named_t arcstat_l2_io_error; 323 kstat_named_t arcstat_l2_size; 324 kstat_named_t arcstat_l2_hdr_size; |
324 kstat_named_t arcstat_memory_throttle_count; | |
325 kstat_named_t arcstat_l2_write_trylock_fail; 326 kstat_named_t arcstat_l2_write_passed_headroom; 327 kstat_named_t arcstat_l2_write_spa_mismatch; 328 kstat_named_t arcstat_l2_write_in_l2; 329 kstat_named_t arcstat_l2_write_hdr_io_in_progress; 330 kstat_named_t arcstat_l2_write_not_cacheable; 331 kstat_named_t arcstat_l2_write_full; 332 kstat_named_t arcstat_l2_write_buffer_iter; 333 kstat_named_t arcstat_l2_write_pios; 334 kstat_named_t arcstat_l2_write_buffer_bytes_scanned; 335 kstat_named_t arcstat_l2_write_buffer_list_iter; 336 kstat_named_t arcstat_l2_write_buffer_list_null_iter; | 325 kstat_named_t arcstat_l2_write_trylock_fail; 326 kstat_named_t arcstat_l2_write_passed_headroom; 327 kstat_named_t arcstat_l2_write_spa_mismatch; 328 kstat_named_t arcstat_l2_write_in_l2; 329 kstat_named_t arcstat_l2_write_hdr_io_in_progress; 330 kstat_named_t arcstat_l2_write_not_cacheable; 331 kstat_named_t arcstat_l2_write_full; 332 kstat_named_t arcstat_l2_write_buffer_iter; 333 kstat_named_t arcstat_l2_write_pios; 334 kstat_named_t arcstat_l2_write_buffer_bytes_scanned; 335 kstat_named_t arcstat_l2_write_buffer_list_iter; 336 kstat_named_t arcstat_l2_write_buffer_list_null_iter; |
337 kstat_named_t arcstat_memory_throttle_count; 338 kstat_named_t arcstat_duplicate_buffers; 339 kstat_named_t arcstat_duplicate_buffers_size; 340 kstat_named_t arcstat_duplicate_reads; |
|
337} arc_stats_t; 338 339static arc_stats_t arc_stats = { 340 { "hits", KSTAT_DATA_UINT64 }, 341 { "misses", KSTAT_DATA_UINT64 }, 342 { "demand_data_hits", KSTAT_DATA_UINT64 }, 343 { "demand_data_misses", KSTAT_DATA_UINT64 }, 344 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, --- 41 unchanged lines hidden (view full) --- 386 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 387 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 388 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 389 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 390 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 391 { "l2_io_error", KSTAT_DATA_UINT64 }, 392 { "l2_size", KSTAT_DATA_UINT64 }, 393 { "l2_hdr_size", KSTAT_DATA_UINT64 }, | 341} arc_stats_t; 342 343static arc_stats_t arc_stats = { 344 { "hits", KSTAT_DATA_UINT64 }, 345 { "misses", KSTAT_DATA_UINT64 }, 346 { "demand_data_hits", KSTAT_DATA_UINT64 }, 347 { "demand_data_misses", KSTAT_DATA_UINT64 }, 348 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, --- 41 unchanged lines hidden (view full) --- 390 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 391 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 392 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 393 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 394 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 395 { "l2_io_error", KSTAT_DATA_UINT64 }, 396 { "l2_size", KSTAT_DATA_UINT64 }, 397 { "l2_hdr_size", KSTAT_DATA_UINT64 }, |
394 { "memory_throttle_count", KSTAT_DATA_UINT64 }, | |
395 { "l2_write_trylock_fail", KSTAT_DATA_UINT64 }, 396 { "l2_write_passed_headroom", KSTAT_DATA_UINT64 }, 397 { "l2_write_spa_mismatch", KSTAT_DATA_UINT64 }, 398 { "l2_write_in_l2", KSTAT_DATA_UINT64 }, 399 { "l2_write_io_in_progress", KSTAT_DATA_UINT64 }, 400 { "l2_write_not_cacheable", KSTAT_DATA_UINT64 }, 401 { "l2_write_full", KSTAT_DATA_UINT64 }, 402 { "l2_write_buffer_iter", KSTAT_DATA_UINT64 }, 403 { "l2_write_pios", KSTAT_DATA_UINT64 }, 404 { "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 }, 405 { "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 }, | 398 { "l2_write_trylock_fail", KSTAT_DATA_UINT64 }, 399 { "l2_write_passed_headroom", KSTAT_DATA_UINT64 }, 400 { "l2_write_spa_mismatch", KSTAT_DATA_UINT64 }, 401 { "l2_write_in_l2", KSTAT_DATA_UINT64 }, 402 { "l2_write_io_in_progress", KSTAT_DATA_UINT64 }, 403 { "l2_write_not_cacheable", KSTAT_DATA_UINT64 }, 404 { "l2_write_full", KSTAT_DATA_UINT64 }, 405 { "l2_write_buffer_iter", KSTAT_DATA_UINT64 }, 406 { "l2_write_pios", KSTAT_DATA_UINT64 }, 407 { "l2_write_buffer_bytes_scanned", KSTAT_DATA_UINT64 }, 408 { "l2_write_buffer_list_iter", KSTAT_DATA_UINT64 }, |
406 { "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 } | 409 { "l2_write_buffer_list_null_iter", KSTAT_DATA_UINT64 }, 410 { "memory_throttle_count", KSTAT_DATA_UINT64 }, 411 { "duplicate_buffers", KSTAT_DATA_UINT64 }, 412 { "duplicate_buffers_size", KSTAT_DATA_UINT64 }, 413 { "duplicate_reads", KSTAT_DATA_UINT64 } |
407}; 408 409#define ARCSTAT(stat) (arc_stats.stat.value.ui64) 410 411#define ARCSTAT_INCR(stat, val) \ 412 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 413 414#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) --- 1098 unchanged lines hidden (view full) --- 1513 buf->b_hdr = hdr; 1514 buf->b_data = NULL; 1515 buf->b_efunc = NULL; 1516 buf->b_private = NULL; 1517 buf->b_next = hdr->b_buf; 1518 hdr->b_buf = buf; 1519 arc_get_data_buf(buf); 1520 bcopy(from->b_data, buf->b_data, size); | 414}; 415 416#define ARCSTAT(stat) (arc_stats.stat.value.ui64) 417 418#define ARCSTAT_INCR(stat, val) \ 419 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 420 421#define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) --- 1098 unchanged lines hidden (view full) --- 1520 buf->b_hdr = hdr; 1521 buf->b_data = NULL; 1522 buf->b_efunc = NULL; 1523 buf->b_private = NULL; 1524 buf->b_next = hdr->b_buf; 1525 hdr->b_buf = buf; 1526 arc_get_data_buf(buf); 1527 bcopy(from->b_data, buf->b_data, size); |
1528 1529 /* 1530 * This buffer already exists in the arc so create a duplicate 1531 * copy for the caller. If the buffer is associated with user data 1532 * then track the size and number of duplicates. These stats will be 1533 * updated as duplicate buffers are created and destroyed. 1534 */ 1535 if (hdr->b_type == ARC_BUFC_DATA) { 1536 ARCSTAT_BUMP(arcstat_duplicate_buffers); 1537 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size); 1538 } |
|
1521 hdr->b_datacnt += 1; 1522 return (buf); 1523} 1524 1525void 1526arc_buf_add_ref(arc_buf_t *buf, void* tag) 1527{ 1528 arc_buf_hdr_t *hdr; --- 84 unchanged lines hidden (view full) --- 1613 ASSERT(state != arc_anon); 1614 1615 ASSERT3U(*cnt, >=, size); 1616 atomic_add_64(cnt, -size); 1617 } 1618 ASSERT3U(state->arcs_size, >=, size); 1619 atomic_add_64(&state->arcs_size, -size); 1620 buf->b_data = NULL; | 1539 hdr->b_datacnt += 1; 1540 return (buf); 1541} 1542 1543void 1544arc_buf_add_ref(arc_buf_t *buf, void* tag) 1545{ 1546 arc_buf_hdr_t *hdr; --- 84 unchanged lines hidden (view full) --- 1631 ASSERT(state != arc_anon); 1632 1633 ASSERT3U(*cnt, >=, size); 1634 atomic_add_64(cnt, -size); 1635 } 1636 ASSERT3U(state->arcs_size, >=, size); 1637 atomic_add_64(&state->arcs_size, -size); 1638 buf->b_data = NULL; |
1639 1640 /* 1641 * If we're destroying a duplicate buffer make sure 1642 * that the appropriate statistics are updated. 1643 */ 1644 if (buf->b_hdr->b_datacnt > 1 && 1645 buf->b_hdr->b_type == ARC_BUFC_DATA) { 1646 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers); 1647 ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size); 1648 } |
|
1621 ASSERT(buf->b_hdr->b_datacnt > 0); 1622 buf->b_hdr->b_datacnt -= 1; 1623 } 1624 1625 /* only remove the buf if requested */ 1626 if (!all) 1627 return; 1628 --- 168 unchanged lines hidden (view full) --- 1797 1798int 1799arc_buf_size(arc_buf_t *buf) 1800{ 1801 return (buf->b_hdr->b_size); 1802} 1803 1804/* | 1649 ASSERT(buf->b_hdr->b_datacnt > 0); 1650 buf->b_hdr->b_datacnt -= 1; 1651 } 1652 1653 /* only remove the buf if requested */ 1654 if (!all) 1655 return; 1656 --- 168 unchanged lines hidden (view full) --- 1825 1826int 1827arc_buf_size(arc_buf_t *buf) 1828{ 1829 return (buf->b_hdr->b_size); 1830} 1831 1832/* |
1833 * Called from the DMU to determine if the current buffer should be 1834 * evicted. In order to ensure proper locking, the eviction must be initiated 1835 * from the DMU. Return true if the buffer is associated with user data and 1836 * duplicate buffers still exist. 1837 */ 1838boolean_t 1839arc_buf_eviction_needed(arc_buf_t *buf) 1840{ 1841 arc_buf_hdr_t *hdr; 1842 boolean_t evict_needed = B_FALSE; 1843 1844 if (zfs_disable_dup_eviction) 1845 return (B_FALSE); 1846 1847 mutex_enter(&buf->b_evict_lock); 1848 hdr = buf->b_hdr; 1849 if (hdr == NULL) { 1850 /* 1851 * We are in arc_do_user_evicts(); let that function 1852 * perform the eviction. 1853 */ 1854 ASSERT(buf->b_data == NULL); 1855 mutex_exit(&buf->b_evict_lock); 1856 return (B_FALSE); 1857 } else if (buf->b_data == NULL) { 1858 /* 1859 * We have already been added to the arc eviction list; 1860 * recommend eviction. 1861 */ 1862 ASSERT3P(hdr, ==, &arc_eviction_hdr); 1863 mutex_exit(&buf->b_evict_lock); 1864 return (B_TRUE); 1865 } 1866 1867 if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA) 1868 evict_needed = B_TRUE; 1869 1870 mutex_exit(&buf->b_evict_lock); 1871 return (evict_needed); 1872} 1873 1874/* |
|
1805 * Evict buffers from list until we've removed the specified number of 1806 * bytes. Move the removed buffers to the appropriate evict state. 1807 * If the recycle flag is set, then attempt to "recycle" a buffer: 1808 * - look for a buffer to evict that is `bytes' long. 1809 * - return the data block from this buffer rather than freeing it. 1810 * This flag is used by callers that are trying to make space for a 1811 * new buffer in a full arc cache. 1812 * --- 1069 unchanged lines hidden (view full) --- 2882 */ 2883 arc_access(hdr, hash_lock); 2884 } 2885 2886 /* create copies of the data buffer for the callers */ 2887 abuf = buf; 2888 for (acb = callback_list; acb; acb = acb->acb_next) { 2889 if (acb->acb_done) { | 1875 * Evict buffers from list until we've removed the specified number of 1876 * bytes. Move the removed buffers to the appropriate evict state. 1877 * If the recycle flag is set, then attempt to "recycle" a buffer: 1878 * - look for a buffer to evict that is `bytes' long. 1879 * - return the data block from this buffer rather than freeing it. 1880 * This flag is used by callers that are trying to make space for a 1881 * new buffer in a full arc cache. 1882 * --- 1069 unchanged lines hidden (view full) --- 2952 */ 2953 arc_access(hdr, hash_lock); 2954 } 2955 2956 /* create copies of the data buffer for the callers */ 2957 abuf = buf; 2958 for (acb = callback_list; acb; acb = acb->acb_next) { 2959 if (acb->acb_done) { |
2890 if (abuf == NULL) | 2960 if (abuf == NULL) { 2961 ARCSTAT_BUMP(arcstat_duplicate_reads); |
2891 abuf = arc_buf_clone(buf); | 2962 abuf = arc_buf_clone(buf); |
2963 } |
|
2892 acb->acb_buf = abuf; 2893 abuf = NULL; 2894 } 2895 } 2896 hdr->b_acb = NULL; 2897 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2898 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2899 if (abuf == buf) { --- 529 unchanged lines hidden (view full) --- 3429 3430 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 3431 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3432 if (refcount_is_zero(&hdr->b_refcnt)) { 3433 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 3434 ASSERT3U(*size, >=, hdr->b_size); 3435 atomic_add_64(size, -hdr->b_size); 3436 } | 2964 acb->acb_buf = abuf; 2965 abuf = NULL; 2966 } 2967 } 2968 hdr->b_acb = NULL; 2969 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2970 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2971 if (abuf == buf) { --- 529 unchanged lines hidden (view full) --- 3501 3502 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 3503 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3504 if (refcount_is_zero(&hdr->b_refcnt)) { 3505 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 3506 ASSERT3U(*size, >=, hdr->b_size); 3507 atomic_add_64(size, -hdr->b_size); 3508 } |
3509 3510 /* 3511 * We're releasing a duplicate user data buffer, update 3512 * our statistics accordingly. 3513 */ 3514 if (hdr->b_type == ARC_BUFC_DATA) { 3515 ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers); 3516 ARCSTAT_INCR(arcstat_duplicate_buffers_size, 3517 -hdr->b_size); 3518 } |
|
3437 hdr->b_datacnt -= 1; 3438 arc_cksum_verify(buf); 3439#ifdef illumos 3440 arc_buf_unwatch(buf); 3441#endif /* illumos */ 3442 3443 mutex_exit(hash_lock); 3444 --- 1670 unchanged lines hidden --- | 3519 hdr->b_datacnt -= 1; 3520 arc_cksum_verify(buf); 3521#ifdef illumos 3522 arc_buf_unwatch(buf); 3523#endif /* illumos */ 3524 3525 mutex_exit(hash_lock); 3526 --- 1670 unchanged lines hidden --- |