Deleted Added
sdiff udiff text old ( 209275 ) new ( 209962 )
full compact
arc.c (209275) arc.c (209962)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 448 unchanged lines hidden (view full) ---

457#define arc_size ARCSTAT(arcstat_size) /* actual total arc size */
458#define arc_p ARCSTAT(arcstat_p) /* target size of MRU */
459#define arc_c ARCSTAT(arcstat_c) /* target size of cache */
460#define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */
461#define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */
462
463static int arc_no_grow; /* Don't try to grow cache size */
464static uint64_t arc_tempreserve;
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 448 unchanged lines hidden (view full) ---

457#define arc_size ARCSTAT(arcstat_size) /* actual total arc size */
458#define arc_p ARCSTAT(arcstat_p) /* target size of MRU */
459#define arc_c ARCSTAT(arcstat_c) /* target size of cache */
460#define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */
461#define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */
462
463static int arc_no_grow; /* Don't try to grow cache size */
464static uint64_t arc_tempreserve;
465static uint64_t arc_loaned_bytes;
465static uint64_t arc_meta_used;
466static uint64_t arc_meta_limit;
467static uint64_t arc_meta_max = 0;
468SYSCTL_QUAD(_vfs_zfs, OID_AUTO, arc_meta_used, CTLFLAG_RDTUN,
469 &arc_meta_used, 0, "ARC metadata used");
470SYSCTL_QUAD(_vfs_zfs, OID_AUTO, arc_meta_limit, CTLFLAG_RDTUN,
471 &arc_meta_limit, 0, "ARC metadata limit");
472

--- 33 unchanged lines hidden (view full) ---

506 uint32_t b_datacnt;
507
508 arc_callback_t *b_acb;
509 kcondvar_t b_cv;
510
511 /* immutable */
512 arc_buf_contents_t b_type;
513 uint64_t b_size;
466static uint64_t arc_meta_used;
467static uint64_t arc_meta_limit;
468static uint64_t arc_meta_max = 0;
469SYSCTL_QUAD(_vfs_zfs, OID_AUTO, arc_meta_used, CTLFLAG_RDTUN,
470 &arc_meta_used, 0, "ARC metadata used");
471SYSCTL_QUAD(_vfs_zfs, OID_AUTO, arc_meta_limit, CTLFLAG_RDTUN,
472 &arc_meta_limit, 0, "ARC metadata limit");
473

--- 33 unchanged lines hidden (view full) ---

507 uint32_t b_datacnt;
508
509 arc_callback_t *b_acb;
510 kcondvar_t b_cv;
511
512 /* immutable */
513 arc_buf_contents_t b_type;
514 uint64_t b_size;
514 spa_t *b_spa;
515 uint64_t b_spa;
515
516 /* protected by arc state mutex */
517 arc_state_t *b_state;
518 list_node_t b_arc_node;
519
520 /* updated atomically */
521 clock_t b_arc_access;
522

--- 5 unchanged lines hidden (view full) ---

528};
529
530static arc_buf_t *arc_eviction_list;
531static kmutex_t arc_eviction_mtx;
532static arc_buf_hdr_t arc_eviction_hdr;
533static void arc_get_data_buf(arc_buf_t *buf);
534static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
535static int arc_evict_needed(arc_buf_contents_t type);
516
517 /* protected by arc state mutex */
518 arc_state_t *b_state;
519 list_node_t b_arc_node;
520
521 /* updated atomically */
522 clock_t b_arc_access;
523

--- 5 unchanged lines hidden (view full) ---

529};
530
531static arc_buf_t *arc_eviction_list;
532static kmutex_t arc_eviction_mtx;
533static arc_buf_hdr_t arc_eviction_hdr;
534static void arc_get_data_buf(arc_buf_t *buf);
535static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
536static int arc_evict_needed(arc_buf_contents_t type);
536static void arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes);
537static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
537
538
538static boolean_t l2arc_write_eligible(spa_t *spa, arc_buf_hdr_t *ab);
539static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
539
540#define GHOST_STATE(state) \
541 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
542 (state) == arc_l2c_only)
543
544/*
545 * Private ARC flags. These flags are private ARC only flags that will show up
546 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can

--- 209 unchanged lines hidden (view full) ---

756static kcondvar_t l2arc_feed_thr_cv;
757static uint8_t l2arc_thread_exit;
758
759static void l2arc_read_done(zio_t *zio);
760static void l2arc_hdr_stat_add(void);
761static void l2arc_hdr_stat_remove(void);
762
763static uint64_t
540
541#define GHOST_STATE(state) \
542 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
543 (state) == arc_l2c_only)
544
545/*
546 * Private ARC flags. These flags are private ARC only flags that will show up
547 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can

--- 209 unchanged lines hidden (view full) ---

757static kcondvar_t l2arc_feed_thr_cv;
758static uint8_t l2arc_thread_exit;
759
760static void l2arc_read_done(zio_t *zio);
761static void l2arc_hdr_stat_add(void);
762static void l2arc_hdr_stat_remove(void);
763
764static uint64_t
764buf_hash(spa_t *spa, const dva_t *dva, uint64_t birth)
765buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
765{
766{
766 uintptr_t spav = (uintptr_t)spa;
767 uint8_t *vdva = (uint8_t *)dva;
768 uint64_t crc = -1ULL;
769 int i;
770
771 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
772
773 for (i = 0; i < sizeof (dva_t); i++)
774 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
775
767 uint8_t *vdva = (uint8_t *)dva;
768 uint64_t crc = -1ULL;
769 int i;
770
771 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
772
773 for (i = 0; i < sizeof (dva_t); i++)
774 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
775
776 crc ^= (spav>>8) ^ birth;
776 crc ^= (spa>>8) ^ birth;
777
778 return (crc);
779}
780
781#define BUF_EMPTY(buf) \
782 ((buf)->b_dva.dva_word[0] == 0 && \
783 (buf)->b_dva.dva_word[1] == 0 && \
784 (buf)->b_birth == 0)
785
786#define BUF_EQUAL(spa, dva, birth, buf) \
787 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
788 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
789 ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
790
791static arc_buf_hdr_t *
777
778 return (crc);
779}
780
781#define BUF_EMPTY(buf) \
782 ((buf)->b_dva.dva_word[0] == 0 && \
783 (buf)->b_dva.dva_word[1] == 0 && \
784 (buf)->b_birth == 0)
785
786#define BUF_EQUAL(spa, dva, birth, buf) \
787 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
788 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
789 ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
790
791static arc_buf_hdr_t *
792buf_hash_find(spa_t *spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
792buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
793{
794 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
795 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
796 arc_buf_hdr_t *buf;
797
798 mutex_enter(hash_lock);
799 for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
800 buf = buf->b_hash_next) {

--- 539 unchanged lines hidden (view full) ---

1340 arc_buf_hdr_t *hdr;
1341 arc_buf_t *buf;
1342
1343 ASSERT3U(size, >, 0);
1344 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1345 ASSERT(BUF_EMPTY(hdr));
1346 hdr->b_size = size;
1347 hdr->b_type = type;
793{
794 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
795 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
796 arc_buf_hdr_t *buf;
797
798 mutex_enter(hash_lock);
799 for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
800 buf = buf->b_hash_next) {

--- 539 unchanged lines hidden (view full) ---

1340 arc_buf_hdr_t *hdr;
1341 arc_buf_t *buf;
1342
1343 ASSERT3U(size, >, 0);
1344 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1345 ASSERT(BUF_EMPTY(hdr));
1346 hdr->b_size = size;
1347 hdr->b_type = type;
1348 hdr->b_spa = spa;
1348 hdr->b_spa = spa_guid(spa);
1349 hdr->b_state = arc_anon;
1350 hdr->b_arc_access = 0;
1351 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1352 buf->b_hdr = hdr;
1353 buf->b_data = NULL;
1354 buf->b_efunc = NULL;
1355 buf->b_private = NULL;
1356 buf->b_next = NULL;
1357 hdr->b_buf = buf;
1358 arc_get_data_buf(buf);
1359 hdr->b_datacnt = 1;
1360 hdr->b_flags = 0;
1361 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1362 (void) refcount_add(&hdr->b_refcnt, tag);
1363
1364 return (buf);
1365}
1366
1349 hdr->b_state = arc_anon;
1350 hdr->b_arc_access = 0;
1351 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1352 buf->b_hdr = hdr;
1353 buf->b_data = NULL;
1354 buf->b_efunc = NULL;
1355 buf->b_private = NULL;
1356 buf->b_next = NULL;
1357 hdr->b_buf = buf;
1358 arc_get_data_buf(buf);
1359 hdr->b_datacnt = 1;
1360 hdr->b_flags = 0;
1361 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1362 (void) refcount_add(&hdr->b_refcnt, tag);
1363
1364 return (buf);
1365}
1366
1367static char *arc_onloan_tag = "onloan";
1368
1369/*
1370 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1371 * flight data by arc_tempreserve_space() until they are "returned". Loaned
1372 * buffers must be returned to the arc before they can be used by the DMU or
1373 * freed.
1374 */
1375arc_buf_t *
1376arc_loan_buf(spa_t *spa, int size)
1377{
1378 arc_buf_t *buf;
1379
1380 buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1381
1382 atomic_add_64(&arc_loaned_bytes, size);
1383 return (buf);
1384}
1385
1386/*
1387 * Return a loaned arc buffer to the arc.
1388 */
1389void
1390arc_return_buf(arc_buf_t *buf, void *tag)
1391{
1392 arc_buf_hdr_t *hdr = buf->b_hdr;
1393
1394 ASSERT(hdr->b_state == arc_anon);
1395 ASSERT(buf->b_data != NULL);
1396 VERIFY(refcount_remove(&hdr->b_refcnt, arc_onloan_tag) == 0);
1397 VERIFY(refcount_add(&hdr->b_refcnt, tag) == 1);
1398
1399 atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1400}
1401
1367static arc_buf_t *
1368arc_buf_clone(arc_buf_t *from)
1369{
1370 arc_buf_t *buf;
1371 arc_buf_hdr_t *hdr = from->b_hdr;
1372 uint64_t size = hdr->b_size;
1373
1374 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);

--- 281 unchanged lines hidden (view full) ---

1656 * This flag is used by callers that are trying to make space for a
1657 * new buffer in a full arc cache.
1658 *
1659 * This function makes a "best effort". It skips over any buffers
1660 * it can't get a hash_lock on, and so may not catch all candidates.
1661 * It may also return without evicting as much space as requested.
1662 */
1663static void *
1402static arc_buf_t *
1403arc_buf_clone(arc_buf_t *from)
1404{
1405 arc_buf_t *buf;
1406 arc_buf_hdr_t *hdr = from->b_hdr;
1407 uint64_t size = hdr->b_size;
1408
1409 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);

--- 281 unchanged lines hidden (view full) ---

1691 * This flag is used by callers that are trying to make space for a
1692 * new buffer in a full arc cache.
1693 *
1694 * This function makes a "best effort". It skips over any buffers
1695 * it can't get a hash_lock on, and so may not catch all candidates.
1696 * It may also return without evicting as much space as requested.
1697 */
1698static void *
1664arc_evict(arc_state_t *state, spa_t *spa, int64_t bytes, boolean_t recycle,
1699arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1665 arc_buf_contents_t type)
1666{
1667 arc_state_t *evicted_state;
1668 uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1669 int64_t bytes_remaining;
1670 arc_buf_hdr_t *ab, *ab_prev = NULL;
1671 list_t *evicted_list, *list, *evicted_list_start, *list_start;
1672 kmutex_t *lock, *evicted_lock;

--- 152 unchanged lines hidden (view full) ---

1825 if (arc_no_grow &&
1826 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1827 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1828 arc_mru_ghost->arcs_size - arc_c;
1829
1830 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1831 int64_t todelete =
1832 MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1700 arc_buf_contents_t type)
1701{
1702 arc_state_t *evicted_state;
1703 uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1704 int64_t bytes_remaining;
1705 arc_buf_hdr_t *ab, *ab_prev = NULL;
1706 list_t *evicted_list, *list, *evicted_list_start, *list_start;
1707 kmutex_t *lock, *evicted_lock;

--- 152 unchanged lines hidden (view full) ---

1860 if (arc_no_grow &&
1861 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1862 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1863 arc_mru_ghost->arcs_size - arc_c;
1864
1865 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1866 int64_t todelete =
1867 MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1833 arc_evict_ghost(arc_mru_ghost, NULL, todelete);
1868 arc_evict_ghost(arc_mru_ghost, 0, todelete);
1834 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1835 int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1836 arc_mru_ghost->arcs_size +
1837 arc_mfu_ghost->arcs_size - arc_c);
1869 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1870 int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1871 arc_mru_ghost->arcs_size +
1872 arc_mfu_ghost->arcs_size - arc_c);
1838 arc_evict_ghost(arc_mfu_ghost, NULL, todelete);
1873 arc_evict_ghost(arc_mfu_ghost, 0, todelete);
1839 }
1840 }
1841 if (stolen)
1842 ARCSTAT_BUMP(arcstat_stolen);
1843
1844 return (stolen);
1845}
1846
1847/*
1848 * Remove buffers from list until we've removed the specified number of
1849 * bytes. Destroy the buffers that are removed.
1850 */
1851static void
1874 }
1875 }
1876 if (stolen)
1877 ARCSTAT_BUMP(arcstat_stolen);
1878
1879 return (stolen);
1880}
1881
1882/*
1883 * Remove buffers from list until we've removed the specified number of
1884 * bytes. Destroy the buffers that are removed.
1885 */
1886static void
1852arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes)
1887arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1853{
1854 arc_buf_hdr_t *ab, *ab_prev;
1855 list_t *list, *list_start;
1856 kmutex_t *hash_lock, *lock;
1857 uint64_t bytes_deleted = 0;
1858 uint64_t bufs_skipped = 0;
1859 static int evict_offset;
1860 int list_count, idx = evict_offset;

--- 89 unchanged lines hidden (view full) ---

1950 */
1951
1952 adjustment = MIN((int64_t)(arc_size - arc_c),
1953 (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
1954 arc_p));
1955
1956 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
1957 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
1888{
1889 arc_buf_hdr_t *ab, *ab_prev;
1890 list_t *list, *list_start;
1891 kmutex_t *hash_lock, *lock;
1892 uint64_t bytes_deleted = 0;
1893 uint64_t bufs_skipped = 0;
1894 static int evict_offset;
1895 int list_count, idx = evict_offset;

--- 89 unchanged lines hidden (view full) ---

1985 */
1986
1987 adjustment = MIN((int64_t)(arc_size - arc_c),
1988 (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
1989 arc_p));
1990
1991 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
1992 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
1958 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
1993 (void) arc_evict(arc_mru, 0, delta, FALSE, ARC_BUFC_DATA);
1959 adjustment -= delta;
1960 }
1961
1962 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1963 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
1994 adjustment -= delta;
1995 }
1996
1997 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1998 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
1964 (void) arc_evict(arc_mru, NULL, delta, FALSE,
1999 (void) arc_evict(arc_mru, 0, delta, FALSE,
1965 ARC_BUFC_METADATA);
1966 }
1967
1968 /*
1969 * Adjust MFU size
1970 */
1971
1972 adjustment = arc_size - arc_c;
1973
1974 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
1975 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
2000 ARC_BUFC_METADATA);
2001 }
2002
2003 /*
2004 * Adjust MFU size
2005 */
2006
2007 adjustment = arc_size - arc_c;
2008
2009 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
2010 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
1976 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
2011 (void) arc_evict(arc_mfu, 0, delta, FALSE, ARC_BUFC_DATA);
1977 adjustment -= delta;
1978 }
1979
1980 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1981 int64_t delta = MIN(adjustment,
1982 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
2012 adjustment -= delta;
2013 }
2014
2015 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2016 int64_t delta = MIN(adjustment,
2017 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
1983 (void) arc_evict(arc_mfu, NULL, delta, FALSE,
2018 (void) arc_evict(arc_mfu, 0, delta, FALSE,
1984 ARC_BUFC_METADATA);
1985 }
1986
1987 /*
1988 * Adjust ghost lists
1989 */
1990
1991 adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
1992
1993 if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
1994 delta = MIN(arc_mru_ghost->arcs_size, adjustment);
2019 ARC_BUFC_METADATA);
2020 }
2021
2022 /*
2023 * Adjust ghost lists
2024 */
2025
2026 adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2027
2028 if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
2029 delta = MIN(arc_mru_ghost->arcs_size, adjustment);
1995 arc_evict_ghost(arc_mru_ghost, NULL, delta);
2030 arc_evict_ghost(arc_mru_ghost, 0, delta);
1996 }
1997
1998 adjustment =
1999 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2000
2001 if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2002 delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2031 }
2032
2033 adjustment =
2034 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2035
2036 if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2037 delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2003 arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2038 arc_evict_ghost(arc_mfu_ghost, 0, delta);
2004 }
2005}
2006
2007static void
2008arc_do_user_evicts(void)
2009{
2010 static arc_buf_t *tmp_arc_eviction_list;
2011

--- 27 unchanged lines hidden (view full) ---

2039
2040/*
2041 * Flush all *evictable* data from the cache for the given spa.
2042 * NOTE: this will not touch "active" (i.e. referenced) data.
2043 */
2044void
2045arc_flush(spa_t *spa)
2046{
2039 }
2040}
2041
2042static void
2043arc_do_user_evicts(void)
2044{
2045 static arc_buf_t *tmp_arc_eviction_list;
2046

--- 27 unchanged lines hidden (view full) ---

2074
2075/*
2076 * Flush all *evictable* data from the cache for the given spa.
2077 * NOTE: this will not touch "active" (i.e. referenced) data.
2078 */
2079void
2080arc_flush(spa_t *spa)
2081{
2082 uint64_t guid = 0;
2083
2084 if (spa)
2085 guid = spa_guid(spa);
2086
2047 while (arc_mru->arcs_lsize[ARC_BUFC_DATA]) {
2087 while (arc_mru->arcs_lsize[ARC_BUFC_DATA]) {
2048 (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_DATA);
2088 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2049 if (spa)
2050 break;
2051 }
2052 while (arc_mru->arcs_lsize[ARC_BUFC_METADATA]) {
2089 if (spa)
2090 break;
2091 }
2092 while (arc_mru->arcs_lsize[ARC_BUFC_METADATA]) {
2053 (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_METADATA);
2093 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2054 if (spa)
2055 break;
2056 }
2057 while (arc_mfu->arcs_lsize[ARC_BUFC_DATA]) {
2094 if (spa)
2095 break;
2096 }
2097 while (arc_mfu->arcs_lsize[ARC_BUFC_DATA]) {
2058 (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_DATA);
2098 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2059 if (spa)
2060 break;
2061 }
2062 while (arc_mfu->arcs_lsize[ARC_BUFC_METADATA]) {
2099 if (spa)
2100 break;
2101 }
2102 while (arc_mfu->arcs_lsize[ARC_BUFC_METADATA]) {
2063 (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_METADATA);
2103 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2064 if (spa)
2065 break;
2066 }
2067
2104 if (spa)
2105 break;
2106 }
2107
2068 arc_evict_ghost(arc_mru_ghost, spa, -1);
2069 arc_evict_ghost(arc_mfu_ghost, spa, -1);
2108 arc_evict_ghost(arc_mru_ghost, guid, -1);
2109 arc_evict_ghost(arc_mfu_ghost, guid, -1);
2070
2071 mutex_enter(&arc_reclaim_thr_lock);
2072 arc_do_user_evicts();
2073 mutex_exit(&arc_reclaim_thr_lock);
2074 ASSERT(spa || arc_eviction_list == NULL);
2075}
2076
2077void

--- 380 unchanged lines hidden (view full) ---

2458 state = (arc_mfu->arcs_lsize[type] >= size &&
2459 arc_p > mru_used) ? arc_mfu : arc_mru;
2460 } else {
2461 /* MFU cases */
2462 uint64_t mfu_space = arc_c - arc_p;
2463 state = (arc_mru->arcs_lsize[type] >= size &&
2464 mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2465 }
2110
2111 mutex_enter(&arc_reclaim_thr_lock);
2112 arc_do_user_evicts();
2113 mutex_exit(&arc_reclaim_thr_lock);
2114 ASSERT(spa || arc_eviction_list == NULL);
2115}
2116
2117void

--- 380 unchanged lines hidden (view full) ---

2498 state = (arc_mfu->arcs_lsize[type] >= size &&
2499 arc_p > mru_used) ? arc_mfu : arc_mru;
2500 } else {
2501 /* MFU cases */
2502 uint64_t mfu_space = arc_c - arc_p;
2503 state = (arc_mru->arcs_lsize[type] >= size &&
2504 mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2505 }
2466 if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2506 if ((buf->b_data = arc_evict(state, 0, size, TRUE, type)) == NULL) {
2467 if (type == ARC_BUFC_METADATA) {
2468 buf->b_data = zio_buf_alloc(size);
2469 arc_space_consume(size, ARC_SPACE_DATA);
2470 } else {
2471 ASSERT(type == ARC_BUFC_DATA);
2472 buf->b_data = zio_data_buf_alloc(size);
2473 ARCSTAT_INCR(arcstat_data_size, size);
2474 atomic_add_64(&arc_size, size);

--- 193 unchanged lines hidden (view full) ---

2668 /*
2669 * The hdr was inserted into hash-table and removed from lists
2670 * prior to starting I/O. We should find this header, since
2671 * it's in the hash table, and it should be legit since it's
2672 * not possible to evict it during the I/O. The only possible
2673 * reason for it not to be found is if we were freed during the
2674 * read.
2675 */
2507 if (type == ARC_BUFC_METADATA) {
2508 buf->b_data = zio_buf_alloc(size);
2509 arc_space_consume(size, ARC_SPACE_DATA);
2510 } else {
2511 ASSERT(type == ARC_BUFC_DATA);
2512 buf->b_data = zio_data_buf_alloc(size);
2513 ARCSTAT_INCR(arcstat_data_size, size);
2514 atomic_add_64(&arc_size, size);

--- 193 unchanged lines hidden (view full) ---

2708 /*
2709 * The hdr was inserted into hash-table and removed from lists
2710 * prior to starting I/O. We should find this header, since
2711 * it's in the hash table, and it should be legit since it's
2712 * not possible to evict it during the I/O. The only possible
2713 * reason for it not to be found is if we were freed during the
2714 * read.
2715 */
2676 found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth,
2716 found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2677 &hash_lock);
2678
2679 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2680 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2681 (found == hdr && HDR_L2_READING(hdr)));
2682
2683 hdr->b_flags &= ~ARC_L2_EVICTED;
2684 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))

--- 127 unchanged lines hidden (view full) ---

2812arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp,
2813 arc_done_func_t *done, void *private, int priority, int zio_flags,
2814 uint32_t *arc_flags, const zbookmark_t *zb)
2815{
2816 arc_buf_hdr_t *hdr;
2817 arc_buf_t *buf;
2818 kmutex_t *hash_lock;
2819 zio_t *rzio;
2717 &hash_lock);
2718
2719 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2720 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2721 (found == hdr && HDR_L2_READING(hdr)));
2722
2723 hdr->b_flags &= ~ARC_L2_EVICTED;
2724 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))

--- 127 unchanged lines hidden (view full) ---

2852arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp,
2853 arc_done_func_t *done, void *private, int priority, int zio_flags,
2854 uint32_t *arc_flags, const zbookmark_t *zb)
2855{
2856 arc_buf_hdr_t *hdr;
2857 arc_buf_t *buf;
2858 kmutex_t *hash_lock;
2859 zio_t *rzio;
2860 uint64_t guid = spa_guid(spa);
2820
2821top:
2861
2862top:
2822 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
2863 hdr = buf_hash_find(guid, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
2823 if (hdr && hdr->b_datacnt > 0) {
2824
2825 *arc_flags |= ARC_CACHED;
2826
2827 if (HDR_IO_IN_PROGRESS(hdr)) {
2828
2829 if (*arc_flags & ARC_WAIT) {
2830 cv_wait(&hdr->b_cv, hash_lock);

--- 6 unchanged lines hidden (view full) ---

2837 arc_callback_t *acb = NULL;
2838
2839 acb = kmem_zalloc(sizeof (arc_callback_t),
2840 KM_SLEEP);
2841 acb->acb_done = done;
2842 acb->acb_private = private;
2843 if (pio != NULL)
2844 acb->acb_zio_dummy = zio_null(pio,
2864 if (hdr && hdr->b_datacnt > 0) {
2865
2866 *arc_flags |= ARC_CACHED;
2867
2868 if (HDR_IO_IN_PROGRESS(hdr)) {
2869
2870 if (*arc_flags & ARC_WAIT) {
2871 cv_wait(&hdr->b_cv, hash_lock);

--- 6 unchanged lines hidden (view full) ---

2878 arc_callback_t *acb = NULL;
2879
2880 acb = kmem_zalloc(sizeof (arc_callback_t),
2881 KM_SLEEP);
2882 acb->acb_done = done;
2883 acb->acb_private = private;
2884 if (pio != NULL)
2885 acb->acb_zio_dummy = zio_null(pio,
2845 spa, NULL, NULL, zio_flags);
2886 spa, NULL, NULL, NULL, zio_flags);
2846
2847 ASSERT(acb->acb_done != NULL);
2848 acb->acb_next = hdr->b_acb;
2849 hdr->b_acb = acb;
2850 add_reference(hdr, hash_lock, private);
2851 mutex_exit(hash_lock);
2852 return (0);
2853 }

--- 225 unchanged lines hidden (view full) ---

3079 * The idea is that we don't want pool traversal filling up memory, but
3080 * if the ARC already has the data anyway, we shouldn't pay for the I/O.
3081 */
3082int
3083arc_tryread(spa_t *spa, blkptr_t *bp, void *data)
3084{
3085 arc_buf_hdr_t *hdr;
3086 kmutex_t *hash_mtx;
2887
2888 ASSERT(acb->acb_done != NULL);
2889 acb->acb_next = hdr->b_acb;
2890 hdr->b_acb = acb;
2891 add_reference(hdr, hash_lock, private);
2892 mutex_exit(hash_lock);
2893 return (0);
2894 }

--- 225 unchanged lines hidden (view full) ---

3120 * The idea is that we don't want pool traversal filling up memory, but
3121 * if the ARC already has the data anyway, we shouldn't pay for the I/O.
3122 */
3123int
3124arc_tryread(spa_t *spa, blkptr_t *bp, void *data)
3125{
3126 arc_buf_hdr_t *hdr;
3127 kmutex_t *hash_mtx;
3128 uint64_t guid = spa_guid(spa);
3087 int rc = 0;
3088
3129 int rc = 0;
3130
3089 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx);
3131 hdr = buf_hash_find(guid, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx);
3090
3091 if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) {
3092 arc_buf_t *buf = hdr->b_buf;
3093
3094 ASSERT(buf);
3095 while (buf->b_data == NULL) {
3096 buf = buf->b_next;
3097 ASSERT(buf);

--- 151 unchanged lines hidden (view full) ---

3249
3250 /*
3251 * Do we have more than one buf?
3252 */
3253 if (hdr->b_datacnt > 1) {
3254 arc_buf_hdr_t *nhdr;
3255 arc_buf_t **bufp;
3256 uint64_t blksz = hdr->b_size;
3132
3133 if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) {
3134 arc_buf_t *buf = hdr->b_buf;
3135
3136 ASSERT(buf);
3137 while (buf->b_data == NULL) {
3138 buf = buf->b_next;
3139 ASSERT(buf);

--- 151 unchanged lines hidden (view full) ---

3291
3292 /*
3293 * Do we have more than one buf?
3294 */
3295 if (hdr->b_datacnt > 1) {
3296 arc_buf_hdr_t *nhdr;
3297 arc_buf_t **bufp;
3298 uint64_t blksz = hdr->b_size;
3257 spa_t *spa = hdr->b_spa;
3299 uint64_t spa = hdr->b_spa;
3258 arc_buf_contents_t type = hdr->b_type;
3259 uint32_t flags = hdr->b_flags;
3260
3261 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3262 /*
3263 * Pull the data off of this buf and attach it to
3264 * a new anonymous buf.
3265 */

--- 268 unchanged lines hidden (view full) ---

3534
3535int
3536arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
3537 zio_done_func_t *done, void *private, uint32_t arc_flags)
3538{
3539 arc_buf_hdr_t *ab;
3540 kmutex_t *hash_lock;
3541 zio_t *zio;
3300 arc_buf_contents_t type = hdr->b_type;
3301 uint32_t flags = hdr->b_flags;
3302
3303 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3304 /*
3305 * Pull the data off of this buf and attach it to
3306 * a new anonymous buf.
3307 */

--- 268 unchanged lines hidden (view full) ---

3576
3577int
3578arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
3579 zio_done_func_t *done, void *private, uint32_t arc_flags)
3580{
3581 arc_buf_hdr_t *ab;
3582 kmutex_t *hash_lock;
3583 zio_t *zio;
3584 uint64_t guid = spa_guid(spa);
3542
3543 /*
3544 * If this buffer is in the cache, release it, so it
3545 * can be re-used.
3546 */
3585
3586 /*
3587 * If this buffer is in the cache, release it, so it
3588 * can be re-used.
3589 */
3547 ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
3590 ab = buf_hash_find(guid, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
3548 if (ab != NULL) {
3549 /*
3550 * The checksum of blocks to free is not always
3551 * preserved (eg. on the deadlist). However, if it is
3552 * nonzero, it should match what we have in the cache.
3553 */
3554 ASSERT(bp->blk_cksum.zc_word[0] == 0 ||
3555 bp->blk_cksum.zc_word[0] == ab->b_cksum0 ||

--- 46 unchanged lines hidden (view full) ---

3602
3603 ASSERT(arc_flags & ARC_NOWAIT);
3604 zio_nowait(zio);
3605
3606 return (0);
3607}
3608
3609static int
3591 if (ab != NULL) {
3592 /*
3593 * The checksum of blocks to free is not always
3594 * preserved (eg. on the deadlist). However, if it is
3595 * nonzero, it should match what we have in the cache.
3596 */
3597 ASSERT(bp->blk_cksum.zc_word[0] == 0 ||
3598 bp->blk_cksum.zc_word[0] == ab->b_cksum0 ||

--- 46 unchanged lines hidden (view full) ---

3645
3646 ASSERT(arc_flags & ARC_NOWAIT);
3647 zio_nowait(zio);
3648
3649 return (0);
3650}
3651
3652static int
3610arc_memory_throttle(uint64_t reserve, uint64_t txg)
3653arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg)
3611{
3612#ifdef _KERNEL
3654{
3655#ifdef _KERNEL
3613 uint64_t inflight_data = arc_anon->arcs_size;
3614 uint64_t available_memory = ptoa((uintmax_t)cnt.v_free_count);
3615 static uint64_t page_load = 0;
3616 static uint64_t last_txg = 0;
3617
3618#if 0
3619#if defined(__i386)
3620 available_memory =
3621 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));

--- 47 unchanged lines hidden (view full) ---

3669 atomic_add_64(&arc_tempreserve, -reserve);
3670 ASSERT((int64_t)arc_tempreserve >= 0);
3671}
3672
3673int
3674arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3675{
3676 int error;
3656 uint64_t available_memory = ptoa((uintmax_t)cnt.v_free_count);
3657 static uint64_t page_load = 0;
3658 static uint64_t last_txg = 0;
3659
3660#if 0
3661#if defined(__i386)
3662 available_memory =
3663 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));

--- 47 unchanged lines hidden (view full) ---

3711 atomic_add_64(&arc_tempreserve, -reserve);
3712 ASSERT((int64_t)arc_tempreserve >= 0);
3713}
3714
3715int
3716arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3717{
3718 int error;
3719 uint64_t anon_size;
3677
3678#ifdef ZFS_DEBUG
3679 /*
3680 * Once in a while, fail for no reason. Everything should cope.
3681 */
3682 if (spa_get_random(10000) == 0) {
3683 dprintf("forcing random failure\n");
3684 return (ERESTART);
3685 }
3686#endif
3687 if (reserve > arc_c/4 && !arc_no_grow)
3688 arc_c = MIN(arc_c_max, reserve * 4);
3689 if (reserve > arc_c)
3690 return (ENOMEM);
3691
3692 /*
3720
3721#ifdef ZFS_DEBUG
3722 /*
3723 * Once in a while, fail for no reason. Everything should cope.
3724 */
3725 if (spa_get_random(10000) == 0) {
3726 dprintf("forcing random failure\n");
3727 return (ERESTART);
3728 }
3729#endif
3730 if (reserve > arc_c/4 && !arc_no_grow)
3731 arc_c = MIN(arc_c_max, reserve * 4);
3732 if (reserve > arc_c)
3733 return (ENOMEM);
3734
3735 /*
3736 * Don't count loaned bufs as in flight dirty data to prevent long
3737 * network delays from blocking transactions that are ready to be
3738 * assigned to a txg.
3739 */
3740 anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3741
3742 /*
3693 * Writes will, almost always, require additional memory allocations
3694 * in order to compress/encrypt/etc the data. We therefor need to
3695 * make sure that there is sufficient available memory for this.
3696 */
3743 * Writes will, almost always, require additional memory allocations
3744 * in order to compress/encrypt/etc the data. We therefor need to
3745 * make sure that there is sufficient available memory for this.
3746 */
3697 if (error = arc_memory_throttle(reserve, txg))
3747 if (error = arc_memory_throttle(reserve, anon_size, txg))
3698 return (error);
3699
3700 /*
3701 * Throttle writes when the amount of dirty data in the cache
3702 * gets too large. We try to keep the cache less than half full
3703 * of dirty blocks so that our sync times don't grow too large.
3704 * Note: if two requests come in concurrently, we might let them
3705 * both succeed, when one of them should fail. Not a huge deal.
3706 */
3748 return (error);
3749
3750 /*
3751 * Throttle writes when the amount of dirty data in the cache
3752 * gets too large. We try to keep the cache less than half full
3753 * of dirty blocks so that our sync times don't grow too large.
3754 * Note: if two requests come in concurrently, we might let them
3755 * both succeed, when one of them should fail. Not a huge deal.
3756 */
3707 if (reserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 &&
3708 arc_anon->arcs_size > arc_c / 4) {
3757
3758 if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3759 anon_size > arc_c / 4) {
3709 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3710 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3711 arc_tempreserve>>10,
3712 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3713 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3714 reserve>>10, arc_c>>10);
3715 return (ERESTART);
3716 }

--- 237 unchanged lines hidden (view full) ---

3954 mutex_destroy(&arc_mfu_ghost->arcs_locks[i].arcs_lock);
3955 mutex_destroy(&arc_l2c_only->arcs_locks[i].arcs_lock);
3956 }
3957
3958 mutex_destroy(&zfs_write_limit_lock);
3959
3960 buf_fini();
3961
3760 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3761 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3762 arc_tempreserve>>10,
3763 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3764 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3765 reserve>>10, arc_c>>10);
3766 return (ERESTART);
3767 }

--- 237 unchanged lines hidden (view full) ---

4005 mutex_destroy(&arc_mfu_ghost->arcs_locks[i].arcs_lock);
4006 mutex_destroy(&arc_l2c_only->arcs_locks[i].arcs_lock);
4007 }
4008
4009 mutex_destroy(&zfs_write_limit_lock);
4010
4011 buf_fini();
4012
4013 ASSERT(arc_loaned_bytes == 0);
4014
3962 mutex_destroy(&arc_lowmem_lock);
3963#ifdef _KERNEL
3964 if (arc_event_lowmem != NULL)
3965 EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem);
3966#endif
3967}
3968
3969/*

--- 128 unchanged lines hidden (view full) ---

4098 * l2arc_write_size() calculate how much to write
4099 * l2arc_write_interval() calculate sleep delay between writes
4100 *
4101 * These three functions determine what to write, how much, and how quickly
4102 * to send writes.
4103 */
4104
4105static boolean_t
4015 mutex_destroy(&arc_lowmem_lock);
4016#ifdef _KERNEL
4017 if (arc_event_lowmem != NULL)
4018 EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem);
4019#endif
4020}
4021
4022/*

--- 128 unchanged lines hidden (view full) ---

4151 * l2arc_write_size() calculate how much to write
4152 * l2arc_write_interval() calculate sleep delay between writes
4153 *
4154 * These three functions determine what to write, how much, and how quickly
4155 * to send writes.
4156 */
4157
4158static boolean_t
4106l2arc_write_eligible(spa_t *spa, arc_buf_hdr_t *ab)
4159l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
4107{
4108 /*
4109 * A buffer is *not* eligible for the L2ARC if it:
4110 * 1. belongs to a different spa.
4111 * 2. is already cached on the L2ARC.
4112 * 3. has an I/O in progress (it may be an incomplete read).
4113 * 4. is flagged not eligible (zfs property).
4114 */
4160{
4161 /*
4162 * A buffer is *not* eligible for the L2ARC if it:
4163 * 1. belongs to a different spa.
4164 * 2. is already cached on the L2ARC.
4165 * 3. has an I/O in progress (it may be an incomplete read).
4166 * 4. is flagged not eligible (zfs property).
4167 */
4115 if (ab->b_spa != spa) {
4168 if (ab->b_spa != spa_guid) {
4116 ARCSTAT_BUMP(arcstat_l2_write_spa_mismatch);
4117 return (B_FALSE);
4118 }
4119 if (ab->b_l2hdr != NULL) {
4120 ARCSTAT_BUMP(arcstat_l2_write_in_l2);
4121 return (B_FALSE);
4122 }
4123 if (HDR_IO_IN_PROGRESS(ab)) {

--- 270 unchanged lines hidden (view full) ---

4394 if (!equal)
4395 ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4396
4397 /*
4398 * If there's no waiter, issue an async i/o to the primary
4399 * storage now. If there *is* a waiter, the caller must
4400 * issue the i/o in a context where it's OK to block.
4401 */
4169 ARCSTAT_BUMP(arcstat_l2_write_spa_mismatch);
4170 return (B_FALSE);
4171 }
4172 if (ab->b_l2hdr != NULL) {
4173 ARCSTAT_BUMP(arcstat_l2_write_in_l2);
4174 return (B_FALSE);
4175 }
4176 if (HDR_IO_IN_PROGRESS(ab)) {

--- 270 unchanged lines hidden (view full) ---

4447 if (!equal)
4448 ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4449
4450 /*
4451 * If there's no waiter, issue an async i/o to the primary
4452 * storage now. If there *is* a waiter, the caller must
4453 * issue the i/o in a context where it's OK to block.
4454 */
4402 if (zio->io_waiter == NULL)
4403 zio_nowait(zio_read(zio->io_parent,
4404 cb->l2rcb_spa, &cb->l2rcb_bp,
4455 if (zio->io_waiter == NULL) {
4456 zio_t *pio = zio_unique_parent(zio);
4457
4458 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4459
4460 zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4405 buf->b_data, zio->io_size, arc_read_done, buf,
4406 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4461 buf->b_data, zio->io_size, arc_read_done, buf,
4462 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4463 }
4407 }
4408
4409 kmem_free(cb, sizeof (l2arc_read_callback_t));
4410}
4411
4412/*
4413 * This is the list priority from which the L2ARC will search for pages to
4414 * cache. This is used within loops (0..3) to cycle through lists in the

--- 180 unchanged lines hidden (view full) ---

4595 l2arc_buf_hdr_t *hdrl2;
4596 list_t *list;
4597 uint64_t passed_sz, write_sz, buf_sz, headroom;
4598 void *buf_data;
4599 kmutex_t *hash_lock, *list_lock;
4600 boolean_t have_lock, full;
4601 l2arc_write_callback_t *cb;
4602 zio_t *pio, *wzio;
4464 }
4465
4466 kmem_free(cb, sizeof (l2arc_read_callback_t));
4467}
4468
4469/*
4470 * This is the list priority from which the L2ARC will search for pages to
4471 * cache. This is used within loops (0..3) to cycle through lists in the

--- 180 unchanged lines hidden (view full) ---

4652 l2arc_buf_hdr_t *hdrl2;
4653 list_t *list;
4654 uint64_t passed_sz, write_sz, buf_sz, headroom;
4655 void *buf_data;
4656 kmutex_t *hash_lock, *list_lock;
4657 boolean_t have_lock, full;
4658 l2arc_write_callback_t *cb;
4659 zio_t *pio, *wzio;
4660 uint64_t guid = spa_guid(spa);
4603 int try;
4604
4605 ASSERT(dev->l2ad_vdev != NULL);
4606
4607 pio = NULL;
4608 write_sz = 0;
4609 full = B_FALSE;
4610 head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);

--- 45 unchanged lines hidden (view full) ---

4656 /*
4657 * Searched too far.
4658 */
4659 mutex_exit(hash_lock);
4660 ARCSTAT_BUMP(arcstat_l2_write_passed_headroom);
4661 break;
4662 }
4663
4661 int try;
4662
4663 ASSERT(dev->l2ad_vdev != NULL);
4664
4665 pio = NULL;
4666 write_sz = 0;
4667 full = B_FALSE;
4668 head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);

--- 45 unchanged lines hidden (view full) ---

4714 /*
4715 * Searched too far.
4716 */
4717 mutex_exit(hash_lock);
4718 ARCSTAT_BUMP(arcstat_l2_write_passed_headroom);
4719 break;
4720 }
4721
4664 if (!l2arc_write_eligible(spa, ab)) {
4722 if (!l2arc_write_eligible(guid, ab)) {
4665 mutex_exit(hash_lock);
4666 continue;
4667 }
4668
4669 if ((write_sz + ab->b_size) > target_sz) {
4670 full = B_TRUE;
4671 mutex_exit(hash_lock);
4672 ARCSTAT_BUMP(arcstat_l2_write_full);

--- 323 unchanged lines hidden (view full) ---

4996
4997 list_destroy(l2arc_dev_list);
4998 list_destroy(l2arc_free_on_write);
4999}
5000
5001void
5002l2arc_start(void)
5003{
4723 mutex_exit(hash_lock);
4724 continue;
4725 }
4726
4727 if ((write_sz + ab->b_size) > target_sz) {
4728 full = B_TRUE;
4729 mutex_exit(hash_lock);
4730 ARCSTAT_BUMP(arcstat_l2_write_full);

--- 323 unchanged lines hidden (view full) ---

5054
5055 list_destroy(l2arc_dev_list);
5056 list_destroy(l2arc_free_on_write);
5057}
5058
5059void
5060l2arc_start(void)
5061{
5004 if (!(spa_mode & FWRITE))
5062 if (!(spa_mode_global & FWRITE))
5005 return;
5006
5007 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
5008 TS_RUN, minclsyspri);
5009}
5010
5011void
5012l2arc_stop(void)
5013{
5063 return;
5064
5065 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
5066 TS_RUN, minclsyspri);
5067}
5068
5069void
5070l2arc_stop(void)
5071{
5014 if (!(spa_mode & FWRITE))
5072 if (!(spa_mode_global & FWRITE))
5015 return;
5016
5017 mutex_enter(&l2arc_feed_thr_lock);
5018 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
5019 l2arc_thread_exit = 1;
5020 while (l2arc_thread_exit != 0)
5021 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
5022 mutex_exit(&l2arc_feed_thr_lock);
5023}
5073 return;
5074
5075 mutex_enter(&l2arc_feed_thr_lock);
5076 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
5077 l2arc_thread_exit = 1;
5078 while (l2arc_thread_exit != 0)
5079 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
5080 mutex_exit(&l2arc_feed_thr_lock);
5081}