Deleted Added
sdiff udiff text old ( 321552 ) new ( 321553 )
full compact
arc.c (321552) arc.c (321553)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 459 unchanged lines hidden (view full) ---

468 * buffer header in the hash table, so that reads that hit the
469 * second level ARC benefit from these fast lookups.
470 */
471
472typedef struct arc_state {
473 /*
474 * list of evictable buffers
475 */
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 459 unchanged lines hidden (view full) ---

468 * buffer header in the hash table, so that reads that hit the
469 * second level ARC benefit from these fast lookups.
470 */
471
472typedef struct arc_state {
473 /*
474 * list of evictable buffers
475 */
476 multilist_t arcs_list[ARC_BUFC_NUMTYPES];
476 multilist_t *arcs_list[ARC_BUFC_NUMTYPES];
477 /*
478 * total amount of evictable data in this state
479 */
480 refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
481 /*
482 * total amount of data in this state; this includes: evictable,
483 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
484 */

--- 1869 unchanged lines hidden (view full) ---

2354 }
2355
2356 arc_state_t *state = hdr->b_l1hdr.b_state;
2357
2358 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2359 (state != arc_anon)) {
2360 /* We don't use the L2-only state list. */
2361 if (state != arc_l2c_only) {
477 /*
478 * total amount of evictable data in this state
479 */
480 refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
481 /*
482 * total amount of data in this state; this includes: evictable,
483 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
484 */

--- 1869 unchanged lines hidden (view full) ---

2354 }
2355
2356 arc_state_t *state = hdr->b_l1hdr.b_state;
2357
2358 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2359 (state != arc_anon)) {
2360 /* We don't use the L2-only state list. */
2361 if (state != arc_l2c_only) {
2362 multilist_remove(&state->arcs_list[arc_buf_type(hdr)],
2362 multilist_remove(state->arcs_list[arc_buf_type(hdr)],
2363 hdr);
2364 arc_evictable_space_decrement(hdr, state);
2365 }
2366 /* remove the prefetch flag if we get a reference */
2367 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2368 }
2369}
2370

--- 13 unchanged lines hidden (view full) ---

2384 ASSERT(!GHOST_STATE(state));
2385
2386 /*
2387 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2388 * check to prevent usage of the arc_l2c_only list.
2389 */
2390 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2391 (state != arc_anon)) {
2363 hdr);
2364 arc_evictable_space_decrement(hdr, state);
2365 }
2366 /* remove the prefetch flag if we get a reference */
2367 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2368 }
2369}
2370

--- 13 unchanged lines hidden (view full) ---

2384 ASSERT(!GHOST_STATE(state));
2385
2386 /*
2387 * arc_l2c_only counts as a ghost state so we don't need to explicitly
2388 * check to prevent usage of the arc_l2c_only list.
2389 */
2390 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2391 (state != arc_anon)) {
2392 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2392 multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
2393 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2394 arc_evictable_space_increment(hdr, state);
2395 }
2396 return (cnt);
2397}
2398
2399/*
2400 * Move the supplied buffer to the indicated state. The hash lock

--- 36 unchanged lines hidden (view full) ---

2437
2438 /*
2439 * If this buffer is evictable, transfer it from the
2440 * old state list to the new state list.
2441 */
2442 if (refcnt == 0) {
2443 if (old_state != arc_anon && old_state != arc_l2c_only) {
2444 ASSERT(HDR_HAS_L1HDR(hdr));
2393 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2394 arc_evictable_space_increment(hdr, state);
2395 }
2396 return (cnt);
2397}
2398
2399/*
2400 * Move the supplied buffer to the indicated state. The hash lock

--- 36 unchanged lines hidden (view full) ---

2437
2438 /*
2439 * If this buffer is evictable, transfer it from the
2440 * old state list to the new state list.
2441 */
2442 if (refcnt == 0) {
2443 if (old_state != arc_anon && old_state != arc_l2c_only) {
2444 ASSERT(HDR_HAS_L1HDR(hdr));
2445 multilist_remove(&old_state->arcs_list[buftype], hdr);
2445 multilist_remove(old_state->arcs_list[buftype], hdr);
2446
2447 if (GHOST_STATE(old_state)) {
2448 ASSERT0(bufcnt);
2449 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2450 update_old = B_TRUE;
2451 }
2452 arc_evictable_space_decrement(hdr, old_state);
2453 }
2454 if (new_state != arc_anon && new_state != arc_l2c_only) {
2455
2456 /*
2457 * An L1 header always exists here, since if we're
2458 * moving to some L1-cached state (i.e. not l2c_only or
2459 * anonymous), we realloc the header to add an L1hdr
2460 * beforehand.
2461 */
2462 ASSERT(HDR_HAS_L1HDR(hdr));
2446
2447 if (GHOST_STATE(old_state)) {
2448 ASSERT0(bufcnt);
2449 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2450 update_old = B_TRUE;
2451 }
2452 arc_evictable_space_decrement(hdr, old_state);
2453 }
2454 if (new_state != arc_anon && new_state != arc_l2c_only) {
2455
2456 /*
2457 * An L1 header always exists here, since if we're
2458 * moving to some L1-cached state (i.e. not l2c_only or
2459 * anonymous), we realloc the header to add an L1hdr
2460 * beforehand.
2461 */
2462 ASSERT(HDR_HAS_L1HDR(hdr));
2463 multilist_insert(&new_state->arcs_list[buftype], hdr);
2463 multilist_insert(new_state->arcs_list[buftype], hdr);
2464
2465 if (GHOST_STATE(new_state)) {
2466 ASSERT0(bufcnt);
2467 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2468 update_new = B_TRUE;
2469 }
2470 arc_evictable_space_increment(hdr, new_state);
2471 }

--- 109 unchanged lines hidden (view full) ---

2581
2582 if (HDR_HAS_L1HDR(hdr))
2583 hdr->b_l1hdr.b_state = new_state;
2584
2585 /*
2586 * L2 headers should never be on the L2 state list since they don't
2587 * have L1 headers allocated.
2588 */
2464
2465 if (GHOST_STATE(new_state)) {
2466 ASSERT0(bufcnt);
2467 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2468 update_new = B_TRUE;
2469 }
2470 arc_evictable_space_increment(hdr, new_state);
2471 }

--- 109 unchanged lines hidden (view full) ---

2581
2582 if (HDR_HAS_L1HDR(hdr))
2583 hdr->b_l1hdr.b_state = new_state;
2584
2585 /*
2586 * L2 headers should never be on the L2 state list since they don't
2587 * have L1 headers allocated.
2588 */
2589 ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2590 multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2589 ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
2590 multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
2591}
2592
2593void
2594arc_space_consume(uint64_t space, arc_space_type_t type)
2595{
2596 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2597
2598 switch (type) {

--- 1067 unchanged lines hidden (view full) ---

3666 * will evict all available (i.e. unlocked and evictable) buffers from
3667 * the given arc state; which is used by arc_flush().
3668 */
3669static uint64_t
3670arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3671 arc_buf_contents_t type)
3672{
3673 uint64_t total_evicted = 0;
2591}
2592
2593void
2594arc_space_consume(uint64_t space, arc_space_type_t type)
2595{
2596 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2597
2598 switch (type) {

--- 1067 unchanged lines hidden (view full) ---

3666 * will evict all available (i.e. unlocked and evictable) buffers from
3667 * the given arc state; which is used by arc_flush().
3668 */
3669static uint64_t
3670arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
3671 arc_buf_contents_t type)
3672{
3673 uint64_t total_evicted = 0;
3674 multilist_t *ml = &state->arcs_list[type];
3674 multilist_t *ml = state->arcs_list[type];
3675 int num_sublists;
3676 arc_buf_hdr_t **markers;
3677
3678 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3679
3680 num_sublists = multilist_get_num_sublists(ml);
3681
3682 /*

--- 187 unchanged lines hidden (view full) ---

3870 * This function will select a random sublist of type ARC_BUFC_DATA and
3871 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
3872 * is compared, and the type which contains the "older" buffer will be
3873 * returned.
3874 */
3875static arc_buf_contents_t
3876arc_adjust_type(arc_state_t *state)
3877{
3675 int num_sublists;
3676 arc_buf_hdr_t **markers;
3677
3678 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
3679
3680 num_sublists = multilist_get_num_sublists(ml);
3681
3682 /*

--- 187 unchanged lines hidden (view full) ---

3870 * This function will select a random sublist of type ARC_BUFC_DATA and
3871 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
3872 * is compared, and the type which contains the "older" buffer will be
3873 * returned.
3874 */
3875static arc_buf_contents_t
3876arc_adjust_type(arc_state_t *state)
3877{
3878 multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
3879 multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
3878 multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
3879 multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
3880 int data_idx = multilist_get_random_index(data_ml);
3881 int meta_idx = multilist_get_random_index(meta_ml);
3882 multilist_sublist_t *data_mls;
3883 multilist_sublist_t *meta_mls;
3884 arc_buf_contents_t type;
3885 arc_buf_hdr_t *data_hdr;
3886 arc_buf_hdr_t *meta_hdr;
3887

--- 2316 unchanged lines hidden (view full) ---

6204{
6205 arc_anon = &ARC_anon;
6206 arc_mru = &ARC_mru;
6207 arc_mru_ghost = &ARC_mru_ghost;
6208 arc_mfu = &ARC_mfu;
6209 arc_mfu_ghost = &ARC_mfu_ghost;
6210 arc_l2c_only = &ARC_l2c_only;
6211
3880 int data_idx = multilist_get_random_index(data_ml);
3881 int meta_idx = multilist_get_random_index(meta_ml);
3882 multilist_sublist_t *data_mls;
3883 multilist_sublist_t *meta_mls;
3884 arc_buf_contents_t type;
3885 arc_buf_hdr_t *data_hdr;
3886 arc_buf_hdr_t *meta_hdr;
3887

--- 2316 unchanged lines hidden (view full) ---

6204{
6205 arc_anon = &ARC_anon;
6206 arc_mru = &ARC_mru;
6207 arc_mru_ghost = &ARC_mru_ghost;
6208 arc_mfu = &ARC_mfu;
6209 arc_mfu_ghost = &ARC_mfu_ghost;
6210 arc_l2c_only = &ARC_l2c_only;
6211
6212 multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
6213 sizeof (arc_buf_hdr_t),
6212 arc_mru->arcs_list[ARC_BUFC_METADATA] =
6213 multilist_create(sizeof (arc_buf_hdr_t),
6214 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6215 arc_state_multilist_index_func);
6214 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6215 arc_state_multilist_index_func);
6216 multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
6217 sizeof (arc_buf_hdr_t),
6216 arc_mru->arcs_list[ARC_BUFC_DATA] =
6217 multilist_create(sizeof (arc_buf_hdr_t),
6218 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6219 arc_state_multilist_index_func);
6218 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6219 arc_state_multilist_index_func);
6220 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
6221 sizeof (arc_buf_hdr_t),
6220 arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] =
6221 multilist_create(sizeof (arc_buf_hdr_t),
6222 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6223 arc_state_multilist_index_func);
6222 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6223 arc_state_multilist_index_func);
6224 multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
6225 sizeof (arc_buf_hdr_t),
6224 arc_mru_ghost->arcs_list[ARC_BUFC_DATA] =
6225 multilist_create(sizeof (arc_buf_hdr_t),
6226 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6227 arc_state_multilist_index_func);
6226 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6227 arc_state_multilist_index_func);
6228 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
6229 sizeof (arc_buf_hdr_t),
6228 arc_mfu->arcs_list[ARC_BUFC_METADATA] =
6229 multilist_create(sizeof (arc_buf_hdr_t),
6230 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6231 arc_state_multilist_index_func);
6230 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6231 arc_state_multilist_index_func);
6232 multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
6233 sizeof (arc_buf_hdr_t),
6232 arc_mfu->arcs_list[ARC_BUFC_DATA] =
6233 multilist_create(sizeof (arc_buf_hdr_t),
6234 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6235 arc_state_multilist_index_func);
6234 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6235 arc_state_multilist_index_func);
6236 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
6237 sizeof (arc_buf_hdr_t),
6236 arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] =
6237 multilist_create(sizeof (arc_buf_hdr_t),
6238 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6239 arc_state_multilist_index_func);
6238 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6239 arc_state_multilist_index_func);
6240 multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
6241 sizeof (arc_buf_hdr_t),
6240 arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] =
6241 multilist_create(sizeof (arc_buf_hdr_t),
6242 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6243 arc_state_multilist_index_func);
6242 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6243 arc_state_multilist_index_func);
6244 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
6245 sizeof (arc_buf_hdr_t),
6244 arc_l2c_only->arcs_list[ARC_BUFC_METADATA] =
6245 multilist_create(sizeof (arc_buf_hdr_t),
6246 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6247 arc_state_multilist_index_func);
6246 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6247 arc_state_multilist_index_func);
6248 multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
6249 sizeof (arc_buf_hdr_t),
6248 arc_l2c_only->arcs_list[ARC_BUFC_DATA] =
6249 multilist_create(sizeof (arc_buf_hdr_t),
6250 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6251 arc_state_multilist_index_func);
6252
6253 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6254 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6255 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6256 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6257 refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);

--- 31 unchanged lines hidden (view full) ---

6289
6290 refcount_destroy(&arc_anon->arcs_size);
6291 refcount_destroy(&arc_mru->arcs_size);
6292 refcount_destroy(&arc_mru_ghost->arcs_size);
6293 refcount_destroy(&arc_mfu->arcs_size);
6294 refcount_destroy(&arc_mfu_ghost->arcs_size);
6295 refcount_destroy(&arc_l2c_only->arcs_size);
6296
6250 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
6251 arc_state_multilist_index_func);
6252
6253 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6254 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6255 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
6256 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
6257 refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);

--- 31 unchanged lines hidden (view full) ---

6289
6290 refcount_destroy(&arc_anon->arcs_size);
6291 refcount_destroy(&arc_mru->arcs_size);
6292 refcount_destroy(&arc_mru_ghost->arcs_size);
6293 refcount_destroy(&arc_mfu->arcs_size);
6294 refcount_destroy(&arc_mfu_ghost->arcs_size);
6295 refcount_destroy(&arc_l2c_only->arcs_size);
6296
6297 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
6298 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
6299 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
6300 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
6301 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
6302 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
6303 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
6304 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
6297 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]);
6298 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
6299 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]);
6300 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
6301 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]);
6302 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
6303 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]);
6304 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
6305}
6306
6307uint64_t
6308arc_max_bytes(void)
6309{
6310 return (arc_c_max);
6311}
6312

--- 780 unchanged lines hidden (view full) ---

7093{
7094 multilist_t *ml = NULL;
7095 unsigned int idx;
7096
7097 ASSERT(list_num >= 0 && list_num <= 3);
7098
7099 switch (list_num) {
7100 case 0:
6305}
6306
6307uint64_t
6308arc_max_bytes(void)
6309{
6310 return (arc_c_max);
6311}
6312

--- 780 unchanged lines hidden (view full) ---

7093{
7094 multilist_t *ml = NULL;
7095 unsigned int idx;
7096
7097 ASSERT(list_num >= 0 && list_num <= 3);
7098
7099 switch (list_num) {
7100 case 0:
7101 ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
7101 ml = arc_mfu->arcs_list[ARC_BUFC_METADATA];
7102 break;
7103 case 1:
7102 break;
7103 case 1:
7104 ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
7104 ml = arc_mru->arcs_list[ARC_BUFC_METADATA];
7105 break;
7106 case 2:
7105 break;
7106 case 2:
7107 ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
7107 ml = arc_mfu->arcs_list[ARC_BUFC_DATA];
7108 break;
7109 case 3:
7108 break;
7109 case 3:
7110 ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
7110 ml = arc_mru->arcs_list[ARC_BUFC_DATA];
7111 break;
7112 }
7113
7114 /*
7115 * Return a randomly-selected sublist. This is acceptable
7116 * because the caller feeds only a little bit of data for each
7117 * call (8MB). Subsequent calls will result in different
7118 * sublists being selected.

--- 604 unchanged lines hidden --- 		7111 break;
7112 }
7113
7114 /*
7115 * Return a randomly-selected sublist. This is acceptable
7116 * because the caller feeds only a little bit of data for each
7117 * call (8MB). Subsequent calls will result in different
7118 * sublists being selected.

--- 604 unchanged lines hidden ---