Deleted Added
sdiff udiff text old ( 323752 ) new ( 323754 )
full compact
arc.c (323752) arc.c (323754)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 708 unchanged lines hidden (view full) ---

717 kstat_named_t arcstat_l2_writes_lock_retry;
718 kstat_named_t arcstat_l2_evict_lock_retry;
719 kstat_named_t arcstat_l2_evict_reading;
720 kstat_named_t arcstat_l2_evict_l1cached;
721 kstat_named_t arcstat_l2_free_on_write;
722 kstat_named_t arcstat_l2_abort_lowmem;
723 kstat_named_t arcstat_l2_cksum_bad;
724 kstat_named_t arcstat_l2_io_error;
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 708 unchanged lines hidden (view full) ---

717 kstat_named_t arcstat_l2_writes_lock_retry;
718 kstat_named_t arcstat_l2_evict_lock_retry;
719 kstat_named_t arcstat_l2_evict_reading;
720 kstat_named_t arcstat_l2_evict_l1cached;
721 kstat_named_t arcstat_l2_free_on_write;
722 kstat_named_t arcstat_l2_abort_lowmem;
723 kstat_named_t arcstat_l2_cksum_bad;
724 kstat_named_t arcstat_l2_io_error;
725 kstat_named_t arcstat_l2_size;
726 kstat_named_t arcstat_l2_asize;
725 kstat_named_t arcstat_l2_lsize;
726 kstat_named_t arcstat_l2_psize;
727 kstat_named_t arcstat_l2_hdr_size;
728 kstat_named_t arcstat_l2_write_trylock_fail;
729 kstat_named_t arcstat_l2_write_passed_headroom;
730 kstat_named_t arcstat_l2_write_spa_mismatch;
731 kstat_named_t arcstat_l2_write_in_l2;
732 kstat_named_t arcstat_l2_write_hdr_io_in_progress;
733 kstat_named_t arcstat_l2_write_not_cacheable;
734 kstat_named_t arcstat_l2_write_full;

--- 2626 unchanged lines hidden (view full) ---

3361 return (buf);
3362}
3363
3364static void
3365arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3366{
3367 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3368 l2arc_dev_t *dev = l2hdr->b_dev;
727 kstat_named_t arcstat_l2_hdr_size;
728 kstat_named_t arcstat_l2_write_trylock_fail;
729 kstat_named_t arcstat_l2_write_passed_headroom;
730 kstat_named_t arcstat_l2_write_spa_mismatch;
731 kstat_named_t arcstat_l2_write_in_l2;
732 kstat_named_t arcstat_l2_write_hdr_io_in_progress;
733 kstat_named_t arcstat_l2_write_not_cacheable;
734 kstat_named_t arcstat_l2_write_full;

--- 2626 unchanged lines hidden (view full) ---

3361 return (buf);
3362}
3363
3364static void
3365arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3366{
3367 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3368 l2arc_dev_t *dev = l2hdr->b_dev;
3369 uint64_t asize = arc_hdr_size(hdr);
3369 uint64_t psize = arc_hdr_size(hdr);
3370
3371 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3372 ASSERT(HDR_HAS_L2HDR(hdr));
3373
3374 list_remove(&dev->l2ad_buflist, hdr);
3375
3370
3371 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3372 ASSERT(HDR_HAS_L2HDR(hdr));
3373
3374 list_remove(&dev->l2ad_buflist, hdr);
3375
3376 ARCSTAT_INCR(arcstat_l2_asize, -asize);
3377 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
3376 ARCSTAT_INCR(arcstat_l2_psize, -psize);
3377 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr));
3378
3378
3379 vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3379 vdev_space_update(dev->l2ad_vdev, -psize, 0, 0);
3380
3380
3381 (void) refcount_remove_many(&dev->l2ad_alloc, asize, hdr);
3381 (void) refcount_remove_many(&dev->l2ad_alloc, psize, hdr);
3382 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3383}
3384
3385static void
3386arc_hdr_destroy(arc_buf_hdr_t *hdr)
3387{
3388 if (HDR_HAS_L1HDR(hdr)) {
3389 ASSERT(hdr->b_l1hdr.b_buf == NULL ||

--- 3686 unchanged lines hidden (view full) ---

7076 if (zio->io_error != 0) {
7077 /*
7078 * Error - drop L2ARC entry.
7079 */
7080 list_remove(buflist, hdr);
7081 l2arc_trim(hdr);
7082 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
7083
3382 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3383}
3384
3385static void
3386arc_hdr_destroy(arc_buf_hdr_t *hdr)
3387{
3388 if (HDR_HAS_L1HDR(hdr)) {
3389 ASSERT(hdr->b_l1hdr.b_buf == NULL ||

--- 3686 unchanged lines hidden (view full) ---

7076 if (zio->io_error != 0) {
7077 /*
7078 * Error - drop L2ARC entry.
7079 */
7080 list_remove(buflist, hdr);
7081 l2arc_trim(hdr);
7082 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
7083
7084 ARCSTAT_INCR(arcstat_l2_asize, -arc_hdr_size(hdr));
7085 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
7084 ARCSTAT_INCR(arcstat_l2_psize, -arc_hdr_size(hdr));
7085 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr));
7086
7087 bytes_dropped += arc_hdr_size(hdr);
7088 (void) refcount_remove_many(&dev->l2ad_alloc,
7089 arc_hdr_size(hdr), hdr);
7090 }
7091
7092 /*
7093 * Allow ARC to begin reads and ghost list evictions to

--- 239 unchanged lines hidden (view full) ---

7333 break;
7334 }
7335
7336 if (!HDR_HAS_L1HDR(hdr)) {
7337 ASSERT(!HDR_L2_READING(hdr));
7338 /*
7339 * This doesn't exist in the ARC. Destroy.
7340 * arc_hdr_destroy() will call list_remove()
7086
7087 bytes_dropped += arc_hdr_size(hdr);
7088 (void) refcount_remove_many(&dev->l2ad_alloc,
7089 arc_hdr_size(hdr), hdr);
7090 }
7091
7092 /*
7093 * Allow ARC to begin reads and ghost list evictions to

--- 239 unchanged lines hidden (view full) ---

7333 break;
7334 }
7335
7336 if (!HDR_HAS_L1HDR(hdr)) {
7337 ASSERT(!HDR_L2_READING(hdr));
7338 /*
7339 * This doesn't exist in the ARC. Destroy.
7340 * arc_hdr_destroy() will call list_remove()
7341 * and decrement arcstat_l2_size.
7341 * and decrement arcstat_l2_lsize.
7342 */
7343 arc_change_state(arc_anon, hdr, hash_lock);
7344 arc_hdr_destroy(hdr);
7345 } else {
7346 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
7347 ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
7348 /*
7349 * Invalidate issued or about to be issued

--- 22 unchanged lines hidden (view full) ---

7372 *
7373 * Returns the number of bytes actually written (which may be smaller than
7374 * the delta by which the device hand has changed due to alignment).
7375 */
7376static uint64_t
7377l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
7378{
7379 arc_buf_hdr_t *hdr, *hdr_prev, *head;
7342 */
7343 arc_change_state(arc_anon, hdr, hash_lock);
7344 arc_hdr_destroy(hdr);
7345 } else {
7346 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
7347 ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
7348 /*
7349 * Invalidate issued or about to be issued

--- 22 unchanged lines hidden (view full) ---

7372 *
7373 * Returns the number of bytes actually written (which may be smaller than
7374 * the delta by which the device hand has changed due to alignment).
7375 */
7376static uint64_t
7377l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
7378{
7379 arc_buf_hdr_t *hdr, *hdr_prev, *head;
7380 uint64_t write_asize, write_psize, write_sz, headroom;
7380 uint64_t write_asize, write_psize, write_lsize, headroom;
7381 boolean_t full;
7382 l2arc_write_callback_t *cb;
7383 zio_t *pio, *wzio;
7384 uint64_t guid = spa_load_guid(spa);
7385 int try;
7386
7387 ASSERT3P(dev->l2ad_vdev, !=, NULL);
7388
7389 pio = NULL;
7381 boolean_t full;
7382 l2arc_write_callback_t *cb;
7383 zio_t *pio, *wzio;
7384 uint64_t guid = spa_load_guid(spa);
7385 int try;
7386
7387 ASSERT3P(dev->l2ad_vdev, !=, NULL);
7388
7389 pio = NULL;
7390 write_sz = write_asize = write_psize = 0;
7390 write_lsize = write_asize = write_psize = 0;
7391 full = B_FALSE;
7392 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
7393 arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
7394
7395 ARCSTAT_BUMP(arcstat_l2_write_buffer_iter);
7396 /*
7397 * Copy buffers for L2ARC writing.
7398 */

--- 60 unchanged lines hidden (view full) ---

7459 * of the ghost cache, prior to being written out. The
7460 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
7461 */
7462 ASSERT(HDR_HAS_L1HDR(hdr));
7463
7464 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
7465 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
7466 ASSERT3U(arc_hdr_size(hdr), >, 0);
7391 full = B_FALSE;
7392 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
7393 arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
7394
7395 ARCSTAT_BUMP(arcstat_l2_write_buffer_iter);
7396 /*
7397 * Copy buffers for L2ARC writing.
7398 */

--- 60 unchanged lines hidden (view full) ---

7459 * of the ghost cache, prior to being written out. The
7460 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
7461 */
7462 ASSERT(HDR_HAS_L1HDR(hdr));
7463
7464 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
7465 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
7466 ASSERT3U(arc_hdr_size(hdr), >, 0);
7467 uint64_t size = arc_hdr_size(hdr);
7467 uint64_t psize = arc_hdr_size(hdr);
7468 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
7468 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
7469 size);
7469 psize);
7470
7470
7471 if ((write_psize + asize) > target_sz) {
7471 if ((write_asize + asize) > target_sz) {
7472 full = B_TRUE;
7473 mutex_exit(hash_lock);
7474 ARCSTAT_BUMP(arcstat_l2_write_full);
7475 break;
7476 }
7477
7478 if (pio == NULL) {
7479 /*

--- 18 unchanged lines hidden (view full) ---

7498 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
7499 arc_hdr_set_flags(hdr,
7500 ARC_FLAG_L2_WRITING | ARC_FLAG_HAS_L2HDR);
7501
7502 mutex_enter(&dev->l2ad_mtx);
7503 list_insert_head(&dev->l2ad_buflist, hdr);
7504 mutex_exit(&dev->l2ad_mtx);
7505
7472 full = B_TRUE;
7473 mutex_exit(hash_lock);
7474 ARCSTAT_BUMP(arcstat_l2_write_full);
7475 break;
7476 }
7477
7478 if (pio == NULL) {
7479 /*

--- 18 unchanged lines hidden (view full) ---

7498 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
7499 arc_hdr_set_flags(hdr,
7500 ARC_FLAG_L2_WRITING | ARC_FLAG_HAS_L2HDR);
7501
7502 mutex_enter(&dev->l2ad_mtx);
7503 list_insert_head(&dev->l2ad_buflist, hdr);
7504 mutex_exit(&dev->l2ad_mtx);
7505
7506 (void) refcount_add_many(&dev->l2ad_alloc, size, hdr);
7506 (void) refcount_add_many(&dev->l2ad_alloc, psize, hdr);
7507
7508 /*
7509 * Normally the L2ARC can use the hdr's data, but if
7510 * we're sharing data between the hdr and one of its
7511 * bufs, L2ARC needs its own copy of the data so that
7512 * the ZIO below can't race with the buf consumer.
7513 * Another case where we need to create a copy of the
7514 * data is when the buffer size is not device-aligned
7515 * and we need to pad the block to make it such.
7516 * That also keeps the clock hand suitably aligned.
7517 *
7518 * To ensure that the copy will be available for the
7519 * lifetime of the ZIO and be cleaned up afterwards, we
7520 * add it to the l2arc_free_on_write queue.
7521 */
7522 abd_t *to_write;
7507
7508 /*
7509 * Normally the L2ARC can use the hdr's data, but if
7510 * we're sharing data between the hdr and one of its
7511 * bufs, L2ARC needs its own copy of the data so that
7512 * the ZIO below can't race with the buf consumer.
7513 * Another case where we need to create a copy of the
7514 * data is when the buffer size is not device-aligned
7515 * and we need to pad the block to make it such.
7516 * That also keeps the clock hand suitably aligned.
7517 *
7518 * To ensure that the copy will be available for the
7519 * lifetime of the ZIO and be cleaned up afterwards, we
7520 * add it to the l2arc_free_on_write queue.
7521 */
7522 abd_t *to_write;
7523 if (!HDR_SHARED_DATA(hdr) && size == asize) {
7523 if (!HDR_SHARED_DATA(hdr) && psize == asize) {
7524 to_write = hdr->b_l1hdr.b_pabd;
7525 } else {
7526 to_write = abd_alloc_for_io(asize,
7527 HDR_ISTYPE_METADATA(hdr));
7524 to_write = hdr->b_l1hdr.b_pabd;
7525 } else {
7526 to_write = abd_alloc_for_io(asize,
7527 HDR_ISTYPE_METADATA(hdr));
7528 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
7529 if (asize != size) {
7530 abd_zero_off(to_write, size,
7531 asize - size);
7528 abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize);
7529 if (asize != psize) {
7530 abd_zero_off(to_write, psize,
7531 asize - psize);
7532 }
7533 l2arc_free_abd_on_write(to_write, asize,
7534 arc_buf_type(hdr));
7535 }
7536 wzio = zio_write_phys(pio, dev->l2ad_vdev,
7537 hdr->b_l2hdr.b_daddr, asize, to_write,
7538 ZIO_CHECKSUM_OFF, NULL, hdr,
7539 ZIO_PRIORITY_ASYNC_WRITE,
7540 ZIO_FLAG_CANFAIL, B_FALSE);
7541
7532 }
7533 l2arc_free_abd_on_write(to_write, asize,
7534 arc_buf_type(hdr));
7535 }
7536 wzio = zio_write_phys(pio, dev->l2ad_vdev,
7537 hdr->b_l2hdr.b_daddr, asize, to_write,
7538 ZIO_CHECKSUM_OFF, NULL, hdr,
7539 ZIO_PRIORITY_ASYNC_WRITE,
7540 ZIO_FLAG_CANFAIL, B_FALSE);
7541
7542 write_sz += HDR_GET_LSIZE(hdr);
7542 write_lsize += HDR_GET_LSIZE(hdr);
7543 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
7544 zio_t *, wzio);
7545
7543 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
7544 zio_t *, wzio);
7545
7546 write_asize += size;
7547 write_psize += asize;
7546 write_psize += psize;
7547 write_asize += asize;
7548 dev->l2ad_hand += asize;
7549
7550 mutex_exit(hash_lock);
7551
7552 (void) zio_nowait(wzio);
7553 }
7554
7555 multilist_sublist_unlock(mls);
7556
7557 if (full == B_TRUE)
7558 break;
7559 }
7560
7561 /* No buffers selected for writing? */
7562 if (pio == NULL) {
7548 dev->l2ad_hand += asize;
7549
7550 mutex_exit(hash_lock);
7551
7552 (void) zio_nowait(wzio);
7553 }
7554
7555 multilist_sublist_unlock(mls);
7556
7557 if (full == B_TRUE)
7558 break;
7559 }
7560
7561 /* No buffers selected for writing? */
7562 if (pio == NULL) {
7563 ASSERT0(write_sz);
7563 ASSERT0(write_lsize);
7564 ASSERT(!HDR_HAS_L1HDR(head));
7565 kmem_cache_free(hdr_l2only_cache, head);
7566 return (0);
7567 }
7568
7569 ASSERT3U(write_psize, <=, target_sz);
7570 ARCSTAT_BUMP(arcstat_l2_writes_sent);
7564 ASSERT(!HDR_HAS_L1HDR(head));
7565 kmem_cache_free(hdr_l2only_cache, head);
7566 return (0);
7567 }
7568
7569 ASSERT3U(write_psize, <=, target_sz);
7570 ARCSTAT_BUMP(arcstat_l2_writes_sent);
7571 ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
7572 ARCSTAT_INCR(arcstat_l2_size, write_sz);
7573 ARCSTAT_INCR(arcstat_l2_asize, write_asize);
7574 vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
7571 ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);
7572 ARCSTAT_INCR(arcstat_l2_lsize, write_lsize);
7573 ARCSTAT_INCR(arcstat_l2_psize, write_psize);
7574 vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
7575
7576 /*
7577 * Bump device hand to the device start if it is approaching the end.
7578 * l2arc_evict() will already have evicted ahead for this case.
7579 */
7580 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
7581 dev->l2ad_hand = dev->l2ad_start;
7582 dev->l2ad_first = B_FALSE;

--- 270 unchanged lines hidden --- 		7575
7576 /*
7577 * Bump device hand to the device start if it is approaching the end.
7578 * l2arc_evict() will already have evicted ahead for this case.
7579 */
7580 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
7581 dev->l2ad_hand = dev->l2ad_start;
7582 dev->l2ad_first = B_FALSE;

--- 270 unchanged lines hidden ---