Deleted Added
sdiff udiff text old ( 339034 ) new ( 339114 )
full compact
arc.c (339034) arc.c (339114)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 7 unchanged lines hidden (view full) ---

16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 7 unchanged lines hidden (view full) ---

16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is

--- 5139 unchanged lines hidden (view full) ---

5172
5173/* a generic arc_read_done_func_t */
5174/* ARGSUSED */
5175void
5176arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5177 arc_buf_t *buf, void *arg)
5178{
5179 arc_buf_t **bufp = arg;
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
27 */
28
29/*
30 * DVA-based Adjustable Replacement Cache
31 *
32 * While much of the theory of operation used here is

--- 5139 unchanged lines hidden (view full) ---

5172
5173/* a generic arc_read_done_func_t */
5174/* ARGSUSED */
5175void
5176arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5177 arc_buf_t *buf, void *arg)
5178{
5179 arc_buf_t **bufp = arg;
5180
5181 if (buf == NULL) {
5180 if (buf == NULL) {
5181 ASSERT(zio == NULL || zio->io_error != 0);
5182 *bufp = NULL;
5183 } else {
5182 *bufp = NULL;
5183 } else {
5184 ASSERT(zio == NULL || zio->io_error == 0);
5184 *bufp = buf;
5185 *bufp = buf;
5185 ASSERT(buf->b_data);
5186 ASSERT(buf->b_data != NULL);
5186 }
5187}
5188
5189static void
5190arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5191{
5192 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5193 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);

--- 11 unchanged lines hidden (view full) ---

5205static void
5206arc_read_done(zio_t *zio)
5207{
5208 arc_buf_hdr_t *hdr = zio->io_private;
5209 kmutex_t *hash_lock = NULL;
5210 arc_callback_t *callback_list;
5211 arc_callback_t *acb;
5212 boolean_t freeable = B_FALSE;
5187 }
5188}
5189
5190static void
5191arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5192{
5193 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5194 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);

--- 11 unchanged lines hidden (view full) ---

5206static void
5207arc_read_done(zio_t *zio)
5208{
5209 arc_buf_hdr_t *hdr = zio->io_private;
5210 kmutex_t *hash_lock = NULL;
5211 arc_callback_t *callback_list;
5212 arc_callback_t *acb;
5213 boolean_t freeable = B_FALSE;
5214 boolean_t no_zio_error = (zio->io_error == 0);
5213
5214 /*
5215 * The hdr was inserted into hash-table and removed from lists
5216 * prior to starting I/O. We should find this header, since
5217 * it's in the hash table, and it should be legit since it's
5218 * not possible to evict it during the I/O. The only possible
5219 * reason for it not to be found is if we were freed during the
5220 * read.

--- 9 unchanged lines hidden (view full) ---

5230 &hash_lock);
5231
5232 ASSERT((found == hdr &&
5233 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5234 (found == hdr && HDR_L2_READING(hdr)));
5235 ASSERT3P(hash_lock, !=, NULL);
5236 }
5237
5215
5216 /*
5217 * The hdr was inserted into hash-table and removed from lists
5218 * prior to starting I/O. We should find this header, since
5219 * it's in the hash table, and it should be legit since it's
5220 * not possible to evict it during the I/O. The only possible
5221 * reason for it not to be found is if we were freed during the
5222 * read.

--- 9 unchanged lines hidden (view full) ---

5232 &hash_lock);
5233
5234 ASSERT((found == hdr &&
5235 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5236 (found == hdr && HDR_L2_READING(hdr)));
5237 ASSERT3P(hash_lock, !=, NULL);
5238 }
5239
5238 if (zio->io_error == 0) {
5240 if (no_zio_error) {
5239 /* byteswap if necessary */
5240 if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5241 if (BP_GET_LEVEL(zio->io_bp) > 0) {
5242 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5243 } else {
5244 hdr->b_l1hdr.b_byteswap =
5245 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5246 }

--- 4 unchanged lines hidden (view full) ---

5251
5252 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5253 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5254 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5255
5256 callback_list = hdr->b_l1hdr.b_acb;
5257 ASSERT3P(callback_list, !=, NULL);
5258
5241 /* byteswap if necessary */
5242 if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5243 if (BP_GET_LEVEL(zio->io_bp) > 0) {
5244 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5245 } else {
5246 hdr->b_l1hdr.b_byteswap =
5247 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5248 }

--- 4 unchanged lines hidden (view full) ---

5253
5254 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5255 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5256 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5257
5258 callback_list = hdr->b_l1hdr.b_acb;
5259 ASSERT3P(callback_list, !=, NULL);
5260
5259 if (hash_lock && zio->io_error == 0 &&
5260 hdr->b_l1hdr.b_state == arc_anon) {
5261 if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
5261 /*
5262 * Only call arc_access on anonymous buffers. This is because
5263 * if we've issued an I/O for an evicted buffer, we've already
5264 * called arc_access (to prevent any simultaneous readers from
5265 * getting confused).
5266 */
5267 arc_access(hdr, hash_lock);
5268 }

--- 6 unchanged lines hidden (view full) ---

5275 */
5276 int callback_cnt = 0;
5277 for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5278 if (!acb->acb_done)
5279 continue;
5280
5281 callback_cnt++;
5282
5262 /*
5263 * Only call arc_access on anonymous buffers. This is because
5264 * if we've issued an I/O for an evicted buffer, we've already
5265 * called arc_access (to prevent any simultaneous readers from
5266 * getting confused).
5267 */
5268 arc_access(hdr, hash_lock);
5269 }

--- 6 unchanged lines hidden (view full) ---

5276 */
5277 int callback_cnt = 0;
5278 for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5279 if (!acb->acb_done)
5280 continue;
5281
5282 callback_cnt++;
5283
5283 if (zio->io_error != 0)
5284 continue;
5285
5286 int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5287 acb->acb_compressed,
5288 B_TRUE, &acb->acb_buf);
5289 if (error != 0) {
5290 arc_buf_destroy(acb->acb_buf, acb->acb_private);
5291 acb->acb_buf = NULL;
5284 if (no_zio_error) {
5285 int error = arc_buf_alloc_impl(hdr, acb->acb_private,
5286 acb->acb_compressed, zio->io_error == 0,
5287 &acb->acb_buf);
5288 if (error != 0) {
5289 /*
5290 * Decompression failed. Set io_error
5291 * so that when we call acb_done (below),
5292 * we will indicate that the read failed.
5293 * Note that in the unusual case where one
5294 * callback is compressed and another
5295 * uncompressed, we will mark all of them
5296 * as failed, even though the uncompressed
5297 * one can't actually fail. In this case,
5298 * the hdr will not be anonymous, because
5299 * if there are multiple callbacks, it's
5300 * because multiple threads found the same
5301 * arc buf in the hash table.
5302 */
5303 zio->io_error = error;
5304 }
5292 }
5305 }
5293
5294 if (zio->io_error == 0)
5295 zio->io_error = error;
5296 }
5306 }
5307 /*
5308 * If there are multiple callbacks, we must have the hash lock,
5309 * because the only way for multiple threads to find this hdr is
5310 * in the hash table. This ensures that if there are multiple
5311 * callbacks, the hdr is not anonymous. If it were anonymous,
5312 * we couldn't use arc_buf_destroy() in the error case below.
5313 */
5314 ASSERT(callback_cnt < 2 || hash_lock != NULL);
5315
5297 hdr->b_l1hdr.b_acb = NULL;
5298 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5299 if (callback_cnt == 0) {
5300 ASSERT(HDR_PREFETCH(hdr));
5301 ASSERT0(hdr->b_l1hdr.b_bufcnt);
5302 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5303 }
5304
5305 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5306 callback_list != NULL);
5307
5316 hdr->b_l1hdr.b_acb = NULL;
5317 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5318 if (callback_cnt == 0) {
5319 ASSERT(HDR_PREFETCH(hdr));
5320 ASSERT0(hdr->b_l1hdr.b_bufcnt);
5321 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5322 }
5323
5324 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
5325 callback_list != NULL);
5326
5308 if (zio->io_error == 0) {
5327 if (no_zio_error) {
5309 arc_hdr_verify(hdr, zio->io_bp);
5310 } else {
5311 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5312 if (hdr->b_l1hdr.b_state != arc_anon)
5313 arc_change_state(arc_anon, hdr, hash_lock);
5314 if (HDR_IN_HASH_TABLE(hdr))
5315 buf_hash_remove(hdr);
5316 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);

--- 16 unchanged lines hidden (view full) ---

5333 * in the cache).
5334 */
5335 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5336 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5337 }
5338
5339 /* execute each callback and free its structure */
5340 while ((acb = callback_list) != NULL) {
5328 arc_hdr_verify(hdr, zio->io_bp);
5329 } else {
5330 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5331 if (hdr->b_l1hdr.b_state != arc_anon)
5332 arc_change_state(arc_anon, hdr, hash_lock);
5333 if (HDR_IN_HASH_TABLE(hdr))
5334 buf_hash_remove(hdr);
5335 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);

--- 16 unchanged lines hidden (view full) ---

5352 * in the cache).
5353 */
5354 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
5355 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
5356 }
5357
5358 /* execute each callback and free its structure */
5359 while ((acb = callback_list) != NULL) {
5341 if (acb->acb_done) {
5360 if (acb->acb_done != NULL) {
5361 if (zio->io_error != 0 && acb->acb_buf != NULL) {
5362 /*
5363 * If arc_buf_alloc_impl() fails during
5364 * decompression, the buf will still be
5365 * allocated, and needs to be freed here.
5366 */
5367 arc_buf_destroy(acb->acb_buf, acb->acb_private);
5368 acb->acb_buf = NULL;
5369 }
5342 acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5343 acb->acb_buf, acb->acb_private);
5344 }
5345
5346 if (acb->acb_zio_dummy != NULL) {
5347 acb->acb_zio_dummy->io_error = zio->io_error;
5348 zio_nowait(acb->acb_zio_dummy);
5349 }

--- 2706 unchanged lines hidden --- 		5370 acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5371 acb->acb_buf, acb->acb_private);
5372 }
5373
5374 if (acb->acb_zio_dummy != NULL) {
5375 acb->acb_zio_dummy->io_error = zio->io_error;
5376 zio_nowait(acb->acb_zio_dummy);
5377 }

--- 2706 unchanged lines hidden ---