342c342,343
< static int arc_min_prefetch_lifespan;
---
> static int zfs_arc_min_prefetch_ms = 1;
> static int zfs_arc_min_prescient_prefetch_ms = 6;
786c787
< kstat_named_t arcstat_sync_wait_for_async;
---
> kstat_named_t arcstat_async_upgrade_sync;
787a789
> kstat_named_t arcstat_demand_hit_prescient_prefetch;
884c886
< { "sync_wait_for_async", KSTAT_DATA_UINT64 },
---
> { "async_upgrade_sync", KSTAT_DATA_UINT64 },
885a888
> { "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
981c984
< arc_done_func_t *acb_done;
---
> arc_read_done_func_t *acb_done;
984a988
> zio_t *acb_zio_head;
991,996c995,1000
< void *awcb_private;
< arc_done_func_t *awcb_ready;
< arc_done_func_t *awcb_children_ready;
< arc_done_func_t *awcb_physdone;
< arc_done_func_t *awcb_done;
< arc_buf_t *awcb_buf;
---
> void *awcb_private;
> arc_write_done_func_t *awcb_ready;
> arc_write_done_func_t *awcb_children_ready;
> arc_write_done_func_t *awcb_physdone;
> arc_write_done_func_t *awcb_done;
> arc_buf_t *awcb_buf;
1235a1240,1241
> #define HDR_PRESCIENT_PREFETCH(hdr) \
> ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
1398a1405,1409
> SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prefetch_ms, CTLFLAG_RW,
> &zfs_arc_min_prefetch_ms, 0, "Min life of prefetch block in ms");
> SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prescient_prefetch_ms, CTLFLAG_RW,
> &zfs_arc_min_prescient_prefetch_ms, 0, "Min life of prescient prefetched block in ms");
>
3550a3562,3563
> int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
> zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms;
3603,3604c3616
< ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
< arc_min_prefetch_lifespan)) {
---
> ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) {
5000c5012
< if (HDR_PREFETCH(hdr)) {
---
> if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5006c5018,5020
< arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
---
> arc_hdr_clear_flags(hdr,
> ARC_FLAG_PREFETCH |
> ARC_FLAG_PRESCIENT_PREFETCH);
5037c5051
< if (HDR_PREFETCH(hdr)) {
---
> if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5039,5040c5053,5057
< if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
< arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
---
> if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) {
> arc_hdr_clear_flags(hdr,
> ARC_FLAG_PREFETCH |
> ARC_FLAG_PRESCIENT_PREFETCH);
> }
5061,5065c5078
< if ((HDR_PREFETCH(hdr)) != 0) {
< ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
< /* link protected by hash_lock */
< ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
< }
---
>
5076c5089
< if (HDR_PREFETCH(hdr)) {
---
> if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) {
5081d5093
< ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
5148c5160
< /* a generic arc_done_func_t which you can use */
---
> /* a generic arc_read_done_func_t which you can use */
5151c5163,5164
< arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
---
> arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
> arc_buf_t *buf, void *arg)
5153,5154c5166,5169
< if (zio == NULL || zio->io_error == 0)
< bcopy(buf->b_data, arg, arc_buf_size(buf));
---
> if (buf == NULL)
> return;
>
> bcopy(buf->b_data, arg, arc_buf_size(buf));
5158c5173,5174
< /* a generic arc_done_func_t */
---
> /* a generic arc_read_done_func_t */
> /* ARGSUSED */
5160c5176,5177
< arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
---
> arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
> arc_buf_t *buf, void *arg)
5163,5164c5180,5181
< if (zio && zio->io_error) {
< arc_buf_destroy(buf, arg);
---
>
> if (buf == NULL) {
5196d5212
< boolean_t no_zio_error = (zio->io_error == 0);
5222c5238
< if (no_zio_error) {
---
> if (zio->io_error == 0) {
5243c5259,5260
< if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
---
> if (hash_lock && zio->io_error == 0 &&
> hdr->b_l1hdr.b_state == arc_anon) {
5264d5280
< /* This is a demand read since prefetches don't use callbacks */
5266a5283,5285
> if (zio->io_error != 0)
> continue;
>
5268,5270c5287,5291
< acb->acb_compressed, no_zio_error, &acb->acb_buf);
< if (no_zio_error) {
< zio->io_error = error;
---
> acb->acb_compressed,
> B_TRUE, &acb->acb_buf);
> if (error != 0) {
> arc_buf_destroy(acb->acb_buf, acb->acb_private);
> acb->acb_buf = NULL;
5271a5293,5295
>
> if (zio->io_error == 0)
> zio->io_error = error;
5284c5308
< if (no_zio_error) {
---
> if (zio->io_error == 0) {
5317,5318c5341,5344
< if (acb->acb_done)
< acb->acb_done(zio, acb->acb_buf, acb->acb_private);
---
> if (acb->acb_done) {
> acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
> acb->acb_buf, acb->acb_private);
> }
5352c5378
< arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
---
> arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done,
5361c5387,5388
<
---
> int rc = 0;
>
5378a5406
> zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5379a5408
> ASSERT3P(head_zio, !=, NULL);
5383,5400c5412,5414
< * This sync read must wait for an
< * in-progress async read (e.g. a predictive
< * prefetch). Async reads are queued
< * separately at the vdev_queue layer, so
< * this is a form of priority inversion.
< * Ideally, we would "inherit" the demand
< * i/o's priority by moving the i/o from
< * the async queue to the synchronous queue,
< * but there is currently no mechanism to do
< * so. Track this so that we can evaluate
< * the magnitude of this potential performance
< * problem.
< *
< * Note that if the prefetch i/o is already
< * active (has been issued to the device),
< * the prefetch improved performance, because
< * we issued it sooner than we would have
< * without the prefetch.
---
> * This is a sync read that needs to wait for
> * an in-flight async read. Request that the
> * zio have its priority upgraded.
5402c5416,5417
< DTRACE_PROBE1(arc__sync__wait__for__async,
---
> zio_change_priority(head_zio, priority);
> DTRACE_PROBE1(arc__async__upgrade__sync,
5404c5419
< ARCSTAT_BUMP(arcstat_sync_wait_for_async);
---
> ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5430a5446
> acb->acb_zio_head = head_zio;
5458d5473
< ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5459a5475,5482
> if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
> ARCSTAT_BUMP(
> arcstat_demand_hit_prescient_prefetch);
> arc_hdr_clear_flags(hdr,
> ARC_FLAG_PRESCIENT_PREFETCH);
> }
>
> ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp));
5461,5462c5484,5491
< VERIFY0(arc_buf_alloc_impl(hdr, private,
< compressed_read, B_TRUE, &buf));
---
> rc = arc_buf_alloc_impl(hdr, private,
> compressed_read, B_TRUE, &buf);
> if (rc != 0) {
> arc_buf_destroy(buf, private);
> buf = NULL;
> }
> ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
> rc == 0 || rc != ENOENT);
5468a5498,5499
> if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
> arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5478c5509
< done(NULL, buf, private);
---
> done(NULL, zb, bp, buf, private);
5551a5583,5585
> if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH)
> arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
>
5581c5615,5621
< if (priority == ZIO_PRIORITY_ASYNC_READ)
---
> /*
> * We count both async reads and scrub IOs as asynchronous so
> * that both can be upgraded in the event of a cache hit while
> * the read IO is still in-flight.
> */
> if (priority == ZIO_PRIORITY_ASYNC_READ ||
> priority == ZIO_PRIORITY_SCRUB)
5586,5588d5625
< if (hash_lock != NULL)
< mutex_exit(hash_lock);
<
5668a5706,5710
> acb->acb_zio_head = rzio;
>
> if (hash_lock != NULL)
> mutex_exit(hash_lock);
>
5682a5725,5726
> if (hash_lock != NULL)
> mutex_enter(hash_lock);
5702a5747
> acb->acb_zio_head = rzio;
5703a5749,5751
> if (hash_lock != NULL)
> mutex_exit(hash_lock);
>
6194,6196c6242,6244
< boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready,
< arc_done_func_t *children_ready, arc_done_func_t *physdone,
< arc_done_func_t *done, void *private, zio_priority_t priority,
---
> boolean_t l2arc, const zio_prop_t *zp, arc_write_done_func_t *ready,
> arc_write_done_func_t *children_ready, arc_write_done_func_t *physdone,
> arc_write_done_func_t *done, void *private, zio_priority_t priority,
6624,6626d6671
< /* Convert seconds to clock ticks */
< arc_min_prefetch_lifespan = 1 * hz;
<