module/zfs/dmu_zfetch.c

117 	zfetch_stats_t *zs = ksp->ks_data;
121 	zs->zfetchstat_hits.value.ui64 =
123 	zs->zfetchstat_future.value.ui64 =
125 	zs->zfetchstat_stride.value.ui64 =
127 	zs->zfetchstat_past.value.ui64 =
129 	zs->zfetchstat_misses.value.ui64 =
131 	zs->zfetchstat_max_streams.value.ui64 =
133 	zs->zfetchstat_io_issued.value.ui64 =
135 	zs->zfetchstat_io_active.value.ui64 =
202 dmu_zfetch_stream_fini(zstream_t *zs)
204 	ASSERT(!list_link_active(&zs->zs_node));
205 	zfs_refcount_destroy(&zs->zs_callers);
206 	zfs_refcount_destroy(&zs->zs_refs);
207 	kmem_free(zs, sizeof (*zs));
211 dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)
214 	list_remove(&zf->zf_stream, zs);
217 	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
218 		dmu_zfetch_stream_fini(zs);
228 	zstream_t *zs;
231 	while ((zs = list_head(&zf->zf_stream)) != NULL)
232 		dmu_zfetch_stream_remove(zf, zs);
249 	zstream_t *zs, *zs_next, *zs_old = NULL;
258 	for (zs = list_head(&zf->zf_stream); zs != NULL; zs = zs_next) {
259 		zs_next = list_next(&zf->zf_stream, zs);
263 		if ((int)(zs->zs_atime - t) >= 0)
265 		if (zfs_refcount_count(&zs->zs_refs) != 1)
268 			dmu_zfetch_stream_remove(zf, zs);
270 			zs_old = zs;
273 		zs = zs_old;
274 		list_remove(&zf->zf_stream, zs);
288 		for (zs = list_head(&zf->zf_stream); zs != NULL;
289 		    zs = list_next(&zf->zf_stream, zs)) {
290 			if ((int)(zs->zs_atime - t) >= 0)
292 			if (zfs_refcount_count(&zs->zs_refs) != 1)
295 			    (int)(zs_old->zs_atime - zs->zs_atime) >= 0)
296 				zs_old = zs;
299 			zs = zs_old;
300 			list_remove(&zf->zf_stream, zs);
307 	zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
308 	zfs_refcount_create(&zs->zs_callers);
309 	zfs_refcount_create(&zs->zs_refs);
311 	zfs_refcount_add(&zs->zs_refs, NULL);
315 	list_insert_head(&zf->zf_stream, zs);
316 	zs->zs_blkid = blkid;
318 	zs->zs_atime = now - zfetch_min_sec_reap;
319 	memset(zs->zs_ranges, 0, sizeof (zs->zs_ranges));
320 	zs->zs_pf_dist = 0;
321 	zs->zs_ipf_dist = 0;
322 	zs->zs_pf_start = blkid;
323 	zs->zs_pf_end = blkid;
324 	zs->zs_ipf_start = blkid;
325 	zs->zs_ipf_end = blkid;
326 	zs->zs_missed = B_FALSE;
327 	zs->zs_more = B_FALSE;
333 	zstream_t *zs = arg;
335 	if (io_issued && level == 0 && blkid < zs->zs_blkid)
336 		zs->zs_more = B_TRUE;
337 	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
338 		dmu_zfetch_stream_fini(zs);
347 dmu_zfetch_hit(zstream_t *zs, uint64_t nblks)
352 	if (zs->zs_ranges[0].start == 0)
357 		zsrange_t *r = &zs->zs_ranges[i];
369 		if (zs->zs_ranges[i].start == 0)
371 		ASSERT3U(zs->zs_ranges[i].start, >, nblks);
372 		ASSERT3U(zs->zs_ranges[i].end, >, nblks);
373 		zs->zs_ranges[j].start = zs->zs_ranges[i].start - nblks;
374 		zs->zs_ranges[j].end = zs->zs_ranges[i].end - nblks;
377 		zs->zs_ranges[j].start = 0;
378 		zs->zs_ranges[j].end = 0;
382 	zs->zs_blkid += nblks;
391 dmu_zfetch_future(zstream_t *zs, uint64_t blkid, uint64_t nblks)
393 	ASSERT3U(blkid, >, zs->zs_blkid);
394 	blkid -= zs->zs_blkid;
400 		zsrange_t *r = &zs->zs_ranges[i];
412 		if (zs->zs_ranges[f].start > blkid)
413 			zs->zs_ranges[f].start = blkid;
414 		zs->zs_ranges[f].end = MAX(zs->zs_ranges[l].end, blkid + nblks);
418 				zs->zs_ranges[f].start = zs->zs_ranges[l].start;
419 				zs->zs_ranges[f].end = zs->zs_ranges[l].end;
421 			zs->zs_ranges[f].start = 0;
422 			zs->zs_ranges[f].end = 0;
427 			zs->zs_ranges[l].start = zs->zs_ranges[l - 1].start;
428 			zs->zs_ranges[l].end = zs->zs_ranges[l - 1].end;
430 		zs->zs_ranges[i].start = blkid;
431 		zs->zs_ranges[i].end = blkid + nblks;
442 		zsrange_t *r = &zs->zs_ranges[i];
451 		return (dmu_zfetch_hit(zs, l));
472 	zstream_t *zs;
521 	for (zs = list_head(&zf->zf_stream); zs != NULL;
522 	    zs = list_next(&zf->zf_stream, zs)) {
523 		if (blkid == zs->zs_blkid) {
525 		} else if (blkid + 1 == zs->zs_blkid) {
541 	for (zs = list_head(&zf->zf_stream); zs != NULL;
542 	    zs = list_next(&zf->zf_stream, zs)) {
543 		if (blkid > zs->zs_blkid) {
544 			if (end_blkid <= zs->zs_blkid + max_reorder) {
546 					nblks = dmu_zfetch_hit(zs,
547 					    end_blkid - zs->zs_blkid);
551 				nblks = dmu_zfetch_future(zs, blkid, nblks);
558 		} else if (end_blkid >= zs->zs_blkid) {
559 			nblks -= zs->zs_blkid - blkid;
560 			blkid += zs->zs_blkid - blkid;
562 		} else if (end_blkid + max_reorder > zs->zs_blkid &&
563 		    (int)(zs->zs_atime - t) >= 0) {
565 			zs->zs_atime = gethrestime_sec();
583 	nblks = dmu_zfetch_hit(zs, nblks);
587 	zs->zs_atime = gethrestime_sec();
594 	end_blkid = zs->zs_blkid;
596 		dmu_zfetch_stream_remove(zf, zs);
619 		if (unlikely(zs->zs_pf_dist < nbytes))
620 			zs->zs_pf_dist = nbytes;
621 		else if (zs->zs_pf_dist < zfetch_min_distance &&
622 		    (zs->zs_pf_dist < (1 << dbs) ||
625 			zs->zs_pf_dist *= 2;
626 		else if (zs->zs_more)
627 			zs->zs_pf_dist += zs->zs_pf_dist / 8;
628 		zs->zs_more = B_FALSE;
629 		if (zs->zs_pf_dist > zfetch_max_distance)
630 			zs->zs_pf_dist = zfetch_max_distance;
631 		pf_nblks = zs->zs_pf_dist >> dbs;
635 	if (zs->zs_pf_start < end_blkid)
636 		zs->zs_pf_start = end_blkid;
637 	if (zs->zs_pf_end < end_blkid + pf_nblks)
638 		zs->zs_pf_end = end_blkid + pf_nblks;
644 	if (unlikely(zs->zs_ipf_dist < nbytes))
645 		zs->zs_ipf_dist = nbytes;
647 		zs->zs_ipf_dist *= 2;
648 	if (zs->zs_ipf_dist > zfetch_max_idistance)
649 		zs->zs_ipf_dist = zfetch_max_idistance;
650 	pf_nblks = zs->zs_ipf_dist >> dbs;
651 	if (zs->zs_ipf_start < zs->zs_pf_end)
652 		zs->zs_ipf_start = zs->zs_pf_end;
653 	if (zs->zs_ipf_end < zs->zs_pf_end + pf_nblks)
654 		zs->zs_ipf_end = zs->zs_pf_end + pf_nblks;
656 	zfs_refcount_add(&zs->zs_refs, NULL);
658 	zfs_refcount_add(&zs->zs_callers, NULL);
663 	return (zs);
667 dmu_zfetch_run(zfetch_t *zf, zstream_t *zs, boolean_t missed,
674 		zs->zs_missed = missed;
681 	if (zfs_refcount_remove(&zs->zs_callers, NULL) != 0) {
683 		if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
684 			dmu_zfetch_stream_fini(zs);
689 	if (zs->zs_missed) {
690 		pf_start = zs->zs_pf_start;
691 		pf_end = zs->zs_pf_start = zs->zs_pf_end;
695 	ipf_start = zs->zs_ipf_start;
696 	ipf_end = zs->zs_ipf_start = zs->zs_ipf_end;
708 		zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL);
711 		if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
712 			dmu_zfetch_stream_fini(zs);
723 		    ZIO_PRIORITY_ASYNC_READ, 0, dmu_zfetch_done, zs);
727 		    ZIO_PRIORITY_ASYNC_READ, 0, dmu_zfetch_done, zs);
741 	zstream_t *zs;
743 	zs = dmu_zfetch_prepare(zf, blkid, nblks, fetch_data, have_lock);
744 	if (zs)
745 		dmu_zfetch_run(zf, zs, missed, have_lock);