fs/zfs/vdev_queue.c

337 vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)
339 	return (&vq->vq_class[p].vqc_queued_tree);
343 vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t)
346 		return (&vq->vq_read_offset_tree);
348 		return (&vq->vq_write_offset_tree);
380 	vdev_queue_t *vq = &vd->vdev_queue;
382 	mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL);
383 	vq->vq_vdev = vd;
385 	avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,
387 	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ),
390 	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE),
407 		avl_create(vdev_queue_class_tree(vq, p), compfn,
411 	vq->vq_lastoffset = 0;
417 	vdev_queue_t *vq = &vd->vdev_queue;
420 		avl_destroy(vdev_queue_class_tree(vq, p));
421 	avl_destroy(&vq->vq_active_tree);
422 	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ));
423 	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE));
425 	mutex_destroy(&vq->vq_lock);
429 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)
434 	ASSERT(MUTEX_HELD(&vq->vq_lock));
436 	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
437 	qtt = vdev_queue_type_tree(vq, zio->io_type);
451 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)
456 	ASSERT(MUTEX_HELD(&vq->vq_lock));
458 	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
459 	qtt = vdev_queue_type_tree(vq, zio->io_type);
474 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)
477 	ASSERT(MUTEX_HELD(&vq->vq_lock));
479 	vq->vq_class[zio->io_priority].vqc_active++;
480 	avl_add(&vq->vq_active_tree, zio);
492 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)
495 	ASSERT(MUTEX_HELD(&vq->vq_lock));
497 	vq->vq_class[zio->io_priority].vqc_active--;
498 	avl_remove(&vq->vq_active_tree, zio);
631 vdev_queue_class_to_issue(vdev_queue_t *vq)
633 	spa_t *spa = vq->vq_vdev->vdev_spa;
636 	ASSERT(MUTEX_HELD(&vq->vq_lock));
638 	if (avl_numnodes(&vq->vq_active_tree) >= zfs_vdev_max_active)
643 		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
644 		    vq->vq_class[p].vqc_active <
654 		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
655 		    vq->vq_class[p].vqc_active <
674 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
686 	ASSERT(MUTEX_HELD(&vq->vq_lock));
688 	maxblocksize = spa_maxblocksize(vq->vq_vdev->vdev_spa);
689 	if (vq->vq_vdev->vdev_nonrot)
723 	t = vdev_queue_type_tree(vq, zio->io_type);
830 		vdev_queue_io_remove(vq, dio);
837 	mutex_exit(&vq->vq_lock);
842 	mutex_enter(&vq->vq_lock);
848 vdev_queue_io_to_issue(vdev_queue_t *vq)
857 	ASSERT(MUTEX_HELD(&vq->vq_lock));
859 	p = vdev_queue_class_to_issue(vq);
872 	tree = vdev_queue_class_tree(vq, p);
874 	search.io_offset = vq->vq_last_offset + 1;
881 	aio = vdev_queue_aggregate(vq, zio);
885 		vdev_queue_io_remove(vq, zio);
894 		mutex_exit(&vq->vq_lock);
897 		mutex_enter(&vq->vq_lock);
901 	vdev_queue_pending_add(vq, zio);
902 	vq->vq_last_offset = zio->io_offset;
910 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;
940 	mutex_enter(&vq->vq_lock);
942 	vdev_queue_io_add(vq, zio);
943 	nio = vdev_queue_io_to_issue(vq);
944 	mutex_exit(&vq->vq_lock);
960 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;
963 	mutex_enter(&vq->vq_lock);
965 	vdev_queue_pending_remove(vq, zio);
967 	vq->vq_io_complete_ts = gethrtime();
969 	while ((nio = vdev_queue_io_to_issue(vq)) != NULL) {
970 		mutex_exit(&vq->vq_lock);
977 		mutex_enter(&vq->vq_lock);
980 	mutex_exit(&vq->vq_lock);
986 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;
1013 	mutex_enter(&vq->vq_lock);
1022 	tree = vdev_queue_class_tree(vq, zio->io_priority);
1024 		avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
1026 		avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
1027 	} else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {
1031 	mutex_exit(&vq->vq_lock);