Lines Matching refs:vq
253 vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)
255 return (&vq->vq_class[p].vqc_queued_tree);
259 vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t)
263 return (&vq->vq_read_offset_tree);
265 return (&vq->vq_write_offset_tree);
267 return (&vq->vq_trim_offset_tree);
285 vdev_queue_class_min_active(vdev_queue_t *vq, zio_priority_t p)
297 return (vq->vq_ia_active == 0 ? zfs_vdev_scrub_min_active :
298 MIN(vq->vq_nia_credit, zfs_vdev_scrub_min_active));
300 return (vq->vq_ia_active == 0 ? zfs_vdev_removal_min_active :
301 MIN(vq->vq_nia_credit, zfs_vdev_removal_min_active));
303 return (vq->vq_ia_active == 0 ?zfs_vdev_initializing_min_active:
304 MIN(vq->vq_nia_credit, zfs_vdev_initializing_min_active));
308 return (vq->vq_ia_active == 0 ? zfs_vdev_rebuild_min_active :
309 MIN(vq->vq_nia_credit, zfs_vdev_rebuild_min_active));
363 vdev_queue_class_max_active(spa_t *spa, vdev_queue_t *vq, zio_priority_t p)
375 if (vq->vq_ia_active > 0) {
376 return (MIN(vq->vq_nia_credit,
378 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay)
382 if (vq->vq_ia_active > 0) {
383 return (MIN(vq->vq_nia_credit,
385 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay)
389 if (vq->vq_ia_active > 0) {
390 return (MIN(vq->vq_nia_credit,
392 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay)
398 if (vq->vq_ia_active > 0) {
399 return (MIN(vq->vq_nia_credit,
401 } else if (vq->vq_nia_credit < zfs_vdev_nia_delay)
415 vdev_queue_class_to_issue(vdev_queue_t *vq)
417 spa_t *spa = vq->vq_vdev->vdev_spa;
420 if (avl_numnodes(&vq->vq_active_tree) >= zfs_vdev_max_active)
429 p = (vq->vq_last_prio + n + 1) % ZIO_PRIORITY_NUM_QUEUEABLE;
430 if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
431 vq->vq_class[p].vqc_active <
432 vdev_queue_class_min_active(vq, p)) {
433 vq->vq_last_prio = p;
443 if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
444 vq->vq_class[p].vqc_active <
445 vdev_queue_class_max_active(spa, vq, p)) {
446 vq->vq_last_prio = p;
458 vdev_queue_t *vq = &vd->vdev_queue;
461 mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL);
462 vq->vq_vdev = vd;
465 avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,
467 avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ),
470 avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE),
473 avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_TRIM),
492 avl_create(vdev_queue_class_tree(vq, p), compfn,
496 vq->vq_last_offset = 0;
502 vdev_queue_t *vq = &vd->vdev_queue;
505 avl_destroy(vdev_queue_class_tree(vq, p));
506 avl_destroy(&vq->vq_active_tree);
507 avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ));
508 avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE));
509 avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_TRIM));
511 mutex_destroy(&vq->vq_lock);
515 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)
518 avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
519 avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);
523 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)
526 avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
527 avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);
545 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)
547 ASSERT(MUTEX_HELD(&vq->vq_lock));
549 vq->vq_class[zio->io_priority].vqc_active++;
551 if (++vq->vq_ia_active == 1)
552 vq->vq_nia_credit = 1;
553 } else if (vq->vq_ia_active > 0) {
554 vq->vq_nia_credit--;
556 avl_add(&vq->vq_active_tree, zio);
560 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)
562 ASSERT(MUTEX_HELD(&vq->vq_lock));
564 vq->vq_class[zio->io_priority].vqc_active--;
566 if (--vq->vq_ia_active == 0)
567 vq->vq_nia_credit = 0;
569 vq->vq_nia_credit = zfs_vdev_nia_credit;
570 } else if (vq->vq_ia_active == 0)
571 vq->vq_nia_credit++;
572 avl_remove(&vq->vq_active_tree, zio);
599 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
608 avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);
613 maxblocksize = spa_maxblocksize(vq->vq_vdev->vdev_spa);
614 if (vq->vq_vdev->vdev_nonrot)
635 ASSERT(vq->vq_vdev->vdev_ops != &vdev_draid_spare_ops);
761 vdev_queue_io_remove(vq, dio);
804 mutex_exit(&vq->vq_lock);
811 mutex_enter(&vq->vq_lock);
817 vdev_queue_io_to_issue(vdev_queue_t *vq)
825 ASSERT(MUTEX_HELD(&vq->vq_lock));
827 p = vdev_queue_class_to_issue(vq);
840 tree = vdev_queue_class_tree(vq, p);
841 vq->vq_io_search.io_timestamp = 0;
842 vq->vq_io_search.io_offset = vq->vq_last_offset - 1;
843 VERIFY3P(avl_find(tree, &vq->vq_io_search, &idx), ==, NULL);
849 aio = vdev_queue_aggregate(vq, zio);
853 vdev_queue_io_remove(vq, zio);
862 mutex_exit(&vq->vq_lock);
865 mutex_enter(&vq->vq_lock);
869 vdev_queue_pending_add(vq, zio);
870 vq->vq_last_offset = zio->io_offset + zio->io_size;
878 vdev_queue_t *vq = &zio->io_vd->vdev_queue;
916 mutex_enter(&vq->vq_lock);
918 vdev_queue_io_add(vq, zio);
919 nio = vdev_queue_io_to_issue(vq);
920 mutex_exit(&vq->vq_lock);
936 vdev_queue_t *vq = &zio->io_vd->vdev_queue;
939 mutex_enter(&vq->vq_lock);
941 vdev_queue_pending_remove(vq, zio);
944 vq->vq_io_complete_ts = gethrtime();
945 vq->vq_io_delta_ts = vq->vq_io_complete_ts - zio->io_timestamp;
947 while ((nio = vdev_queue_io_to_issue(vq)) != NULL) {
948 mutex_exit(&vq->vq_lock);
955 mutex_enter(&vq->vq_lock);
958 mutex_exit(&vq->vq_lock);
964 vdev_queue_t *vq = &zio->io_vd->vdev_queue;
991 mutex_enter(&vq->vq_lock);
1000 tree = vdev_queue_class_tree(vq, zio->io_priority);
1002 avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
1004 avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
1005 } else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {
1009 mutex_exit(&vq->vq_lock);