fs/btrfs/discard.c

70 static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl,
73 	return &discard_ctl->discard_list[block_group->discard_index];
79  * @discard_ctl: discard control
83 static bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl)
85 	struct btrfs_fs_info *fs_info = container_of(discard_ctl,
87 						     discard_ctl);
93 static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,
96 	lockdep_assert_held(&discard_ctl->lock);
97 	if (!btrfs_run_discard_work(discard_ctl))
112 		       get_discard_list(discard_ctl, block_group));
115 static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,
121 	spin_lock(&discard_ctl->lock);
122 	__add_to_discard_list(discard_ctl, block_group);
123 	spin_unlock(&discard_ctl->lock);
126 static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl,
131 	spin_lock(&discard_ctl->lock);
135 	if (!btrfs_run_discard_work(discard_ctl)) {
136 		spin_unlock(&discard_ctl->lock);
149 		      &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]);
151 	spin_unlock(&discard_ctl->lock);
154 static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl,
160 	spin_lock(&discard_ctl->lock);
162 	if (block_group == discard_ctl->block_group) {
164 		discard_ctl->block_group = NULL;
179 	spin_unlock(&discard_ctl->lock);
187  * @discard_ctl:  discard control
194 					struct btrfs_discard_ctl *discard_ctl,
201 		struct list_head *discard_list = &discard_ctl->discard_list[i];
226  * @discard_ctl:   discard control
238 					struct btrfs_discard_ctl *discard_ctl,
244 	spin_lock(&discard_ctl->lock);
246 	block_group = find_next_block_group(discard_ctl, now);
252 				__add_to_discard_list(discard_ctl, block_group);
263 		discard_ctl->block_group = block_group;
269 	spin_unlock(&discard_ctl->lock);
288 	struct btrfs_discard_ctl *discard_ctl;
294 	discard_ctl = &block_group->fs_info->discard_ctl;
300 		remove_from_discard_list(discard_ctl, block_group);
306 				add_to_discard_list(discard_ctl, block_group);
316  * @discard_ctl: discard control
322 static void btrfs_update_discard_index(struct btrfs_discard_ctl *discard_ctl,
331 	add_to_discard_list(discard_ctl, block_group);
337  * @discard_ctl: discard control
343 void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl,
346 	if (remove_from_discard_list(discard_ctl, block_group)) {
347 		cancel_delayed_work_sync(&discard_ctl->work);
348 		btrfs_discard_schedule_work(discard_ctl, true);
355  * @discard_ctl: discard control
360 void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl,
367 		add_to_discard_unused_list(discard_ctl, block_group);
369 		add_to_discard_list(discard_ctl, block_group);
371 	if (!delayed_work_pending(&discard_ctl->work))
372 		btrfs_discard_schedule_work(discard_ctl, false);
375 static void __btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
380 	if (!btrfs_run_discard_work(discard_ctl))
382 	if (!override && delayed_work_pending(&discard_ctl->work))
385 	block_group = find_next_block_group(discard_ctl, now);
387 		u64 delay = discard_ctl->delay_ms * NSEC_PER_MSEC;
388 		u32 kbps_limit = READ_ONCE(discard_ctl->kbps_limit);
395 		if (kbps_limit && discard_ctl->prev_discard) {
397 			u64 bps_delay = div64_u64(discard_ctl->prev_discard *
413 		if (override && discard_ctl->prev_discard) {
414 			u64 elapsed = now - discard_ctl->prev_discard_time;
422 		mod_delayed_work(discard_ctl->discard_workers,
423 				 &discard_ctl->work, nsecs_to_jiffies(delay));
430  * @discard_ctl:  discard control
437 void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
442 	spin_lock(&discard_ctl->lock);
443 	__btrfs_discard_schedule_work(discard_ctl, now, override);
444 	spin_unlock(&discard_ctl->lock);
450  * @discard_ctl: discard control
458 static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl,
461 	remove_from_discard_list(discard_ctl, block_group);
467 			add_to_discard_unused_list(discard_ctl, block_group);
469 		btrfs_update_discard_index(discard_ctl, block_group);
484 	struct btrfs_discard_ctl *discard_ctl;
492 	discard_ctl = container_of(work, struct btrfs_discard_ctl, work.work);
494 	block_group = peek_discard_list(discard_ctl, &discard_state,
496 	if (!block_group || !btrfs_run_discard_work(discard_ctl))
499 		btrfs_discard_schedule_work(discard_ctl, false);
522 		discard_ctl->discard_bitmap_bytes += trimmed;
528 		discard_ctl->discard_extent_bytes += trimmed;
534 			btrfs_finish_discard_pass(discard_ctl, block_group);
537 			spin_lock(&discard_ctl->lock);
542 			spin_unlock(&discard_ctl->lock);
547 	spin_lock(&discard_ctl->lock);
548 	discard_ctl->prev_discard = trimmed;
549 	discard_ctl->prev_discard_time = now;
553 	 * still owned that reference. But we set the discard_ctl->block_group
557 	if (discard_ctl->block_group == NULL)
559 	discard_ctl->block_group = NULL;
560 	__btrfs_discard_schedule_work(discard_ctl, now, false);
561 	spin_unlock(&discard_ctl->lock);
567  * @discard_ctl: discard control
573 void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)
581 	discardable_extents = atomic_read(&discard_ctl->discardable_extents);
585 	spin_lock(&discard_ctl->lock);
596 			   &discard_ctl->discardable_extents);
598 	discardable_bytes = atomic64_read(&discard_ctl->discardable_bytes);
601 			     &discard_ctl->discardable_bytes);
604 		spin_unlock(&discard_ctl->lock);
608 	iops_limit = READ_ONCE(discard_ctl->iops_limit);
622 	discard_ctl->delay_ms = delay;
624 	spin_unlock(&discard_ctl->lock);
632  * Propagate deltas of counters up to the discard_ctl.  It maintains a current
639 	struct btrfs_discard_ctl *discard_ctl;
649 	discard_ctl = &block_group->fs_info->discard_ctl;
655 		atomic_add(extents_delta, &discard_ctl->discardable_extents);
663 		atomic64_add(bytes_delta, &discard_ctl->discardable_bytes);
689 		btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);
702  * @discard_ctl: discard control
710 static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl)
715 	spin_lock(&discard_ctl->lock);
718 					 &discard_ctl->discard_list[i],
721 			spin_unlock(&discard_ctl->lock);
724 			spin_lock(&discard_ctl->lock);
728 	spin_unlock(&discard_ctl->lock);
750 	struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl;
753 	spin_lock_init(&discard_ctl->lock);
754 	INIT_DELAYED_WORK(&discard_ctl->work, btrfs_discard_workfn);
757 		INIT_LIST_HEAD(&discard_ctl->discard_list[i]);
759 	discard_ctl->prev_discard = 0;
760 	discard_ctl->prev_discard_time = 0;
761 	atomic_set(&discard_ctl->discardable_extents, 0);
762 	atomic64_set(&discard_ctl->discardable_bytes, 0);
763 	discard_ctl->max_discard_size = BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE;
764 	discard_ctl->delay_ms = BTRFS_DISCARD_MAX_DELAY_MSEC;
765 	discard_ctl->iops_limit = BTRFS_DISCARD_MAX_IOPS;
766 	discard_ctl->kbps_limit = 0;
767 	discard_ctl->discard_extent_bytes = 0;
768 	discard_ctl->discard_bitmap_bytes = 0;
769 	atomic64_set(&discard_ctl->discard_bytes_saved, 0);
775 	cancel_delayed_work_sync(&fs_info->discard_ctl.work);
776 	btrfs_discard_purge_list(&fs_info->discard_ctl);