md/dm-vdo/flush.c

23 struct flusher {
25 	/* The vdo to which this flusher belongs */
27 	/* The administrative state of the flusher */
56  * assert_on_flusher_thread() - Check that we are on the flusher thread.
57  * @flusher: The flusher.
60 static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller)
62 	VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id),
63 			    "%s() called from flusher thread", caller);
67  * as_flusher() - Convert a generic vdo_completion to a flusher.
70  * Return: The completion as a flusher.
72 static struct flusher *as_flusher(struct vdo_completion *completion)
75 	return container_of(completion, struct flusher, completion);
115 		struct flusher *flusher = pool_data;
117 		vdo_initialize_completion(&flush->completion, flusher->vdo,
130  * vdo_make_flusher() - Make a flusher for a vdo.
131  * @vdo: The vdo which owns the flusher.
137 	int result = vdo_allocate(1, struct flusher, __func__, &vdo->flusher);
142 	vdo->flusher->vdo = vdo;
143 	vdo->flusher->thread_id = vdo->thread_config.packer_thread;
144 	vdo_set_admin_state_code(&vdo->flusher->state, VDO_ADMIN_STATE_NORMAL_OPERATION);
145 	vdo_initialize_completion(&vdo->flusher->completion, vdo,
148 	spin_lock_init(&vdo->flusher->lock);
149 	bio_list_init(&vdo->flusher->waiting_flush_bios);
150 	vdo->flusher->flush_pool = mempool_create(1, allocate_flush, free_flush,
151 						  vdo->flusher);
152 	return ((vdo->flusher->flush_pool == NULL) ? -ENOMEM : VDO_SUCCESS);
156  * vdo_free_flusher() - Free a flusher.
157  * @flusher: The flusher to free.
159 void vdo_free_flusher(struct flusher *flusher)
161 	if (flusher == NULL)
164 	if (flusher->flush_pool != NULL)
165 		mempool_destroy(vdo_forget(flusher->flush_pool));
166 	vdo_free(flusher);
170  * vdo_get_flusher_thread_id() - Get the ID of the thread on which flusher functions should be
172  * @flusher: The flusher to query.
174  * Return: The ID of the thread which handles the flusher.
176 thread_id_t vdo_get_flusher_thread_id(struct flusher *flusher)
178 	return flusher->thread_id;
181 static void notify_flush(struct flusher *flusher);
186  * @completion: The flusher completion.
194 	struct flusher *flusher = as_flusher(completion);
196 	assert_on_flusher_thread(flusher, __func__);
198 	vdo_waitq_enqueue_waiter(&flusher->pending_flushes,
199 				 vdo_waitq_dequeue_waiter(&flusher->notifiers));
200 	vdo_complete_flushes(flusher);
201 	if (vdo_waitq_has_waiters(&flusher->notifiers))
202 		notify_flush(flusher);
207  * @completion: The flusher completion.
214 	struct flusher *flusher = as_flusher(completion);
216 	vdo_increment_packer_flush_generation(flusher->vdo->packer);
218 				       flusher->thread_id);
223  * @completion: The flusher as a completion.
230 	struct flusher *flusher = as_flusher(completion);
231 	struct logical_zone *zone = flusher->logical_zone_to_notify;
233 	vdo_increment_logical_zone_flush_generation(zone, flusher->notify_generation);
236 					       flusher->thread_id);
240 	flusher->logical_zone_to_notify = zone->next;
242 				       flusher->logical_zone_to_notify->thread_id);
247  * @flusher: The flusher doing the notification.
249 static void notify_flush(struct flusher *flusher)
252 		vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->notifiers));
254 	flusher->notify_generation = flush->flush_generation;
255 	flusher->logical_zone_to_notify = &flusher->vdo->logical_zones->zones[0];
256 	flusher->completion.requeue = true;
257 	vdo_launch_completion_callback(&flusher->completion, increment_generation,
258 				       flusher->logical_zone_to_notify->thread_id);
270 	struct flusher *flusher = completion->vdo->flusher;
274 	assert_on_flusher_thread(flusher, __func__);
275 	result = VDO_ASSERT(vdo_is_state_normal(&flusher->state),
276 			    "flusher is in normal operation");
278 		vdo_enter_read_only_mode(flusher->vdo, result);
283 	flush->flush_generation = flusher->flush_generation++;
284 	may_notify = !vdo_waitq_has_waiters(&flusher->notifiers);
285 	vdo_waitq_enqueue_waiter(&flusher->notifiers, &flush->waiter);
287 		notify_flush(flusher);
291  * check_for_drain_complete() - Check whether the flusher has drained.
292  * @flusher: The flusher.
294 static void check_for_drain_complete(struct flusher *flusher)
298 	if (!vdo_is_state_draining(&flusher->state) ||
299 	    vdo_waitq_has_waiters(&flusher->pending_flushes))
302 	spin_lock(&flusher->lock);
303 	drained = bio_list_empty(&flusher->waiting_flush_bios);
304 	spin_unlock(&flusher->lock);
307 		vdo_finish_draining(&flusher->state);
312  * @flusher: The flusher.
314 void vdo_complete_flushes(struct flusher *flusher)
319 	assert_on_flusher_thread(flusher, __func__);
321 	for (zone = &flusher->vdo->logical_zones->zones[0]; zone != NULL; zone = zone->next)
326 	while (vdo_waitq_has_waiters(&flusher->pending_flushes)) {
328 			vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->pending_flushes));
334 				     flusher->first_unacknowledged_generation),
336 				    (unsigned long long) flusher->first_unacknowledged_generation,
338 		vdo_waitq_dequeue_waiter(&flusher->pending_flushes);
340 		flusher->first_unacknowledged_generation++;
343 	check_for_drain_complete(flusher);
347  * vdo_dump_flusher() - Dump the flusher, in a thread-unsafe fashion.
348  * @flusher: The flusher.
350 void vdo_dump_flusher(const struct flusher *flusher)
352 	vdo_log_info("struct flusher");
354 		     (unsigned long long) flusher->flush_generation,
355 		     (unsigned long long) flusher->first_unacknowledged_generation);
357 		     (vdo_waitq_has_waiters(&flusher->notifiers) ? "not empty" : "empty"),
358 		     (vdo_waitq_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty"));
366  * Initializes a vdo_flush structure, transferring all the bios in the flusher's waiting_flush_bios
372 	bio_list_merge_init(&flush->bios, &vdo->flusher->waiting_flush_bios);
398 	struct vdo_flush *flush = mempool_alloc(vdo->flusher->flush_pool, GFP_NOWAIT);
399 	struct flusher *flusher = vdo->flusher;
400 	const struct admin_state_code *code = vdo_get_admin_state_code(&flusher->state);
405 	spin_lock(&flusher->lock);
408 	bio_list_add(&flusher->waiting_flush_bios, bio);
411 		spin_unlock(&flusher->lock);
417 	spin_unlock(&flusher->lock);
434 	struct flusher *flusher = flush->completion.vdo->flusher;
436 	spin_lock(&flusher->lock);
437 	if (bio_list_empty(&flusher->waiting_flush_bios)) {
441 		initialize_flush(flush, flusher->vdo);
444 	spin_unlock(&flusher->lock);
452 	mempool_free(flush, flusher->flush_pool);
489  * @flusher: The flusher finishing the request.
491 static thread_id_t select_bio_queue(struct flusher *flusher)
493 	struct vdo *vdo = flusher->vdo;
494 	zone_count_t bio_threads = flusher->vdo->thread_config.bio_thread_count;
501 	if (flusher->flush_count == interval) {
502 		flusher->flush_count = 1;
503 		flusher->bio_queue_rotor = ((flusher->bio_queue_rotor + 1) % bio_threads);
505 		flusher->flush_count++;
508 	return vdo->thread_config.bio_threads[flusher->bio_queue_rotor];
521 			       select_bio_queue(completion->vdo->flusher), NULL);
532 	check_for_drain_complete(container_of(state, struct flusher, state));
536  * vdo_drain_flusher() - Drain the flusher.
537  * @flusher: The flusher to drain.
538  * @completion: The completion to finish when the flusher has drained.
540  * Drains the flusher by preventing any more VIOs from entering the flusher and then flushing. The
541  * flusher will be left in the suspended state.
543 void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion)
545 	assert_on_flusher_thread(flusher, __func__);
546 	vdo_start_draining(&flusher->state, VDO_ADMIN_STATE_SUSPENDING, completion,
551  * vdo_resume_flusher() - Resume a flusher which has been suspended.
552  * @flusher: The flusher to resume.
553  * @parent: The completion to finish when the flusher has resumed.
555 void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent)
557 	assert_on_flusher_thread(flusher, __func__);
558 	vdo_continue_completion(parent, vdo_resume_if_quiescent(&flusher->state));