linux-master/io_uring/io-wq.c

110 	struct io_wq_acct acct[IO_WQ_ACCT_NR];
138 					struct io_wq_acct *acct,
156 	return &wq->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];
188 	struct io_wq_acct *acct = io_wq_get_acct(worker);
191 	atomic_dec(&acct->nr_running);
193 	acct->nr_workers--;
244 static inline bool __io_acct_run_queue(struct io_wq_acct *acct)
246 	return !test_bit(IO_ACCT_STALLED_BIT, &acct->flags) &&
247 		!wq_list_empty(&acct->work_list);
251  * If there's work to do, returns true with acct->lock acquired. If not,
254 static inline bool io_acct_run_queue(struct io_wq_acct *acct)
255 	__acquires(&acct->lock)
257 	raw_spin_lock(&acct->lock);
258 	if (__io_acct_run_queue(acct))
261 	raw_spin_unlock(&acct->lock);
270 					struct io_wq_acct *acct)
284 		if (io_wq_get_acct(worker) != acct) {
305 static bool io_wq_create_worker(struct io_wq *wq, struct io_wq_acct *acct)
311 	if (unlikely(!acct->max_workers))
315 	if (acct->nr_workers >= acct->max_workers) {
319 	acct->nr_workers++;
321 	atomic_inc(&acct->nr_running);
323 	return create_io_worker(wq, acct->index);
328 	struct io_wq_acct *acct = io_wq_get_acct(worker);
330 	atomic_inc(&acct->nr_running);
338 	struct io_wq_acct *acct;
343 	acct = &wq->acct[worker->create_index];
346 	if (acct->nr_workers < acct->max_workers) {
347 		acct->nr_workers++;
354 		atomic_dec(&acct->nr_running);
362 				   struct io_wq_acct *acct,
384 	worker->create_index = acct->index;
402 	atomic_dec(&acct->nr_running);
409 	struct io_wq_acct *acct = io_wq_get_acct(worker);
415 	if (!atomic_dec_and_test(&acct->nr_running))
417 	if (!io_acct_run_queue(acct))
420 	raw_spin_unlock(&acct->lock);
421 	atomic_inc(&acct->nr_running);
423 	io_queue_worker_create(worker, acct, create_worker_cb);
474 static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
476 	__must_hold(acct->lock)
483 	wq_list_for_each(node, prev, &acct->work_list) {
490 			wq_list_del(&acct->work_list, node, prev);
501 			wq_list_cut(&acct->work_list, &tail->list, prev);
517 		set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
518 		raw_spin_unlock(&acct->lock);
520 		raw_spin_lock(&acct->lock);
522 			clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
545  * Called with acct->lock held, drops it before returning
547 static void io_worker_handle_work(struct io_wq_acct *acct,
549 	__releases(&acct->lock)
564 		work = io_get_next_work(acct, worker);
578 		raw_spin_unlock(&acct->lock);
614 				clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
621 		if (!__io_acct_run_queue(acct))
623 		raw_spin_lock(&acct->lock);
630 	struct io_wq_acct *acct = io_wq_get_acct(worker);
648 		 * the acct->lock held. If not, it will drop it.
650 		while (io_acct_run_queue(acct))
651 			io_worker_handle_work(acct, worker);
658 		if (last_timeout && (exit_mask || acct->nr_workers > 1)) {
659 			acct->nr_workers--;
684 	if (test_bit(IO_WQ_BIT_EXIT, &wq->state) && io_acct_run_queue(acct))
685 		io_worker_handle_work(acct, worker);
782 		struct io_wq_acct *acct = io_wq_get_acct(worker);
784 		atomic_dec(&acct->nr_running);
786 		acct->nr_workers--;
787 		if (!acct->nr_workers) {
794 			while (io_acct_cancel_pending_work(wq, acct, &match))
812 	struct io_wq_acct *acct = io_wq_get_acct(worker);
814 	if (!io_queue_worker_create(worker, acct, create_worker_cont))
820 	struct io_wq_acct *acct = &wq->acct[index];
829 		atomic_dec(&acct->nr_running);
831 		acct->nr_workers--;
902 	struct io_wq_acct *acct = io_work_get_acct(wq, work);
908 		wq_list_add_tail(&work->list, &acct->work_list);
918 	wq_list_add_after(&work->list, &tail->list, &acct->work_list);
928 	struct io_wq_acct *acct = io_work_get_acct(wq, work);
943 	raw_spin_lock(&acct->lock);
945 	clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
946 	raw_spin_unlock(&acct->lock);
949 	do_create = !io_wq_activate_free_worker(wq, acct);
953 	    !atomic_read(&acct->nr_running))) {
956 		did_create = io_wq_create_worker(wq, acct);
961 		if (acct->nr_workers) {
972 		io_acct_cancel_pending_work(wq, acct, &match);
1021 	struct io_wq_acct *acct = io_work_get_acct(wq, work);
1033 	wq_list_del(&acct->work_list, &work->list, prev);
1037 					struct io_wq_acct *acct,
1043 	raw_spin_lock(&acct->lock);
1044 	wq_list_for_each(node, prev, &acct->work_list) {
1049 		raw_spin_unlock(&acct->lock);
1055 	raw_spin_unlock(&acct->lock);
1066 		struct io_wq_acct *acct = io_get_acct(wq, i == 0);
1068 		if (io_acct_cancel_pending_work(wq, acct, match)) {
1133 		struct io_wq_acct *acct = &wq->acct[i];
1135 		if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))
1136 			io_wq_activate_free_worker(wq, acct);
1166 	wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
1167 	wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =
1172 		struct io_wq_acct *acct = &wq->acct[i];
1174 		acct->index = i;
1175 		atomic_set(&acct->nr_running, 0);
1176 		INIT_WQ_LIST(&acct->work_list);
1177 		raw_spin_lock_init(&acct->lock);
1339 	struct io_wq_acct *acct;
1359 		acct = &wq->acct[i];
1360 		prev[i] = max_t(int, acct->max_workers, prev[i]);
1362 			acct->max_workers = new_count[i];