linux-master/lib/sbitmap.c

384 static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq,
399 	 * Each word can be limited to sbq->min_shallow_depth bits.
401 	shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth);
402 	depth = ((depth >> sbq->sb.shift) * shallow_depth +
403 		 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));
410 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
416 	ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node,
421 	sbq->min_shallow_depth = UINT_MAX;
422 	sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);
423 	atomic_set(&sbq->wake_index, 0);
424 	atomic_set(&sbq->ws_active, 0);
425 	atomic_set(&sbq->completion_cnt, 0);
426 	atomic_set(&sbq->wakeup_cnt, 0);
428 	sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node);
429 	if (!sbq->ws) {
430 		sbitmap_free(&sbq->sb);
435 		init_waitqueue_head(&sbq->ws[i].wait);
441 static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq,
446 	wake_batch = sbq_calc_wake_batch(sbq, depth);
447 	if (sbq->wake_batch != wake_batch)
448 		WRITE_ONCE(sbq->wake_batch, wake_batch);
451 void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
455 	unsigned int depth = (sbq->sb.depth + users - 1) / users;
460 	WRITE_ONCE(sbq->wake_batch, wake_batch);
464 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
466 	sbitmap_queue_update_wake_batch(sbq, depth);
467 	sbitmap_resize(&sbq->sb, depth);
471 int __sbitmap_queue_get(struct sbitmap_queue *sbq)
473 	return sbitmap_get(&sbq->sb);
477 unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
480 	struct sbitmap *sb = &sbq->sb;
529 int sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
532 	WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth);
534 	return sbitmap_get_shallow(&sbq->sb, shallow_depth);
538 void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
541 	sbq->min_shallow_depth = min_shallow_depth;
542 	sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);
546 static void __sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr)
550 	if (!atomic_read(&sbq->ws_active))
553 	wake_index = atomic_read(&sbq->wake_index);
555 		struct sbq_wait_state *ws = &sbq->ws[wake_index];
573 	if (wake_index != atomic_read(&sbq->wake_index))
574 		atomic_set(&sbq->wake_index, wake_index);
577 void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr)
579 	unsigned int wake_batch = READ_ONCE(sbq->wake_batch);
582 	if (!atomic_read(&sbq->ws_active))
585 	atomic_add(nr, &sbq->completion_cnt);
586 	wakeups = atomic_read(&sbq->wakeup_cnt);
589 		if (atomic_read(&sbq->completion_cnt) - wakeups < wake_batch)
591 	} while (!atomic_try_cmpxchg(&sbq->wakeup_cnt,
594 	__sbitmap_queue_wake_up(sbq, wake_batch);
604 void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset,
607 	struct sbitmap *sb = &sbq->sb;
633 	sbitmap_queue_wake_up(sbq, nr_tags);
634 	sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(),
638 void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
652 	sbitmap_deferred_clear_bit(&sbq->sb, nr);
661 	sbitmap_queue_wake_up(sbq, 1);
662 	sbitmap_update_cpu_hint(&sbq->sb, cpu, nr);
666 void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
675 	wake_index = atomic_read(&sbq->wake_index);
677 		struct sbq_wait_state *ws = &sbq->ws[wake_index];
687 void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
692 	sbitmap_show(&sbq->sb, m);
700 		seq_printf(m, "%u", *per_cpu_ptr(sbq->sb.alloc_hint, i));
704 	seq_printf(m, "wake_batch=%u\n", sbq->wake_batch);
705 	seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index));
706 	seq_printf(m, "ws_active=%d\n", atomic_read(&sbq->ws_active));
710 		struct sbq_wait_state *ws = &sbq->ws[i];
716 	seq_printf(m, "round_robin=%d\n", sbq->sb.round_robin);
717 	seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth);
721 void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
725 	if (!sbq_wait->sbq) {
726 		sbq_wait->sbq = sbq;
727 		atomic_inc(&sbq->ws_active);
736 	if (sbq_wait->sbq) {
737 		atomic_dec(&sbq_wait->sbq->ws_active);
738 		sbq_wait->sbq = NULL;
743 void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
747 	if (!sbq_wait->sbq) {
748 		atomic_inc(&sbq->ws_active);
749 		sbq_wait->sbq = sbq;
755 void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
759 	if (sbq_wait->sbq) {
760 		atomic_dec(&sbq->ws_active);
761 		sbq_wait->sbq = NULL;