fs/bcachefs/btree_cache.c

43 static inline unsigned btree_cache_can_free(struct btree_cache *bc)
45 	return max_t(int, 0, bc->used - bc->reserve);
48 static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b)
51 		list_move(&b->list, &bc->freed_pcpu);
53 		list_move(&b->list, &bc->freed_nonpcpu);
58 	struct btree_cache *bc = &c->btree_cache;
73 	bc->used--;
75 	btree_node_to_freedlist(bc, b);
136 	struct btree_cache *bc = &c->btree_cache;
150 	bc->used++;
151 	list_add(&b->list, &bc->freeable);
157 void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b)
159 	int ret = rhashtable_remove_fast(&bc->table, &b->hash, bch_btree_cache_params);
167 int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b)
172 	return rhashtable_lookup_insert_fast(&bc->table, &b->hash,
176 int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b,
184 	mutex_lock(&bc->lock);
185 	ret = __bch2_btree_node_hash_insert(bc, b);
187 		list_add_tail(&b->list, &bc->live);
188 	mutex_unlock(&bc->lock);
194 static inline struct btree *btree_cache_find(struct btree_cache *bc,
199 	return rhashtable_lookup_fast(&bc->table, &v, bch_btree_cache_params);
208 	struct btree_cache *bc = &c->btree_cache;
211 	lockdep_assert_held(&bc->lock);
216 		? bc->pinned_nodes_interior_mask
217 		: bc->pinned_nodes_leaf_mask;
220 	    bbpos_cmp(bc->pinned_nodes_start, pos) < 0 &&
221 	    bbpos_cmp(bc->pinned_nodes_end, pos) >= 0)
303 	struct btree_cache *bc = &c->btree_cache;
311 	bool trigger_writes = atomic_read(&bc->dirty) + nr >=
312 		bc->used * 3 / 4;
317 	mutex_lock(&bc->lock);
327 	can_free = btree_cache_can_free(bc);
331 	list_for_each_entry_safe(b, t, &bc->freeable, list) {
352 	list_for_each_entry_safe(b, t, &bc->live, list) {
361 			bch2_btree_node_hash_remove(bc, b);
372 			list_move(&bc->live, &b->list);
373 			mutex_unlock(&bc->lock);
378 			mutex_lock(&bc->lock);
386 	if (&t->list != &bc->live)
387 		list_move_tail(&bc->live, &t->list);
389 	mutex_unlock(&bc->lock);
401 	struct btree_cache *bc = &c->btree_cache;
406 	return btree_cache_can_free(bc);
411 	struct btree_cache *bc = &c->btree_cache;
415 	shrinker_free(bc->shrink);
419 	mutex_lock(&bc->lock);
422 		list_move(&c->verify_data->list, &bc->live);
430 			list_add(&r->b->list, &bc->live);
433 	list_splice(&bc->freeable, &bc->live);
435 	while (!list_empty(&bc->live)) {
436 		b = list_first_entry(&bc->live, struct btree, list);
447 	list_splice(&bc->freed_pcpu, &bc->freed_nonpcpu);
449 	while (!list_empty(&bc->freed_nonpcpu)) {
450 		b = list_first_entry(&bc->freed_nonpcpu, struct btree, list);
456 	mutex_unlock(&bc->lock);
459 	if (bc->table_init_done)
460 		rhashtable_destroy(&bc->table);
465 	struct btree_cache *bc = &c->btree_cache;
470 	ret = rhashtable_init(&bc->table, &bch_btree_cache_params);
474 	bc->table_init_done = true;
478 	for (i = 0; i < bc->reserve; i++)
482 	list_splice_init(&bc->live, &bc->freeable);
489 	bc->shrink = shrink;
501 void bch2_fs_btree_cache_init_early(struct btree_cache *bc)
503 	mutex_init(&bc->lock);
504 	INIT_LIST_HEAD(&bc->live);
505 	INIT_LIST_HEAD(&bc->freeable);
506 	INIT_LIST_HEAD(&bc->freed_pcpu);
507 	INIT_LIST_HEAD(&bc->freed_nonpcpu);
519 	struct btree_cache *bc = &c->btree_cache;
521 	if (bc->alloc_lock == current) {
523 		bc->alloc_lock = NULL;
524 		closure_wake_up(&bc->alloc_wait);
531 	struct btree_cache *bc = &c->btree_cache;
534 	old = cmpxchg(&bc->alloc_lock, NULL, current);
543 	closure_wait(&bc->alloc_wait, cl);
546 	old = cmpxchg(&bc->alloc_lock, NULL, current);
549 		closure_wake_up(&bc->alloc_wait);
563 	struct btree_cache *bc = &c->btree_cache;
566 	list_for_each_entry_reverse(b, &bc->live, list)
571 		list_for_each_entry_reverse(b, &bc->live, list)
587 	struct btree_cache *bc = &c->btree_cache;
589 		? &bc->freed_pcpu
590 		: &bc->freed_nonpcpu;
596 	mutex_lock(&bc->lock);
610 		mutex_unlock(&bc->lock);
615 		mutex_lock(&bc->lock);
628 	list_for_each_entry(b2, &bc->freeable, list)
632 			btree_node_to_freedlist(bc, b2);
638 	mutex_unlock(&bc->lock);
646 	mutex_lock(&bc->lock);
647 	bc->used++;
649 	mutex_unlock(&bc->lock);
670 	mutex_lock(&bc->lock);
673 	if (bc->alloc_lock == current) {
676 		bch2_btree_node_hash_remove(bc, b2);
681 			btree_node_to_freedlist(bc, b2);
689 		mutex_unlock(&bc->lock);
695 	mutex_unlock(&bc->lock);
710 	struct btree_cache *bc = &c->btree_cache;
761 	if (bch2_btree_node_hash_insert(bc, b, level, btree_id)) {
767 		mutex_lock(&bc->lock);
768 		list_add(&b->list, &bc->freeable);
769 		mutex_unlock(&bc->lock);
847 	struct btree_cache *bc = &c->btree_cache;
855 	b = btree_cache_find(bc, k);
1044 	struct btree_cache *bc = &c->btree_cache;
1057 	b = btree_cache_find(bc, k);
1128 	struct btree_cache *bc = &c->btree_cache;
1133 	struct btree *b = btree_cache_find(bc, k);
1147 	struct btree_cache *bc = &c->btree_cache;
1150 	b = btree_cache_find(bc, k);
1178 	mutex_lock(&bc->lock);
1180 	bch2_btree_node_hash_remove(bc, b);
1181 	mutex_unlock(&bc->lock);