Lines Matching refs:trans
121 static void bch2_btree_path_verify_cached(struct btree_trans *trans,
127 if (!bch2_btree_node_relock(trans, path, 0))
135 btree_node_unlock(trans, path, 0);
138 static void bch2_btree_path_verify_level(struct btree_trans *trans,
159 bch2_btree_path_verify_cached(trans, path);
166 if (!bch2_btree_node_relock_notrace(trans, path, level))
192 btree_node_unlock(trans, path, level);
220 static void bch2_btree_path_verify(struct btree_trans *trans,
223 struct bch_fs *c = trans->c;
235 bch2_btree_path_verify_level(trans, path, i);
241 void bch2_trans_verify_paths(struct btree_trans *trans)
246 trans_for_each_path(trans, path, iter)
247 bch2_btree_path_verify(trans, path);
252 struct btree_trans *trans = iter->trans;
256 BUG_ON(!!(iter->flags & BTREE_ITER_CACHED) != btree_iter_path(trans, iter)->cached);
266 bch2_btree_path_verify(trans, &trans->paths[iter->update_path]);
267 bch2_btree_path_verify(trans, btree_iter_path(trans, iter));
284 struct btree_trans *trans = iter->trans;
298 BUG_ON(!bch2_snapshot_is_ancestor(trans->c,
302 bch2_trans_iter_init(trans, ©, iter->btree_id, iter->pos,
314 bch2_snapshot_is_ancestor(trans->c, iter->snapshot,
328 bch2_trans_iter_exit(trans, ©);
332 void bch2_assert_pos_locked(struct btree_trans *trans, enum btree_id id,
339 btree_trans_sort_paths(trans);
341 trans_for_each_path_inorder(trans, path, iter) {
364 bch2_dump_trans_paths_updates(trans);
374 static inline void bch2_btree_path_verify_level(struct btree_trans *trans,
376 static inline void bch2_btree_path_verify(struct btree_trans *trans,
416 void bch2_btree_path_fix_key_modified(struct btree_trans *trans,
423 trans_for_each_path_with_node(trans, b, path, i) {
425 bch2_btree_path_verify_level(trans, path, b->c.level);
520 void bch2_btree_node_iter_fix(struct btree_trans *trans,
540 trans_for_each_path_with_node(trans, b, linked, i) {
544 bch2_btree_path_verify_level(trans, linked, b->c.level);
575 static inline struct bkey_s_c btree_path_level_peek(struct btree_trans *trans,
580 struct bkey_s_c k = __btree_iter_unpack(trans->c, l, u,
584 trans->paths_sorted = false;
585 bch2_btree_path_verify_level(trans, path, l - path->l);
589 static inline struct bkey_s_c btree_path_level_prev(struct btree_trans *trans,
594 struct bkey_s_c k = __btree_iter_unpack(trans->c, l, u,
598 trans->paths_sorted = false;
599 bch2_btree_path_verify_level(trans, path, l - path->l);
637 void bch2_btree_path_level_init(struct btree_trans *trans,
652 static void bch2_trans_revalidate_updates_in_node(struct btree_trans *trans, struct btree *b)
654 struct bch_fs *c = trans->c;
656 trans_for_each_update(trans, i)
662 i->old_v = bch2_btree_path_peek_slot(trans->paths + i->path, &i->old_k).v;
664 if (unlikely(trans->journal_replay_not_finished)) {
681 void bch2_trans_node_add(struct btree_trans *trans,
689 while ((prev = prev_btree_path(trans, path)) &&
695 path = next_btree_path(trans, path))
701 btree_node_unlock(trans, path, b->c.level);
703 mark_btree_node_locked(trans, path, b->c.level, t);
706 bch2_btree_path_level_init(trans, path, b);
709 bch2_trans_revalidate_updates_in_node(trans, b);
716 void bch2_trans_node_reinit_iter(struct btree_trans *trans, struct btree *b)
721 trans_for_each_path_with_node(trans, b, path, i)
724 bch2_trans_revalidate_updates_in_node(trans, b);
729 static inline int btree_path_lock_root(struct btree_trans *trans,
734 struct bch_fs *c = trans->c;
760 ret = btree_node_lock(trans, path, &b->c,
779 mark_btree_node_locked(trans, path, path->level,
781 bch2_btree_path_level_init(trans, path, b);
790 static int btree_path_prefetch(struct btree_trans *trans, struct btree_path *path)
792 struct bch_fs *c = trans->c;
806 if (!bch2_btree_node_relock(trans, path, path->level))
815 ret = bch2_btree_node_prefetch(trans, path, tmp.k, path->btree_id,
820 btree_node_unlock(trans, path, path->level);
826 static int btree_path_prefetch_j(struct btree_trans *trans, struct btree_path *path,
829 struct bch_fs *c = trans->c;
841 if (!bch2_btree_node_relock(trans, path, path->level))
850 ret = bch2_btree_node_prefetch(trans, path, tmp.k, path->btree_id,
855 btree_node_unlock(trans, path, path->level);
861 static noinline void btree_node_mem_ptr_set(struct btree_trans *trans,
870 if (!bch2_btree_node_relock(trans, path, plevel))
880 btree_node_unlock(trans, path, plevel);
883 static noinline int btree_node_iter_and_journal_peek(struct btree_trans *trans,
888 struct bch_fs *c = trans->c;
894 __bch2_btree_and_journal_iter_init_node_iter(trans, &jiter, l->b, l->iter, path->pos);
902 ret = btree_path_prefetch_j(trans, path, &jiter);
908 static __always_inline int btree_path_down(struct btree_trans *trans,
913 struct bch_fs *c = trans->c;
925 if (unlikely(trans->journal_replay_not_finished)) {
926 ret = btree_node_iter_and_journal_peek(trans, path, flags, &tmp);
949 ret = btree_path_prefetch(trans, path);
955 b = bch2_btree_node_get(trans, path, tmp.k, level, lock_type, trace_ip);
960 if (likely(!trans->journal_replay_not_finished &&
963 btree_node_mem_ptr_set(trans, path, level + 1, b);
966 btree_node_unlock(trans, path, level + 1);
968 mark_btree_node_locked(trans, path, level,
971 bch2_btree_path_level_init(trans, path, b);
979 static int bch2_btree_path_traverse_all(struct btree_trans *trans)
981 struct bch_fs *c = trans->c;
987 if (trans->in_traverse_all)
990 trans->in_traverse_all = true;
992 trans->restarted = 0;
993 trans->last_restarted_ip = 0;
995 trans_for_each_path(trans, path, i)
998 btree_trans_sort_paths(trans);
1000 bch2_trans_unlock(trans);
1003 if (unlikely(trans->memory_allocation_failure)) {
1009 ret = bch2_btree_cache_cannibalize_lock(trans, &cl);
1016 while (i < trans->nr_sorted) {
1017 btree_path_idx_t idx = trans->sorted[i];
1023 if (trans->paths[idx].uptodate) {
1024 __btree_path_get(&trans->paths[idx], false);
1025 ret = bch2_btree_path_traverse_one(trans, idx, 0, _THIS_IP_);
1026 __btree_path_put(&trans->paths[idx], false);
1045 bch2_btree_cache_cannibalize_unlock(trans);
1047 trans->in_traverse_all = false;
1049 trace_and_count(c, trans_traverse_all, trans, trace_ip);
1063 static inline bool btree_path_good_node(struct btree_trans *trans,
1068 bch2_btree_node_relock(trans, path, l) &&
1072 static void btree_path_set_level_down(struct btree_trans *trans,
1082 btree_node_unlock(trans, path, l);
1085 bch2_btree_path_verify(trans, path);
1088 static noinline unsigned __btree_path_up_until_good_node(struct btree_trans *trans,
1095 !btree_path_good_node(trans, path, l, check_pos))
1096 __btree_path_set_level_up(trans, path, l++);
1102 if (!bch2_btree_node_relock(trans, path, i)) {
1104 __btree_path_set_level_up(trans, path, l++);
1111 static inline unsigned btree_path_up_until_good_node(struct btree_trans *trans,
1118 : __btree_path_up_until_good_node(trans, path, check_pos);
1130 int bch2_btree_path_traverse_one(struct btree_trans *trans,
1135 struct btree_path *path = &trans->paths[path_idx];
1137 int ret = -((int) trans->restarted);
1142 if (unlikely(!trans->srcu_held))
1143 bch2_trans_srcu_lock(trans);
1150 ret = bch2_btree_path_relock(trans, path, trace_ip);
1155 ret = bch2_btree_path_traverse_cached(trans, path, flags);
1159 path = &trans->paths[path_idx];
1164 path->level = btree_path_up_until_good_node(trans, path, 0);
1177 ? btree_path_down(trans, path, flags, trace_ip)
1178 : btree_path_lock_root(trans, path, depth_want, trace_ip);
1189 __bch2_btree_path_unlock(trans, path);
1198 if (bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted)
1199 panic("ret %s (%i) trans->restarted %s (%i)\n",
1201 bch2_err_str(trans->restarted), trans->restarted);
1202 bch2_btree_path_verify(trans, path);
1206 static inline void btree_path_copy(struct btree_trans *trans, struct btree_path *dst,
1223 static btree_path_idx_t btree_path_clone(struct btree_trans *trans, btree_path_idx_t src,
1226 btree_path_idx_t new = btree_path_alloc(trans, src);
1227 btree_path_copy(trans, trans->paths + new, trans->paths + src);
1228 __btree_path_get(trans->paths + new, intent);
1233 btree_path_idx_t __bch2_btree_path_make_mut(struct btree_trans *trans,
1236 __btree_path_put(trans->paths + path, intent);
1237 path = btree_path_clone(trans, path, intent);
1238 trans->paths[path].preserve = false;
1243 __bch2_btree_path_set_pos(struct btree_trans *trans,
1247 int cmp = bpos_cmp(new_pos, trans->paths[path_idx].pos);
1249 bch2_trans_verify_not_in_restart(trans);
1250 EBUG_ON(!trans->paths[path_idx].ref);
1252 path_idx = bch2_btree_path_make_mut(trans, path_idx, intent, ip);
1254 struct btree_path *path = trans->paths + path_idx;
1256 trans->paths_sorted = false;
1259 btree_node_unlock(trans, path, 0);
1265 unsigned level = btree_path_up_until_good_node(trans, path, cmp);
1291 __bch2_btree_path_unlock(trans, path);
1294 bch2_btree_path_verify(trans, path);
1300 static struct btree_path *have_path_at_pos(struct btree_trans *trans, struct btree_path *path)
1304 sib = prev_btree_path(trans, path);
1308 sib = next_btree_path(trans, path);
1315 static struct btree_path *have_node_at_pos(struct btree_trans *trans, struct btree_path *path)
1319 sib = prev_btree_path(trans, path);
1323 sib = next_btree_path(trans, path);
1330 static inline void __bch2_path_free(struct btree_trans *trans, btree_path_idx_t path)
1332 __bch2_btree_path_unlock(trans, trans->paths + path);
1333 btree_path_list_remove(trans, trans->paths + path);
1334 __clear_bit(path, trans->paths_allocated);
1337 void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool intent)
1339 struct btree_path *path = trans->paths + path_idx, *dup;
1345 ? have_path_at_pos(trans, path)
1346 : have_node_at_pos(trans, path);
1352 !trans->restarted &&
1353 (!dup || !bch2_btree_path_relock_norestart(trans, dup)))
1361 __bch2_path_free(trans, path_idx);
1364 static void bch2_path_put_nokeep(struct btree_trans *trans, btree_path_idx_t path,
1367 if (!__btree_path_put(trans->paths + path, intent))
1370 __bch2_path_free(trans, path);
1373 void __noreturn bch2_trans_restart_error(struct btree_trans *trans, u32 restart_count)
1375 panic("trans->restart_count %u, should be %u, last restarted by %pS\n",
1376 trans->restart_count, restart_count,
1377 (void *) trans->last_begin_ip);
1380 void __noreturn bch2_trans_in_restart_error(struct btree_trans *trans)
1383 bch2_err_str(trans->restarted),
1384 (void *) trans->last_restarted_ip);
1388 void bch2_trans_updates_to_text(struct printbuf *buf, struct btree_trans *trans)
1391 trans->fn, trans->journal_res.seq);
1395 trans_for_each_update(trans, i) {
1405 bch2_bkey_val_to_text(buf, trans->c, old);
1409 bch2_bkey_val_to_text(buf, trans->c, bkey_i_to_s_c(i->k));
1413 for (struct jset_entry *e = trans->journal_entries;
1414 e != btree_trans_journal_entries_top(trans);
1416 bch2_journal_entry_to_text(buf, trans->c, e);
1422 void bch2_dump_trans_updates(struct btree_trans *trans)
1426 bch2_trans_updates_to_text(&buf, trans);
1431 static void bch2_btree_path_to_text(struct printbuf *out, struct btree_trans *trans, btree_path_idx_t path_idx)
1433 struct btree_path *path = trans->paths + path_idx;
1451 void __bch2_trans_paths_to_text(struct printbuf *out, struct btree_trans *trans,
1457 btree_trans_sort_paths(trans);
1459 trans_for_each_path_idx_inorder(trans, iter)
1460 bch2_btree_path_to_text(out, trans, iter.path_idx);
1464 void bch2_trans_paths_to_text(struct printbuf *out, struct btree_trans *trans)
1466 __bch2_trans_paths_to_text(out, trans, false);
1470 void __bch2_dump_trans_paths_updates(struct btree_trans *trans, bool nosort)
1474 __bch2_trans_paths_to_text(&buf, trans, nosort);
1475 bch2_trans_updates_to_text(&buf, trans);
1482 void bch2_dump_trans_paths_updates(struct btree_trans *trans)
1484 __bch2_dump_trans_paths_updates(trans, false);
1488 static void bch2_trans_update_max_paths(struct btree_trans *trans)
1490 struct btree_transaction_stats *s = btree_trans_stats(trans);
1492 size_t nr = bitmap_weight(trans->paths_allocated, trans->nr_paths);
1494 bch2_trans_paths_to_text(&buf, trans);
1507 trans->nr_paths_max = nr;
1511 int __bch2_btree_trans_too_many_iters(struct btree_trans *trans)
1516 bch2_trans_paths_to_text(&buf, trans);
1517 trace_trans_restart_too_many_iters(trans, _THIS_IP_, buf.buf);
1521 count_event(trans->c, trans_restart_too_many_iters);
1523 return btree_trans_restart(trans, BCH_ERR_transaction_restart_too_many_iters);
1526 static noinline void btree_path_overflow(struct btree_trans *trans)
1528 bch2_dump_trans_paths_updates(trans);
1529 bch_err(trans->c, "trans path overflow");
1532 static noinline void btree_paths_realloc(struct btree_trans *trans)
1534 unsigned nr = trans->nr_paths * 2;
1543 memcpy(paths_allocated, trans->paths_allocated, BITS_TO_LONGS(trans->nr_paths) * sizeof(unsigned long));
1549 memcpy(paths, trans->paths, trans->nr_paths * sizeof(struct btree_path));
1553 memcpy(sorted, trans->sorted, trans->nr_sorted * sizeof(btree_path_idx_t));
1557 memcpy(updates, trans->updates, trans->nr_paths * sizeof(struct btree_insert_entry));
1559 unsigned long *old = trans->paths_allocated;
1561 rcu_assign_pointer(trans->paths_allocated, paths_allocated);
1562 rcu_assign_pointer(trans->paths, paths);
1563 rcu_assign_pointer(trans->sorted, sorted);
1564 rcu_assign_pointer(trans->updates, updates);
1566 trans->nr_paths = nr;
1568 if (old != trans->_paths_allocated)
1572 static inline btree_path_idx_t btree_path_alloc(struct btree_trans *trans,
1575 btree_path_idx_t idx = find_first_zero_bit(trans->paths_allocated, trans->nr_paths);
1577 if (unlikely(idx == trans->nr_paths)) {
1578 if (trans->nr_paths == BTREE_ITER_MAX) {
1579 btree_path_overflow(trans);
1583 btree_paths_realloc(trans);
1590 if (unlikely(idx > trans->nr_paths_max))
1591 bch2_trans_update_max_paths(trans);
1593 __set_bit(idx, trans->paths_allocated);
1595 struct btree_path *path = &trans->paths[idx];
1600 btree_path_list_add(trans, pos, idx);
1601 trans->paths_sorted = false;
1605 btree_path_idx_t bch2_path_get(struct btree_trans *trans,
1616 bch2_trans_verify_not_in_restart(trans);
1617 bch2_trans_verify_locks(trans);
1619 btree_trans_sort_paths(trans);
1621 trans_for_each_path_inorder(trans, path, iter) {
1633 trans->paths[path_pos].cached == cached &&
1634 trans->paths[path_pos].btree_id == btree_id &&
1635 trans->paths[path_pos].level == level) {
1636 __btree_path_get(trans->paths + path_pos, intent);
1637 path_idx = bch2_btree_path_set_pos(trans, path_pos, pos, intent, ip);
1638 path = trans->paths + path_idx;
1640 path_idx = btree_path_alloc(trans, path_pos);
1641 path = trans->paths + path_idx;
1657 trans->paths_sorted = false;
1676 bch2_btree_path_upgrade_noupgrade_sibs(trans, path, locks_want, NULL);
1727 return bch2_btree_path_traverse(iter->trans, iter->path, iter->flags);
1733 struct btree_trans *trans = iter->trans;
1736 iter->path = bch2_btree_path_set_pos(trans, iter->path,
1741 ret = bch2_btree_path_traverse(iter->trans, iter->path, iter->flags);
1745 struct btree_path *path = btree_iter_path(trans, iter);
1755 struct btree_trans *trans = iter->trans;
1759 EBUG_ON(trans->paths[iter->path].cached);
1762 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
1766 struct btree_path *path = btree_iter_path(trans, iter);
1776 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p,
1779 btree_path_set_should_be_locked(btree_iter_path(trans, iter));
1796 bch2_trans_begin(iter->trans);
1803 struct btree_trans *trans = iter->trans;
1807 EBUG_ON(trans->paths[iter->path].cached);
1808 bch2_trans_verify_not_in_restart(trans);
1811 struct btree_path *path = btree_iter_path(trans, iter);
1819 btree_path_set_level_up(trans, path);
1823 if (!bch2_btree_node_relock(trans, path, path->level + 1)) {
1824 __bch2_btree_path_unlock(trans, path);
1828 trace_and_count(trans->c, trans_restart_relock_next_node, trans, _THIS_IP_, path);
1829 ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
1836 __btree_path_set_level_up(trans, path, path->level++);
1842 iter->path = bch2_btree_path_set_pos(trans, iter->path,
1847 path = btree_iter_path(trans, iter);
1848 btree_path_set_level_down(trans, path, iter->min_depth);
1850 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
1854 path = btree_iter_path(trans, iter);
1861 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p,
1864 btree_path_set_should_be_locked(btree_iter_path(trans, iter));
1865 EBUG_ON(btree_iter_path(trans, iter)->uptodate);
1905 void bch2_btree_trans_peek_prev_updates(struct btree_trans *trans, struct btree_iter *iter,
1908 struct bpos end = path_l(btree_iter_path(trans, iter))->b->data->min_key;
1910 trans_for_each_update(trans, i)
1921 void bch2_btree_trans_peek_updates(struct btree_trans *trans, struct btree_iter *iter,
1924 struct btree_path *path = btree_iter_path(trans, iter);
1927 trans_for_each_update(trans, i)
1938 void bch2_btree_trans_peek_slot_updates(struct btree_trans *trans, struct btree_iter *iter,
1941 trans_for_each_update(trans, i)
1950 static struct bkey_i *bch2_btree_journal_peek(struct btree_trans *trans,
1954 struct btree_path *path = btree_iter_path(trans, iter);
1956 return bch2_journal_keys_peek_upto(trans->c, iter->btree_id,
1964 struct bkey_s_c btree_trans_peek_slot_journal(struct btree_trans *trans,
1967 struct btree_path *path = btree_iter_path(trans, iter);
1968 struct bkey_i *k = bch2_btree_journal_peek(trans, iter, path->pos);
1979 struct bkey_s_c btree_trans_peek_journal(struct btree_trans *trans,
1983 struct btree_path *path = btree_iter_path(trans, iter);
1985 bch2_btree_journal_peek(trans, iter,
2003 struct btree_trans *trans = iter->trans;
2004 struct bch_fs *c = trans->c;
2017 iter->key_cache_path = bch2_path_get(trans, iter->btree_id, pos,
2023 iter->key_cache_path = bch2_btree_path_set_pos(trans, iter->key_cache_path, pos,
2027 ret = bch2_btree_path_traverse(trans, iter->key_cache_path,
2029 bch2_btree_path_relock(trans, btree_iter_path(trans, iter), _THIS_IP_);
2033 btree_path_set_should_be_locked(trans->paths + iter->key_cache_path);
2035 k = bch2_btree_path_peek_slot(trans->paths + iter->key_cache_path, &u);
2045 struct btree_trans *trans = iter->trans;
2049 EBUG_ON(btree_iter_path(trans, iter)->cached);
2055 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key,
2059 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
2067 struct btree_path *path = btree_iter_path(trans, iter);
2079 k = btree_path_level_peek_all(trans->c, l, &iter->k);
2093 k = btree_trans_peek_journal(trans, iter, k);
2096 trans->nr_updates))
2097 bch2_btree_trans_peek_updates(trans, iter, &k);
2141 struct btree_trans *trans = iter->trans;
2150 bch2_path_put_nokeep(trans, iter->update_path,
2180 !bkey_eq(trans->paths[iter->update_path].pos, k.k->p)) {
2181 bch2_path_put_nokeep(trans, iter->update_path,
2203 __btree_path_get(trans->paths + iter->path, iter->flags & BTREE_ITER_INTENT);
2206 iter->update_path = bch2_btree_path_set_pos(trans,
2210 ret = bch2_btree_path_traverse(trans, iter->update_path, iter->flags);
2222 !bch2_snapshot_is_ancestor(trans->c,
2255 iter->path = bch2_btree_path_set_pos(trans, iter->path, k.k->p,
2259 btree_path_set_should_be_locked(btree_iter_path(trans, iter));
2262 ret = bch2_btree_path_relock(trans, trans->paths + iter->update_path, _THIS_IP_);
2266 btree_path_set_should_be_locked(trans->paths + iter->update_path);
2311 struct btree_trans *trans = iter->trans;
2319 EBUG_ON(btree_iter_path(trans, iter)->cached ||
2320 btree_iter_path(trans, iter)->level);
2332 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key,
2336 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
2344 struct btree_path *path = btree_iter_path(trans, iter);
2346 k = btree_path_level_peek(trans, path, &path->l[0], &iter->k);
2351 k = btree_path_level_prev(trans, path, &path->l[0], &iter->k);
2354 trans->nr_updates))
2355 bch2_btree_trans_peek_prev_updates(trans, iter, &k);
2368 bch2_path_put_nokeep(trans, iter->path,
2377 if (bch2_snapshot_is_ancestor(trans->c,
2381 bch2_path_put_nokeep(trans, saved_path,
2383 saved_path = btree_path_clone(trans, iter->path,
2385 path = btree_iter_path(trans, iter);
2425 bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_INTENT);
2450 struct btree_trans *trans = iter->trans;
2457 EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_WITH_KEY_CACHE));
2469 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key,
2473 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
2484 trans->nr_updates)) {
2485 bch2_btree_trans_peek_slot_updates(trans, iter, &k);
2491 (k = btree_trans_peek_slot_journal(trans, iter)).k)
2502 k = bch2_btree_path_peek_slot(trans->paths + iter->path, &iter->k);
2512 EBUG_ON(btree_iter_path(trans, iter)->level);
2525 bch2_trans_iter_exit(trans, &iter2);
2559 btree_path_set_should_be_locked(btree_iter_path(trans, iter));
2590 while (btree_trans_too_many_iters(iter->trans) ||
2593 bch2_trans_begin(iter->trans);
2601 static void btree_trans_verify_sorted_refs(struct btree_trans *trans)
2606 BUG_ON(trans->nr_sorted != bitmap_weight(trans->paths_allocated, trans->nr_paths) - 1);
2608 trans_for_each_path(trans, path, i) {
2609 BUG_ON(path->sorted_idx >= trans->nr_sorted);
2610 BUG_ON(trans->sorted[path->sorted_idx] != i);
2613 for (i = 0; i < trans->nr_sorted; i++) {
2614 unsigned idx = trans->sorted[i];
2616 BUG_ON(!test_bit(idx, trans->paths_allocated));
2617 BUG_ON(trans->paths[idx].sorted_idx != i);
2621 static void btree_trans_verify_sorted(struct btree_trans *trans)
2629 trans_for_each_path_inorder(trans, path, iter) {
2631 __bch2_dump_trans_paths_updates(trans, true);
2632 panic("trans paths out of order!\n");
2638 static inline void btree_trans_verify_sorted_refs(struct btree_trans *trans) {}
2639 static inline void btree_trans_verify_sorted(struct btree_trans *trans) {}
2642 void __bch2_btree_trans_sort_paths(struct btree_trans *trans)
2644 int i, l = 0, r = trans->nr_sorted, inc = 1;
2647 btree_trans_verify_sorted_refs(trans);
2649 if (trans->paths_sorted)
2662 if (btree_path_cmp(trans->paths + trans->sorted[i],
2663 trans->paths + trans->sorted[i + 1]) > 0) {
2664 swap(trans->sorted[i], trans->sorted[i + 1]);
2665 trans->paths[trans->sorted[i]].sorted_idx = i;
2666 trans->paths[trans->sorted[i + 1]].sorted_idx = i + 1;
2678 trans->paths_sorted = true;
2680 btree_trans_verify_sorted(trans);
2683 static inline void btree_path_list_remove(struct btree_trans *trans,
2686 EBUG_ON(path->sorted_idx >= trans->nr_sorted);
2688 trans->nr_sorted--;
2689 memmove_u64s_down_small(trans->sorted + path->sorted_idx,
2690 trans->sorted + path->sorted_idx + 1,
2691 DIV_ROUND_UP(trans->nr_sorted - path->sorted_idx,
2694 array_remove_item(trans->sorted, trans->nr_sorted, path->sorted_idx);
2696 for (unsigned i = path->sorted_idx; i < trans->nr_sorted; i++)
2697 trans->paths[trans->sorted[i]].sorted_idx = i;
2700 static inline void btree_path_list_add(struct btree_trans *trans,
2704 struct btree_path *path = trans->paths + path_idx;
2706 path->sorted_idx = pos ? trans->paths[pos].sorted_idx + 1 : trans->nr_sorted;
2709 memmove_u64s_up_small(trans->sorted + path->sorted_idx + 1,
2710 trans->sorted + path->sorted_idx,
2711 DIV_ROUND_UP(trans->nr_sorted - path->sorted_idx,
2713 trans->nr_sorted++;
2714 trans->sorted[path->sorted_idx] = path_idx;
2716 array_insert_item(trans->sorted, trans->nr_sorted, path->sorted_idx, path_idx);
2719 for (unsigned i = path->sorted_idx; i < trans->nr_sorted; i++)
2720 trans->paths[trans->sorted[i]].sorted_idx = i;
2722 btree_trans_verify_sorted_refs(trans);
2725 void bch2_trans_iter_exit(struct btree_trans *trans, struct btree_iter *iter)
2728 bch2_path_put_nokeep(trans, iter->update_path,
2731 bch2_path_put(trans, iter->path,
2734 bch2_path_put(trans, iter->key_cache_path,
2739 iter->trans = NULL;
2742 void bch2_trans_iter_init_outlined(struct btree_trans *trans,
2747 bch2_trans_iter_init_common(trans, iter, btree_id, pos, 0, 0,
2748 bch2_btree_iter_flags(trans, btree_id, flags),
2752 void bch2_trans_node_iter_init(struct btree_trans *trans,
2764 bch2_trans_iter_init_common(trans, iter, btree_id, pos, locks_want, depth,
2765 __bch2_btree_iter_flags(trans, btree_id, flags),
2770 struct btree_path *path = btree_iter_path(trans, iter);
2778 struct btree_trans *trans = src->trans;
2785 __btree_path_get(trans->paths + src->path, src->flags & BTREE_ITER_INTENT);
2787 __btree_path_get(trans->paths + src->update_path, src->flags & BTREE_ITER_INTENT);
2791 void *__bch2_trans_kmalloc(struct btree_trans *trans, size_t size)
2793 struct bch_fs *c = trans->c;
2794 unsigned new_top = trans->mem_top + size;
2795 unsigned old_bytes = trans->mem_bytes;
2803 struct btree_transaction_stats *s = btree_trans_stats(trans);
2806 if (trans->used_mempool) {
2807 if (trans->mem_bytes >= new_bytes)
2813 bch2_trans_unlock(trans);
2819 ret = bch2_trans_relock(trans);
2825 memcpy(new_mem, trans->mem, trans->mem_top);
2826 trans->used_mempool = false;
2827 mempool_free(trans->mem, &c->btree_trans_mem_pool);
2831 new_mem = krealloc(trans->mem, new_bytes, GFP_NOWAIT|__GFP_NOWARN);
2833 bch2_trans_unlock(trans);
2835 new_mem = krealloc(trans->mem, new_bytes, GFP_KERNEL);
2839 memcpy(new_mem, trans->mem, trans->mem_top);
2840 trans->used_mempool = true;
2841 kfree(trans->mem);
2847 trans->mem = new_mem;
2848 trans->mem_bytes = new_bytes;
2850 ret = bch2_trans_relock(trans);
2855 trans->mem = new_mem;
2856 trans->mem_bytes = new_bytes;
2859 trace_and_count(c, trans_restart_mem_realloced, trans, _RET_IP_, new_bytes);
2860 return ERR_PTR(btree_trans_restart(trans, BCH_ERR_transaction_restart_mem_realloced));
2863 p = trans->mem + trans->mem_top;
2864 trans->mem_top += size;
2869 static inline void check_srcu_held_too_long(struct btree_trans *trans)
2871 WARN(trans->srcu_held && time_after(jiffies, trans->srcu_lock_time + HZ * 10),
2872 "btree trans held srcu lock (delaying memory reclaim) for %lu seconds",
2873 (jiffies - trans->srcu_lock_time) / HZ);
2876 void bch2_trans_srcu_unlock(struct btree_trans *trans)
2878 if (trans->srcu_held) {
2879 struct bch_fs *c = trans->c;
2883 trans_for_each_path(trans, path, i)
2887 check_srcu_held_too_long(trans);
2888 srcu_read_unlock(&c->btree_trans_barrier, trans->srcu_idx);
2889 trans->srcu_held = false;
2893 static void bch2_trans_srcu_lock(struct btree_trans *trans)
2895 if (!trans->srcu_held) {
2896 trans->srcu_idx = srcu_read_lock(&trans->c->btree_trans_barrier);
2897 trans->srcu_lock_time = jiffies;
2898 trans->srcu_held = true;
2904 * @trans: transaction to reset
2912 u32 bch2_trans_begin(struct btree_trans *trans)
2918 bch2_trans_reset_updates(trans);
2920 trans->restart_count++;
2921 trans->mem_top = 0;
2922 trans->journal_entries = NULL;
2924 trans_for_each_path(trans, path, i) {
2932 if (!trans->restarted && path->btree_id != BTREE_ID_subvolumes)
2941 __bch2_path_free(trans, i);
2949 time_after64(now, trans->last_begin_time + 10))
2950 __bch2_time_stats_update(&btree_trans_stats(trans)->duration,
2951 trans->last_begin_time, now);
2953 if (!trans->restarted &&
2955 time_after64(now, trans->last_begin_time + BTREE_TRANS_MAX_LOCK_HOLD_TIME_NS))) {
2956 drop_locks_do(trans, (cond_resched(), 0));
2959 trans->last_begin_time = now;
2961 if (unlikely(trans->srcu_held &&
2962 time_after(jiffies, trans->srcu_lock_time + msecs_to_jiffies(10))))
2963 bch2_trans_srcu_unlock(trans);
2965 trans->last_begin_ip = _RET_IP_;
2966 if (trans->restarted) {
2967 bch2_btree_path_traverse_all(trans);
2968 trans->notrace_relock_fail = false;
2971 return trans->restart_count;
2992 struct btree_trans *trans;
2995 trans = this_cpu_xchg(c->btree_trans_bufs->trans, NULL);
2996 if (trans) {
2997 memset(trans, 0, offsetof(struct btree_trans, list));
3002 trans = mempool_alloc(&c->btree_trans_pool, GFP_NOFS);
3003 memset(trans, 0, sizeof(*trans));
3004 closure_init_stack(&trans->ref);
3011 trans->locking_wait.task = current;
3026 list_add_tail(&trans->list, &pos->list);
3031 list_add_tail(&trans->list, &c->btree_trans_list);
3035 trans->c = c;
3036 trans->last_begin_time = local_clock();
3037 trans->fn_idx = fn_idx;
3038 trans->locking_wait.task = current;
3039 trans->journal_replay_not_finished =
3042 trans->nr_paths = ARRAY_SIZE(trans->_paths);
3043 trans->paths_allocated = trans->_paths_allocated;
3044 trans->sorted = trans->_sorted;
3045 trans->paths = trans->_paths;
3046 trans->updates = trans->_updates;
3048 *trans_paths_nr(trans->paths) = BTREE_ITER_INITIAL;
3050 trans->paths_allocated[0] = 1;
3053 trans->fn = bch2_btree_transaction_fns[fn_idx];
3060 trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL);
3061 if (likely(trans->mem))
3062 trans->mem_bytes = expected_mem_bytes;
3065 trans->nr_paths_max = s->nr_max_paths;
3066 trans->journal_entries_size = s->journal_entries_size;
3069 trans->srcu_idx = srcu_read_lock(&c->btree_trans_barrier);
3070 trans->srcu_lock_time = jiffies;
3071 trans->srcu_held = true;
3072 return trans;
3075 static void check_btree_paths_leaked(struct btree_trans *trans)
3078 struct bch_fs *c = trans->c;
3082 trans_for_each_path(trans, path, i)
3087 bch_err(c, "btree paths leaked from %s!", trans->fn);
3088 trans_for_each_path(trans, path, i)
3098 void bch2_trans_put(struct btree_trans *trans)
3101 struct bch_fs *c = trans->c;
3103 bch2_trans_unlock(trans);
3105 trans_for_each_update(trans, i)
3106 __btree_path_put(trans->paths + i->path, true);
3107 trans->nr_updates = 0;
3108 trans->locking_wait.task = NULL;
3110 check_btree_paths_leaked(trans);
3112 if (trans->srcu_held) {
3113 check_srcu_held_too_long(trans);
3114 srcu_read_unlock(&c->btree_trans_barrier, trans->srcu_idx);
3117 if (trans->fs_usage_deltas) {
3118 if (trans->fs_usage_deltas->size + sizeof(trans->fs_usage_deltas) ==
3120 mempool_free(trans->fs_usage_deltas,
3123 kfree(trans->fs_usage_deltas);
3126 if (unlikely(trans->journal_replay_not_finished))
3129 unsigned long *paths_allocated = trans->paths_allocated;
3130 trans->paths_allocated = NULL;
3131 trans->paths = NULL;
3133 if (paths_allocated != trans->_paths_allocated)
3136 if (trans->used_mempool)
3137 mempool_free(trans->mem, &c->btree_trans_mem_pool);
3139 kfree(trans->mem);
3143 trans = this_cpu_xchg(c->btree_trans_bufs->trans, trans);
3145 if (trans) {
3146 closure_sync(&trans->ref);
3149 list_del(&trans->list);
3152 mempool_free(trans, &c->btree_trans_pool);
3179 void bch2_btree_trans_to_text(struct printbuf *out, struct btree_trans *trans)
3183 struct task_struct *task = READ_ONCE(trans->locking_wait.task);
3194 prt_printf(out, "%i %s\n", task ? task->pid : 0, trans->fn);
3196 /* trans->paths is rcu protected vs. freeing */
3200 struct btree_path *paths = rcu_dereference(trans->paths);
3230 b = READ_ONCE(trans->locking);
3233 div_u64(local_clock() - trans->locking_wait.start_time,
3236 prt_printf(out, " %c", lock_types[trans->locking_wait.lock_want]);
3248 struct btree_trans *trans;
3253 struct btree_trans *trans =
3254 per_cpu_ptr(c->btree_trans_bufs, cpu)->trans;
3256 if (trans) {
3257 closure_sync(&trans->ref);
3260 list_del(&trans->list);
3263 kfree(trans);
3267 trans = list_first_entry_or_null(&c->btree_trans_list, struct btree_trans, list);
3268 if (trans)
3269 panic("%s leaked btree_trans\n", trans->fn);