Lines Matching refs:sb
141 * that list. So, there are exactly MB_NUM_ORDERS(sb) (which means total
153 * so we only have MB_NUM_ORDERS(sb) lists.
419 static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
426 static int ext4_try_to_trim_range(struct super_block *sb,
556 struct super_block *sb = e4b->bd_sb;
560 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
565 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
566 blocknr += EXT4_C2B(EXT4_SB(sb), first + i);
567 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
569 ext4_grp_locked_error(sb, e4b->bd_group,
615 static void mb_group_bb_bitmap_alloc(struct super_block *sb,
620 grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);
624 bh = ext4_read_block_bitmap(sb, group);
631 memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize);
656 static inline void mb_group_bb_bitmap_alloc(struct super_block *sb,
683 struct super_block *sb = e4b->bd_sb;
759 grp = ext4_get_group_info(sb, e4b->bd_group);
766 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);
785 static void ext4_mb_mark_free_simple(struct super_block *sb,
789 struct ext4_sb_info *sbi = EXT4_SB(sb);
795 BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb));
797 border = 2 << sb->s_blocksize_bits;
821 static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len)
832 if (order == MB_NUM_ORDERS(sb))
839 mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp)
841 struct ext4_sb_info *sbi = EXT4_SB(sb);
844 if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_fragments == 0)
847 new_order = mb_avg_fragment_size_order(sb,
1146 mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp)
1148 struct ext4_sb_info *sbi = EXT4_SB(sb);
1151 for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--)
1155 if (!test_opt2(sb, MB_OPTIMIZE_SCAN) ||
1180 void ext4_mb_generate_buddy(struct super_block *sb,
1184 struct ext4_sb_info *sbi = EXT4_SB(sb);
1185 ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb);
1204 ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
1213 ext4_grp_locked_error(sb, group, 0, 0,
1222 ext4_mark_group_bitmap_corrupted(sb, group,
1225 mb_set_largest_free_order(sb, grp);
1226 mb_update_avg_fragment_size(sb, grp);
1283 struct super_block *sb;
1292 sb = inode->i_sb;
1293 ngroups = ext4_get_groups_count(sb);
1297 mb_debug(sb, "init page %lu\n", page->index);
1319 grinfo = ext4_get_group_info(sb, group);
1332 bh[i] = ext4_read_block_bitmap_nowait(sb, group, false);
1338 mb_debug(sb, "read bitmap for group %u\n", group);
1347 err2 = ext4_wait_block_bitmap(sb, group, bh[i]);
1380 grinfo = ext4_get_group_info(sb, group);
1388 mb_debug(sb, "put buddy for group %u in page %lu/%x\n",
1390 trace_ext4_mb_buddy_bitmap_load(sb, group);
1394 (MB_NUM_ORDERS(sb)));
1398 ext4_lock_group(sb, group);
1401 ext4_mb_generate_buddy(sb, data, incore, group, grinfo);
1402 ext4_unlock_group(sb, group);
1407 mb_debug(sb, "put bitmap for group %u in page %lu/%x\n",
1409 trace_ext4_mb_bitmap_load(sb, group);
1412 ext4_lock_group(sb, group);
1416 ext4_mb_generate_from_pa(sb, data, group);
1418 ext4_unlock_group(sb, group);
1444 static int ext4_mb_get_buddy_page_lock(struct super_block *sb,
1447 struct inode *inode = EXT4_SB(sb)->s_buddy_cache;
1455 blocks_per_page = PAGE_SIZE / sb->s_blocksize;
1469 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1503 int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp)
1512 mb_debug(sb, "init group %u\n", group);
1513 this_grp = ext4_get_group_info(sb, group);
1526 ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp);
1573 ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group,
1583 struct ext4_sb_info *sbi = EXT4_SB(sb);
1587 mb_debug(sb, "load group %u\n", group);
1589 blocks_per_page = PAGE_SIZE / sb->s_blocksize;
1590 grp = ext4_get_group_info(sb, group);
1594 e4b->bd_blkbits = sb->s_blocksize_bits;
1596 e4b->bd_sb = sb;
1606 ret = ext4_mb_init_group(sb, group, gfp);
1650 (poff * sb->s_blocksize));
1666 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1707 e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
1722 static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
1725 return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS);
1898 struct super_block *sb = e4b->bd_sb;
1902 BUG_ON(last >= (sb->s_blocksize << 3));
1903 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
1917 if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])
1921 struct ext4_sb_info *sbi = EXT4_SB(sb);
1933 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
1935 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
1937 ext4_grp_locked_error(sb, e4b->bd_group,
1973 mb_set_largest_free_order(sb, e4b->bd_info);
1974 mb_update_avg_fragment_size(sb, e4b->bd_info);
2389 struct super_block *sb = ac->ac_sb;
2397 for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) {
2427 if (EXT4_SB(sb)->s_mb_stats)
2428 atomic_inc(&EXT4_SB(sb)->s_bal_2orders);
2443 struct super_block *sb = ac->ac_sb;
2457 EXT4_CLUSTERS_PER_GROUP(sb), i);
2458 if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) {
2464 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2466 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2481 EXT4_CLUSTERS_PER_GROUP(sb), i);
2495 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2497 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2526 struct super_block *sb = ac->ac_sb;
2527 struct ext4_sb_info *sbi = EXT4_SB(sb);
2538 first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);
2546 while (i < EXT4_CLUSTERS_PER_GROUP(sb)) {
2640 struct super_block *sb = ac->ac_sb;
2641 struct ext4_sb_info *sbi = EXT4_SB(sb);
2651 ext4_lock_group(sb, group);
2652 __release(ext4_group_lock_ptr(sb, group));
2667 __acquire(ext4_group_lock_ptr(sb, group));
2668 ext4_unlock_group(sb, group);
2674 ext4_get_group_desc(sb, group, NULL);
2689 !(ext4_has_group_desc_csum(sb) &&
2692 ret = ext4_mb_init_group(sb, group, GFP_NOFS);
2698 ext4_lock_group(sb, group);
2699 __release(ext4_group_lock_ptr(sb, group));
2704 __acquire(ext4_group_lock_ptr(sb, group));
2705 ext4_unlock_group(sb, group);
2714 ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group,
2717 ext4_group_t ngroups = ext4_get_groups_count(sb);
2723 struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group,
2725 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
2736 ext4_free_group_clusters(sb, gdp) > 0 ) {
2737 bh = ext4_read_block_bitmap_nowait(sb, group, true);
2763 void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group,
2771 group = ext4_get_groups_count(sb);
2773 gdp = ext4_get_group_desc(sb, group, NULL);
2774 grp = ext4_get_group_info(sb, group);
2777 ext4_free_group_clusters(sb, gdp) > 0) {
2778 if (ext4_mb_init_group(sb, group, GFP_NOFS))
2792 struct super_block *sb;
2796 sb = ac->ac_sb;
2797 sbi = EXT4_SB(sb);
2798 ngroups = ext4_get_groups_count(sb);
2827 if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) {
2830 MB_NUM_ORDERS(sb));
2880 if (ext4_has_feature_flex_bg(sb)) {
2885 prefetch_grp = ext4_mb_prefetch(sb, group,
2897 err = ext4_mb_load_buddy(sb, group, &e4b);
2901 ext4_lock_group(sb, group);
2909 ext4_unlock_group(sb, group);
2931 ext4_unlock_group(sb, group);
2961 mb_debug(sb, "lost chunk, group: %u, start: %d, len: %d, lost: %d\n",
2981 mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n",
2986 ext4_mb_prefetch_fini(sb, prefetch_grp, nr);
2993 struct super_block *sb = pde_data(file_inode(seq->file));
2996 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
3004 struct super_block *sb = pde_data(file_inode(seq->file));
3008 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
3016 struct super_block *sb = pde_data(file_inode(seq->file));
3023 sb->s_blocksize_bits,
3039 grinfo = ext4_get_group_info(sb, group);
3044 err = ext4_mb_load_buddy(sb, group, &e4b);
3083 struct super_block *sb = seq->private;
3084 struct ext4_sb_info *sbi = EXT4_SB(sb);
3179 ext4_get_groups_count(sb));
3189 __acquires(&EXT4_SB(sb)->s_mb_rb_lock)
3191 struct super_block *sb = pde_data(file_inode(seq->file));
3194 if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb))
3202 struct super_block *sb = pde_data(file_inode(seq->file));
3206 if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb))
3214 struct super_block *sb = pde_data(file_inode(seq->file));
3215 struct ext4_sb_info *sbi = EXT4_SB(sb);
3221 if (position >= MB_NUM_ORDERS(sb)) {
3222 position -= MB_NUM_ORDERS(sb);
3239 test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0);
3278 int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups)
3280 struct ext4_sb_info *sbi = EXT4_SB(sb);
3284 size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>
3285 EXT4_DESC_PER_BLOCK_BITS(sb);
3292 ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group");
3311 int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group,
3316 int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb);
3317 struct ext4_sb_info *sbi = EXT4_SB(sb);
3319 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);
3326 if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
3328 EXT4_DESC_PER_BLOCK_BITS(sb);
3331 ext4_msg(sb, KERN_ERR, "can't allocate mem "
3341 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);
3345 ext4_msg(sb, KERN_ERR, "can't allocate buddy mem");
3355 if (ext4_has_group_desc_csum(sb) &&
3358 ext4_free_clusters_after_init(sb, group, desc);
3361 ext4_free_group_clusters(sb, desc);
3373 mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group);
3378 if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
3390 static int ext4_mb_init_backend(struct super_block *sb)
3392 ext4_group_t ngroups = ext4_get_groups_count(sb);
3394 struct ext4_sb_info *sbi = EXT4_SB(sb);
3400 err = ext4_mb_alloc_groupinfo(sb, ngroups);
3404 sbi->s_buddy_cache = new_inode(sb);
3406 ext4_msg(sb, KERN_ERR, "can't get new inode");
3417 desc = ext4_get_group_desc(sb, i, NULL);
3419 ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i);
3422 if (ext4_mb_add_groupinfo(sb, i, desc) != 0)
3426 if (ext4_has_feature_flex_bg(sb)) {
3432 ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group");
3436 BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9));
3441 if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))
3442 sbi->s_mb_prefetch = ext4_get_groups_count(sb);
3451 if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))
3452 sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);
3457 cachep = get_groupinfo_cache(sb->s_blocksize_bits);
3459 struct ext4_group_info *grp = ext4_get_group_info(sb, i);
3531 struct super_block *sb = sbi->s_sb;
3548 if ((sb->s_flags & SB_ACTIVE) && !err &&
3555 err = ext4_mb_load_buddy(sb, grp, &e4b);
3565 ext4_lock_group(sb, grp);
3566 ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster,
3568 ext4_unlock_group(sb, grp);
3577 int ext4_mb_init(struct super_block *sb)
3579 struct ext4_sb_info *sbi = EXT4_SB(sb);
3585 i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets);
3593 i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs);
3600 ret = ext4_groupinfo_create_slab(sb->s_blocksize);
3605 sbi->s_mb_maxs[0] = sb->s_blocksize << 3;
3610 offset_incr = 1 << (sb->s_blocksize_bits - 1);
3611 max = sb->s_blocksize << 2;
3619 } while (i < MB_NUM_ORDERS(sb));
3622 kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head),
3629 kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t),
3635 for (i = 0; i < MB_NUM_ORDERS(sb); i++) {
3640 kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head),
3647 kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t),
3653 for (i = 0; i < MB_NUM_ORDERS(sb); i++) {
3714 if (bdev_nonrot(sb->s_bdev))
3719 ret = ext4_mb_init_backend(sb);
3756 void ext4_mb_release(struct super_block *sb)
3758 ext4_group_t ngroups = ext4_get_groups_count(sb);
3762 struct ext4_sb_info *sbi = EXT4_SB(sb);
3763 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);
3766 if (test_opt(sb, DISCARD)) {
3777 grinfo = ext4_get_group_info(sb, i);
3781 ext4_lock_group(sb, i);
3784 mb_debug(sb, "mballoc: %d PAs left\n",
3786 ext4_unlock_group(sb, i);
3790 EXT4_DESC_PER_BLOCK(sb) - 1) >>
3791 EXT4_DESC_PER_BLOCK_BITS(sb);
3807 ext4_msg(sb, KERN_INFO,
3812 ext4_msg(sb, KERN_INFO,
3821 ext4_msg(sb, KERN_INFO,
3825 ext4_msg(sb, KERN_INFO,
3834 static inline int ext4_issue_discard(struct super_block *sb,
3839 discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) +
3840 ext4_group_first_block_no(sb, block_group));
3841 count = EXT4_C2B(EXT4_SB(sb), count);
3842 trace_ext4_discard_blocks(sb,
3845 return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0);
3848 static void ext4_free_data_in_buddy(struct super_block *sb,
3855 mb_debug(sb, "gonna free %u blocks in group %u (0x%p):",
3858 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
3862 spin_lock(&EXT4_SB(sb)->s_md_lock);
3863 EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;
3864 spin_unlock(&EXT4_SB(sb)->s_md_lock);
3869 ext4_lock_group(sb, entry->efd_group);
3880 if (!test_opt(sb, DISCARD))
3890 ext4_unlock_group(sb, entry->efd_group);
3893 mb_debug(sb, "freed %d blocks in 1 structures\n", count);
3900 void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid)
3902 struct ext4_sb_info *sbi = EXT4_SB(sb);
3911 ext4_free_data_in_buddy(sb, entry);
3913 if (test_opt(sb, DISCARD)) {
3969 ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state,
3973 struct ext4_sb_info *sbi = EXT4_SB(sb);
3981 handle, sb, state, group, blkoff, len,
3986 bitmap_bh = ext4_read_block_bitmap(sb, group);
3992 err = ext4_journal_get_write_access(handle, sb, bitmap_bh,
3999 gdp = ext4_get_group_desc(sb, group, &gdp_bh);
4005 err = ext4_journal_get_write_access(handle, sb, gdp_bh,
4011 ext4_lock_group(sb, group);
4012 if (ext4_has_group_desc_csum(sb) &&
4015 ext4_free_group_clusters_set(sb, gdp,
4016 ext4_free_clusters_after_init(sb, group, gdp));
4030 ext4_free_group_clusters_set(sb, gdp,
4031 ext4_free_group_clusters(sb, gdp) - changed);
4034 ext4_free_group_clusters_set(sb, gdp,
4035 ext4_free_group_clusters(sb, gdp) + changed);
4038 ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh);
4039 ext4_group_desc_csum_set(sb, group, gdp);
4040 ext4_unlock_group(sb, group);
4082 struct super_block *sb;
4091 sb = ac->ac_sb;
4092 sbi = EXT4_SB(sb);
4094 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, NULL);
4098 ext4_free_group_clusters(sb, gdp));
4100 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
4103 ext4_error(sb, "Allocating blocks %llu-%llu which overlap "
4109 err = ext4_mb_mark_context(handle, sb, true,
4122 err = ext4_mb_mark_context(handle, sb, true, ac->ac_b_ex.fe_group,
4148 void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block,
4151 struct ext4_sb_info *sbi = EXT4_SB(sb);
4158 ext4_get_group_no_and_offset(sb, block, &group, &blkoff);
4169 EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));
4172 if (!ext4_sb_block_valid(sb, NULL, block, thisgrp_len)) {
4173 ext4_error(sb, "Marking blocks in system zone - "
4179 err = ext4_mb_mark_context(NULL, sb, state,
4204 struct super_block *sb = ac->ac_sb;
4208 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
4209 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);
4977 void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
4980 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
5002 ext4_get_group_no_and_offset(sb, pa->pa_pstart,
5012 mb_debug(sb, "preallocated %d for group %u\n", preallocated, group);
5015 static void ext4_mb_mark_pa_deleted(struct super_block *sb,
5021 ext4_warning(sb, "deleted pa, type:%d, pblk:%llu, lblk:%u, len:%d\n",
5056 struct super_block *sb, struct ext4_prealloc_space *pa)
5074 ext4_mb_mark_pa_deleted(sb, pa);
5085 grp = ext4_get_group_number(sb, grp_blk);
5101 ext4_lock_group(sb, grp);
5103 ext4_unlock_group(sb, grp);
5149 struct super_block *sb = ac->ac_sb;
5150 struct ext4_sb_info *sbi = EXT4_SB(sb);
5212 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
5220 mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,
5228 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
5249 struct super_block *sb = ac->ac_sb;
5262 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
5272 mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart,
5277 atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
5279 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
5316 struct super_block *sb = e4b->bd_sb;
5317 struct ext4_sb_info *sbi = EXT4_SB(sb);
5326 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
5336 mb_debug(sb, "free preallocated %u/%u in group %u\n",
5337 (unsigned) ext4_group_first_block_no(sb, group) + bit,
5341 trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);
5354 ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u",
5368 struct super_block *sb = e4b->bd_sb;
5372 trace_ext4_mb_release_group_pa(sb, pa);
5374 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
5376 ext4_warning(sb, "bad group: expected %u, group %u, pa_start %llu",
5381 atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
5382 trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);
5395 ext4_mb_discard_group_preallocations(struct super_block *sb,
5398 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
5409 mb_debug(sb, "discard preallocation for group %u\n", group);
5413 bitmap_bh = ext4_read_block_bitmap(sb, group);
5416 ext4_error_err(sb, -err,
5422 err = ext4_mb_load_buddy(sb, group, &e4b);
5424 ext4_warning(sb, "Error %d loading buddy information for %u",
5430 ext4_lock_group(sb, group);
5445 ext4_mb_mark_pa_deleted(sb, pa);
5485 ext4_unlock_group(sb, group);
5489 mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n",
5506 struct super_block *sb = inode->i_sb;
5518 if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)
5521 mb_debug(sb, "discard preallocation for inode %lu\n",
5541 ext4_msg(sb, KERN_ERR,
5549 ext4_mb_mark_pa_deleted(sb, pa);
5579 group = ext4_get_group_number(sb, pa->pa_pstart);
5581 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
5584 ext4_error_err(sb, -err, "Error %d loading buddy information for %u",
5589 bitmap_bh = ext4_read_block_bitmap(sb, group);
5592 ext4_error_err(sb, -err, "Error %d reading block bitmap for %u",
5598 ext4_lock_group(sb, group);
5601 ext4_unlock_group(sb, group);
5641 static inline void ext4_mb_show_pa(struct super_block *sb)
5645 if (ext4_forced_shutdown(sb))
5648 ngroups = ext4_get_groups_count(sb);
5649 mb_debug(sb, "groups: ");
5651 struct ext4_group_info *grp = ext4_get_group_info(sb, i);
5658 ext4_lock_group(sb, i);
5663 ext4_get_group_no_and_offset(sb, pa->pa_pstart,
5666 mb_debug(sb, "PA:%u:%d:%d\n", i, start,
5669 ext4_unlock_group(sb, i);
5670 mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free,
5677 struct super_block *sb = ac->ac_sb;
5679 if (ext4_forced_shutdown(sb))
5682 mb_debug(sb, "Can't allocate:"
5684 mb_debug(sb, "status %u flags 0x%x",
5686 mb_debug(sb, "orig %lu/%lu/%lu@%lu, "
5702 mb_debug(sb, "%u found", ac->ac_found);
5703 mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no");
5705 mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ?
5707 ext4_mb_show_pa(sb);
5710 static inline void ext4_mb_show_pa(struct super_block *sb)
5782 struct super_block *sb = ar->inode->i_sb;
5783 struct ext4_sb_info *sbi = EXT4_SB(sb);
5794 if (len >= EXT4_CLUSTERS_PER_GROUP(sb))
5795 len = EXT4_CLUSTERS_PER_GROUP(sb);
5802 ext4_get_group_no_and_offset(sb, goal, &group, &block);
5807 ac->ac_sb = sb;
5821 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, "
5831 ext4_mb_discard_lg_preallocations(struct super_block *sb,
5840 mb_debug(sb, "discard locality group preallocation\n");
5864 ext4_mb_mark_pa_deleted(sb, pa);
5886 group = ext4_get_group_number(sb, pa->pa_pstart);
5887 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
5890 ext4_error_err(sb, -err, "Error %d loading buddy information for %u",
5894 ext4_lock_group(sb, group);
5897 ext4_unlock_group(sb, group);
5917 struct super_block *sb = ac->ac_sb;
5955 ext4_mb_discard_lg_preallocations(sb, lg,
6001 static int ext4_mb_discard_preallocations(struct super_block *sb, int needed)
6003 ext4_group_t i, ngroups = ext4_get_groups_count(sb);
6008 trace_ext4_mb_discard_preallocations(sb, needed);
6011 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1;
6014 ret = ext4_mb_discard_group_preallocations(sb, i, &busy);
6028 static bool ext4_mb_discard_preallocations_should_retry(struct super_block *sb,
6035 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);
6048 mb_debug(sb, "freed %d, retry ? %s\n", freed, ret ? "yes" : "no");
6061 struct super_block *sb = ar->inode->i_sb;
6062 struct ext4_sb_info *sbi = EXT4_SB(sb);
6065 ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb);
6076 ext4_get_group_no_and_offset(sb, goal, &group, &blkoff);
6077 for (nr = ext4_get_groups_count(sb); nr > 0; nr--) {
6078 bitmap_bh = ext4_read_block_bitmap(sb, group);
6090 if (ext4_fc_replay_check_excluded(sb,
6091 ext4_group_first_block_no(sb, group) +
6101 if (++group >= ext4_get_groups_count(sb))
6112 block = ext4_group_first_block_no(sb, group) + EXT4_C2B(sbi, i);
6113 ext4_mb_mark_bb(sb, block, 1, true);
6129 struct super_block *sb;
6137 sb = ar->inode->i_sb;
6138 sbi = EXT4_SB(sb);
6161 ext4_mb_show_pa(sb);
6228 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
6233 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq))
6304 struct super_block *sb = e4b->bd_sb;
6305 struct ext4_sb_info *sbi = EXT4_SB(sb);
6333 ext4_grp_locked_error(sb, group, 0,
6334 ext4_group_first_block_no(sb, group) +
6369 struct super_block *sb = inode->i_sb;
6373 ext4_get_group_no_and_offset(sb, block, &group, &blkoff);
6374 ext4_mb_mark_context(NULL, sb, false, group, blkoff, count,
6393 struct super_block *sb = inode->i_sb;
6405 sbi = EXT4_SB(sb);
6409 ext4_error(sb, "Freeing blocks in system zone - "
6418 ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
6420 grp = ext4_get_group_info(sb, block_group);
6428 if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) {
6430 EXT4_BLOCKS_PER_GROUP(sb);
6436 trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters);
6439 err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b,
6446 ext4_error(sb, "Freeing blocks in system zone - "
6455 err = ext4_mb_mark_context(handle, sb, false, block_group, bit,
6487 ext4_lock_group(sb, block_group);
6490 if (test_opt(sb, DISCARD)) {
6491 err = ext4_issue_discard(sb, block_group, bit,
6500 ext4_msg(sb, KERN_WARNING, "discard request in"
6507 ext4_lock_group(sb, block_group);
6511 ext4_unlock_group(sb, block_group);
6537 ext4_std_error(sb, err);
6553 struct super_block *sb = inode->i_sb;
6557 sbi = EXT4_SB(sb);
6575 ext4_error(sb, "Freeing blocks not in datazone - "
6645 * @sb: super block
6651 int ext4_group_add_blocks(handle_t *handle, struct super_block *sb,
6656 struct ext4_sb_info *sbi = EXT4_SB(sb);
6669 ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
6674 if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) {
6675 ext4_warning(sb, "too many blocks added to group %u",
6681 err = ext4_mb_load_buddy(sb, block_group, &e4b);
6685 if (!ext4_sb_block_valid(sb, NULL, block, count)) {
6686 ext4_error(sb, "Adding blocks in system zones - "
6693 err = ext4_mb_mark_context(handle, sb, false, block_group, bit,
6700 ext4_error(sb, "bit already cleared in group %u", block_group);
6702 ext4_lock_group(sb, block_group);
6704 ext4_unlock_group(sb, block_group);
6711 ext4_std_error(sb, err);
6717 * @sb: super block for the file system
6726 static int ext4_trim_extent(struct super_block *sb,
6735 trace_ext4_trim_extent(sb, group, start, count);
6737 assert_spin_locked(ext4_group_lock_ptr(sb, group));
6748 ext4_unlock_group(sb, group);
6749 ret = ext4_issue_discard(sb, group, start, count);
6750 ext4_lock_group(sb, group);
6755 static ext4_grpblk_t ext4_last_grp_cluster(struct super_block *sb,
6760 if (grp < (ext4_get_groups_count(sb) - 1))
6761 nr_clusters_in_group = EXT4_CLUSTERS_PER_GROUP(sb);
6763 nr_clusters_in_group = (ext4_blocks_count(EXT4_SB(sb)->s_es) -
6764 ext4_group_first_block_no(sb, grp))
6765 >> EXT4_CLUSTER_BITS(sb);
6775 static int ext4_try_to_trim_range(struct super_block *sb,
6778 __acquires(ext4_group_lock_ptr(sb, e4b->bd_group))
6779 __releases(ext4_group_lock_ptr(sb, e4b->bd_group))
6788 last = ext4_last_grp_cluster(sb, e4b->bd_group);
6807 int ret = ext4_trim_extent(sb, start, next - start, e4b);
6820 ext4_unlock_group(sb, e4b->bd_group);
6822 ext4_lock_group(sb, e4b->bd_group);
6837 * @sb: super block for file system
6849 ext4_trim_all_free(struct super_block *sb, ext4_group_t group,
6856 trace_ext4_trim_all_free(sb, group, start, max);
6858 ret = ext4_mb_load_buddy(sb, group, &e4b);
6860 ext4_warning(sb, "Error %d loading buddy information for %u",
6865 ext4_lock_group(sb, group);
6868 minblocks < EXT4_SB(sb)->s_last_trim_minblks)
6869 ret = ext4_try_to_trim_range(sb, &e4b, start, max, minblocks);
6873 ext4_unlock_group(sb, group);
6884 * @sb: superblock for filesystem
6894 int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
6896 unsigned int discard_granularity = bdev_discard_granularity(sb->s_bdev);
6902 le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
6903 ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);
6906 start = range->start >> sb->s_blocksize_bits;
6907 end = start + (range->len >> sb->s_blocksize_bits) - 1;
6908 minlen = EXT4_NUM_B2C(EXT4_SB(sb),
6909 range->minlen >> sb->s_blocksize_bits);
6911 if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) ||
6913 range->len < sb->s_blocksize)
6917 minlen = EXT4_NUM_B2C(EXT4_SB(sb),
6918 discard_granularity >> sb->s_blocksize_bits);
6919 if (minlen > EXT4_CLUSTERS_PER_GROUP(sb))
6930 ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start,
6932 ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end,
6936 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;
6941 grp = ext4_get_group_info(sb, group);
6946 ret = ext4_mb_init_group(sb, group, GFP_NOFS);
6953 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to
6960 cnt = ext4_trim_all_free(sb, group, first_cluster,
6977 EXT4_SB(sb)->s_last_trim_minblks = minlen;
6980 range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits;
6987 struct super_block *sb,
6999 error = ext4_mb_load_buddy(sb, group, &e4b);
7004 ext4_lock_group(sb, group);
7007 if (end >= EXT4_CLUSTERS_PER_GROUP(sb))
7008 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;
7016 ext4_unlock_group(sb, group);
7017 error = formatter(sb, group, start, next - start, priv);
7020 ext4_lock_group(sb, group);
7025 ext4_unlock_group(sb, group);