1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (C) 2022-2023 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6#include "xfs.h" 7#include "xfs_fs.h" 8#include "xfs_shared.h" 9#include "xfs_format.h" 10#include "xfs_trans_resv.h" 11#include "xfs_mount.h" 12#include "xfs_ag.h" 13#include "xfs_trace.h" 14 15/* 16 * Use a static key here to reduce the overhead of xfs_drain_rele. If the 17 * compiler supports jump labels, the static branch will be replaced by a nop 18 * sled when there are no xfs_drain_wait callers. Online fsck is currently 19 * the only caller, so this is a reasonable tradeoff. 20 * 21 * Note: Patching the kernel code requires taking the cpu hotplug lock. Other 22 * parts of the kernel allocate memory with that lock held, which means that 23 * XFS callers cannot hold any locks that might be used by memory reclaim or 24 * writeback when calling the static_branch_{inc,dec} functions. 25 */ 26static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate); 27 28void 29xfs_drain_wait_disable(void) 30{ 31 static_branch_dec(&xfs_drain_waiter_gate); 32} 33 34void 35xfs_drain_wait_enable(void) 36{ 37 static_branch_inc(&xfs_drain_waiter_gate); 38} 39 40void 41xfs_defer_drain_init( 42 struct xfs_defer_drain *dr) 43{ 44 atomic_set(&dr->dr_count, 0); 45 init_waitqueue_head(&dr->dr_waiters); 46} 47 48void 49xfs_defer_drain_free(struct xfs_defer_drain *dr) 50{ 51 ASSERT(atomic_read(&dr->dr_count) == 0); 52} 53 54/* Increase the pending intent count. */ 55static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr) 56{ 57 atomic_inc(&dr->dr_count); 58} 59 60static inline bool has_waiters(struct wait_queue_head *wq_head) 61{ 62 /* 63 * This memory barrier is paired with the one in set_current_state on 64 * the waiting side. 65 */ 66 smp_mb__after_atomic(); 67 return waitqueue_active(wq_head); 68} 69 70/* Decrease the pending intent count, and wake any waiters, if appropriate. */ 71static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr) 72{ 73 if (atomic_dec_and_test(&dr->dr_count) && 74 static_branch_unlikely(&xfs_drain_waiter_gate) && 75 has_waiters(&dr->dr_waiters)) 76 wake_up(&dr->dr_waiters); 77} 78 79/* Are there intents pending? */ 80static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr) 81{ 82 return atomic_read(&dr->dr_count) > 0; 83} 84 85/* 86 * Wait for the pending intent count for a drain to hit zero. 87 * 88 * Callers must not hold any locks that would prevent intents from being 89 * finished. 90 */ 91static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr) 92{ 93 return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr)); 94} 95 96/* 97 * Get a passive reference to an AG and declare an intent to update its 98 * metadata. 99 */ 100struct xfs_perag * 101xfs_perag_intent_get( 102 struct xfs_mount *mp, 103 xfs_agnumber_t agno) 104{ 105 struct xfs_perag *pag; 106 107 pag = xfs_perag_get(mp, agno); 108 if (!pag) 109 return NULL; 110 111 xfs_perag_intent_hold(pag); 112 return pag; 113} 114 115/* 116 * Release our intent to update this AG's metadata, and then release our 117 * passive ref to the AG. 118 */ 119void 120xfs_perag_intent_put( 121 struct xfs_perag *pag) 122{ 123 xfs_perag_intent_rele(pag); 124 xfs_perag_put(pag); 125} 126 127/* 128 * Declare an intent to update AG metadata. Other threads that need exclusive 129 * access can decide to back off if they see declared intentions. 130 */ 131void 132xfs_perag_intent_hold( 133 struct xfs_perag *pag) 134{ 135 trace_xfs_perag_intent_hold(pag, __return_address); 136 xfs_defer_drain_grab(&pag->pag_intents_drain); 137} 138 139/* Release our intent to update this AG's metadata. */ 140void 141xfs_perag_intent_rele( 142 struct xfs_perag *pag) 143{ 144 trace_xfs_perag_intent_rele(pag, __return_address); 145 xfs_defer_drain_rele(&pag->pag_intents_drain); 146} 147 148/* 149 * Wait for the intent update count for this AG to hit zero. 150 * Callers must not hold any AG header buffers. 151 */ 152int 153xfs_perag_intent_drain( 154 struct xfs_perag *pag) 155{ 156 trace_xfs_perag_wait_intents(pag, __return_address); 157 return xfs_defer_drain_wait(&pag->pag_intents_drain); 158} 159 160/* Has anyone declared an intent to update this AG? */ 161bool 162xfs_perag_intent_busy( 163 struct xfs_perag *pag) 164{ 165 return xfs_defer_drain_busy(&pag->pag_intents_drain); 166} 167