1/*
2 * Copyright (C) 2008 Oracle.  All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18#include <linux/sched.h>
19#include <linux/pagemap.h>
20#include <linux/spinlock.h>
21#include <linux/page-flags.h>
22#include <asm/bug.h>
23#include "ctree.h"
24#include "extent_io.h"
25#include "locking.h"
26
27static inline void spin_nested(struct extent_buffer *eb)
28{
29	spin_lock(&eb->lock);
30}
31
32/*
33 * Setting a lock to blocking will drop the spinlock and set the
34 * flag that forces other procs who want the lock to wait.  After
35 * this you can safely schedule with the lock held.
36 */
37void btrfs_set_lock_blocking(struct extent_buffer *eb)
38{
39	if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
40		set_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
41		spin_unlock(&eb->lock);
42	}
43	/* exit with the spin lock released and the bit set */
44}
45
46/*
47 * clearing the blocking flag will take the spinlock again.
48 * After this you can't safely schedule
49 */
50void btrfs_clear_lock_blocking(struct extent_buffer *eb)
51{
52	if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
53		spin_nested(eb);
54		clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
55		smp_mb__after_clear_bit();
56	}
57	/* exit with the spin lock held */
58}
59
60/*
61 * unfortunately, many of the places that currently set a lock to blocking
62 * don't end up blocking for very long, and often they don't block
63 * at all.  For a dbench 50 run, if we don't spin on the blocking bit
64 * at all, the context switch rate can jump up to 400,000/sec or more.
65 *
66 * So, we're still stuck with this crummy spin on the blocking bit,
67 * at least until the most common causes of the short blocks
68 * can be dealt with.
69 */
70static int btrfs_spin_on_block(struct extent_buffer *eb)
71{
72	int i;
73
74	for (i = 0; i < 512; i++) {
75		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
76			return 1;
77		if (need_resched())
78			break;
79		cpu_relax();
80	}
81	return 0;
82}
83
84/*
85 * This is somewhat different from trylock.  It will take the
86 * spinlock but if it finds the lock is set to blocking, it will
87 * return without the lock held.
88 *
89 * returns 1 if it was able to take the lock and zero otherwise
90 *
91 * After this call, scheduling is not safe without first calling
92 * btrfs_set_lock_blocking()
93 */
94int btrfs_try_spin_lock(struct extent_buffer *eb)
95{
96	int i;
97
98	if (btrfs_spin_on_block(eb)) {
99		spin_nested(eb);
100		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
101			return 1;
102		spin_unlock(&eb->lock);
103	}
104	/* spin for a bit on the BLOCKING flag */
105	for (i = 0; i < 2; i++) {
106		cpu_relax();
107		if (!btrfs_spin_on_block(eb))
108			break;
109
110		spin_nested(eb);
111		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
112			return 1;
113		spin_unlock(&eb->lock);
114	}
115	return 0;
116}
117
118/*
119 * the autoremove wake function will return 0 if it tried to wake up
120 * a process that was already awake, which means that process won't
121 * count as an exclusive wakeup.  The waitq code will continue waking
122 * procs until it finds one that was actually sleeping.
123 *
124 * For btrfs, this isn't quite what we want.  We want a single proc
125 * to be notified that the lock is ready for taking.  If that proc
126 * already happen to be awake, great, it will loop around and try for
127 * the lock.
128 *
129 * So, btrfs_wake_function always returns 1, even when the proc that we
130 * tried to wake up was already awake.
131 */
132static int btrfs_wake_function(wait_queue_t *wait, unsigned mode,
133			       int sync, void *key)
134{
135	autoremove_wake_function(wait, mode, sync, key);
136	return 1;
137}
138
139/*
140 * returns with the extent buffer spinlocked.
141 *
142 * This will spin and/or wait as required to take the lock, and then
143 * return with the spinlock held.
144 *
145 * After this call, scheduling is not safe without first calling
146 * btrfs_set_lock_blocking()
147 */
148int btrfs_tree_lock(struct extent_buffer *eb)
149{
150	DEFINE_WAIT(wait);
151	wait.func = btrfs_wake_function;
152
153	if (!btrfs_spin_on_block(eb))
154		goto sleep;
155
156	while(1) {
157		spin_nested(eb);
158
159		/* nobody is blocking, exit with the spinlock held */
160		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
161			return 0;
162
163		/*
164		 * we have the spinlock, but the real owner is blocking.
165		 * wait for them
166		 */
167		spin_unlock(&eb->lock);
168
169		/*
170		 * spin for a bit, and if the blocking flag goes away,
171		 * loop around
172		 */
173		cpu_relax();
174		if (btrfs_spin_on_block(eb))
175			continue;
176sleep:
177		prepare_to_wait_exclusive(&eb->lock_wq, &wait,
178					  TASK_UNINTERRUPTIBLE);
179
180		if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
181			schedule();
182
183		finish_wait(&eb->lock_wq, &wait);
184	}
185	return 0;
186}
187
188/*
189 * Very quick trylock, this does not spin or schedule.  It returns
190 * 1 with the spinlock held if it was able to take the lock, or it
191 * returns zero if it was unable to take the lock.
192 *
193 * After this call, scheduling is not safe without first calling
194 * btrfs_set_lock_blocking()
195 */
196int btrfs_try_tree_lock(struct extent_buffer *eb)
197{
198	if (spin_trylock(&eb->lock)) {
199		if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
200			/*
201			 * we've got the spinlock, but the real owner is
202			 * blocking.  Drop the spinlock and return failure
203			 */
204			spin_unlock(&eb->lock);
205			return 0;
206		}
207		return 1;
208	}
209	/* someone else has the spinlock giveup */
210	return 0;
211}
212
213int btrfs_tree_unlock(struct extent_buffer *eb)
214{
215	/*
216	 * if we were a blocking owner, we don't have the spinlock held
217	 * just clear the bit and look for waiters
218	 */
219	if (test_and_clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
220		smp_mb__after_clear_bit();
221	else
222		spin_unlock(&eb->lock);
223
224	if (waitqueue_active(&eb->lock_wq))
225		wake_up(&eb->lock_wq);
226	return 0;
227}
228
229void btrfs_assert_tree_locked(struct extent_buffer *eb)
230{
231	if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
232		assert_spin_locked(&eb->lock);
233}
234