glock.c revision 08d73666
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) Sistina Software, Inc.  1997-2003 All rights reserved.
4 * Copyright (C) 2004-2008 Red Hat, Inc.  All rights reserved.
5 */
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/sched.h>
10#include <linux/slab.h>
11#include <linux/spinlock.h>
12#include <linux/buffer_head.h>
13#include <linux/delay.h>
14#include <linux/sort.h>
15#include <linux/hash.h>
16#include <linux/jhash.h>
17#include <linux/kallsyms.h>
18#include <linux/gfs2_ondisk.h>
19#include <linux/list.h>
20#include <linux/wait.h>
21#include <linux/module.h>
22#include <linux/uaccess.h>
23#include <linux/seq_file.h>
24#include <linux/debugfs.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
27#include <linux/workqueue.h>
28#include <linux/jiffies.h>
29#include <linux/rcupdate.h>
30#include <linux/rculist_bl.h>
31#include <linux/bit_spinlock.h>
32#include <linux/percpu.h>
33#include <linux/list_sort.h>
34#include <linux/lockref.h>
35#include <linux/rhashtable.h>
36
37#include "gfs2.h"
38#include "incore.h"
39#include "glock.h"
40#include "glops.h"
41#include "inode.h"
42#include "lops.h"
43#include "meta_io.h"
44#include "quota.h"
45#include "super.h"
46#include "util.h"
47#include "bmap.h"
48#define CREATE_TRACE_POINTS
49#include "trace_gfs2.h"
50
51struct gfs2_glock_iter {
52	struct gfs2_sbd *sdp;		/* incore superblock           */
53	struct rhashtable_iter hti;	/* rhashtable iterator         */
54	struct gfs2_glock *gl;		/* current glock struct        */
55	loff_t last_pos;		/* last position               */
56};
57
58typedef void (*glock_examiner) (struct gfs2_glock * gl);
59
60static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61
62static struct dentry *gfs2_root;
63static struct workqueue_struct *glock_workqueue;
64struct workqueue_struct *gfs2_delete_workqueue;
65static LIST_HEAD(lru_list);
66static atomic_t lru_count = ATOMIC_INIT(0);
67static DEFINE_SPINLOCK(lru_lock);
68
69#define GFS2_GL_HASH_SHIFT      15
70#define GFS2_GL_HASH_SIZE       BIT(GFS2_GL_HASH_SHIFT)
71
72static const struct rhashtable_params ht_parms = {
73	.nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
74	.key_len = offsetofend(struct lm_lockname, ln_type),
75	.key_offset = offsetof(struct gfs2_glock, gl_name),
76	.head_offset = offsetof(struct gfs2_glock, gl_node),
77};
78
79static struct rhashtable gl_hash_table;
80
81#define GLOCK_WAIT_TABLE_BITS 12
82#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
83static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
84
85struct wait_glock_queue {
86	struct lm_lockname *name;
87	wait_queue_entry_t wait;
88};
89
90static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
91			       int sync, void *key)
92{
93	struct wait_glock_queue *wait_glock =
94		container_of(wait, struct wait_glock_queue, wait);
95	struct lm_lockname *wait_name = wait_glock->name;
96	struct lm_lockname *wake_name = key;
97
98	if (wake_name->ln_sbd != wait_name->ln_sbd ||
99	    wake_name->ln_number != wait_name->ln_number ||
100	    wake_name->ln_type != wait_name->ln_type)
101		return 0;
102	return autoremove_wake_function(wait, mode, sync, key);
103}
104
105static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
106{
107	u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
108
109	return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
110}
111
112/**
113 * wake_up_glock  -  Wake up waiters on a glock
114 * @gl: the glock
115 */
116static void wake_up_glock(struct gfs2_glock *gl)
117{
118	wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
119
120	if (waitqueue_active(wq))
121		__wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
122}
123
124static void gfs2_glock_dealloc(struct rcu_head *rcu)
125{
126	struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
127
128	kfree(gl->gl_lksb.sb_lvbptr);
129	if (gl->gl_ops->go_flags & GLOF_ASPACE)
130		kmem_cache_free(gfs2_glock_aspace_cachep, gl);
131	else
132		kmem_cache_free(gfs2_glock_cachep, gl);
133}
134
135/**
136 * glock_blocked_by_withdraw - determine if we can still use a glock
137 * @gl: the glock
138 *
139 * We need to allow some glocks to be enqueued, dequeued, promoted, and demoted
140 * when we're withdrawn. For example, to maintain metadata integrity, we should
141 * disallow the use of inode and rgrp glocks when withdrawn. Other glocks, like
142 * iopen or the transaction glocks may be safely used because none of their
143 * metadata goes through the journal. So in general, we should disallow all
144 * glocks that are journaled, and allow all the others. One exception is:
145 * we need to allow our active journal to be promoted and demoted so others
146 * may recover it and we can reacquire it when they're done.
147 */
148static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
149{
150	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
151
152	if (likely(!gfs2_withdrawn(sdp)))
153		return false;
154	if (gl->gl_ops->go_flags & GLOF_NONDISK)
155		return false;
156	if (!sdp->sd_jdesc ||
157	    gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
158		return false;
159	return true;
160}
161
162void gfs2_glock_free(struct gfs2_glock *gl)
163{
164	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
165
166	gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0);
167	rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
168	smp_mb();
169	wake_up_glock(gl);
170	call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
171	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
172		wake_up(&sdp->sd_glock_wait);
173}
174
175/**
176 * gfs2_glock_hold() - increment reference count on glock
177 * @gl: The glock to hold
178 *
179 */
180
181void gfs2_glock_hold(struct gfs2_glock *gl)
182{
183	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
184	lockref_get(&gl->gl_lockref);
185}
186
187/**
188 * demote_ok - Check to see if it's ok to unlock a glock
189 * @gl: the glock
190 *
191 * Returns: 1 if it's ok
192 */
193
194static int demote_ok(const struct gfs2_glock *gl)
195{
196	const struct gfs2_glock_operations *glops = gl->gl_ops;
197
198	if (gl->gl_state == LM_ST_UNLOCKED)
199		return 0;
200	if (!list_empty(&gl->gl_holders))
201		return 0;
202	if (glops->go_demote_ok)
203		return glops->go_demote_ok(gl);
204	return 1;
205}
206
207
208void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
209{
210	if (!(gl->gl_ops->go_flags & GLOF_LRU))
211		return;
212
213	spin_lock(&lru_lock);
214
215	list_move_tail(&gl->gl_lru, &lru_list);
216
217	if (!test_bit(GLF_LRU, &gl->gl_flags)) {
218		set_bit(GLF_LRU, &gl->gl_flags);
219		atomic_inc(&lru_count);
220	}
221
222	spin_unlock(&lru_lock);
223}
224
225static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
226{
227	if (!(gl->gl_ops->go_flags & GLOF_LRU))
228		return;
229
230	spin_lock(&lru_lock);
231	if (test_bit(GLF_LRU, &gl->gl_flags)) {
232		list_del_init(&gl->gl_lru);
233		atomic_dec(&lru_count);
234		clear_bit(GLF_LRU, &gl->gl_flags);
235	}
236	spin_unlock(&lru_lock);
237}
238
239/*
240 * Enqueue the glock on the work queue.  Passes one glock reference on to the
241 * work queue.
242 */
243static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
244	if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
245		/*
246		 * We are holding the lockref spinlock, and the work was still
247		 * queued above.  The queued work (glock_work_func) takes that
248		 * spinlock before dropping its glock reference(s), so it
249		 * cannot have dropped them in the meantime.
250		 */
251		GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
252		gl->gl_lockref.count--;
253	}
254}
255
256static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
257	spin_lock(&gl->gl_lockref.lock);
258	__gfs2_glock_queue_work(gl, delay);
259	spin_unlock(&gl->gl_lockref.lock);
260}
261
262static void __gfs2_glock_put(struct gfs2_glock *gl)
263{
264	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
265	struct address_space *mapping = gfs2_glock2aspace(gl);
266
267	lockref_mark_dead(&gl->gl_lockref);
268
269	gfs2_glock_remove_from_lru(gl);
270	spin_unlock(&gl->gl_lockref.lock);
271	GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
272	if (mapping) {
273		truncate_inode_pages_final(mapping);
274		if (!gfs2_withdrawn(sdp))
275			GLOCK_BUG_ON(gl, !mapping_empty(mapping));
276	}
277	trace_gfs2_glock_put(gl);
278	sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
279}
280
281/*
282 * Cause the glock to be put in work queue context.
283 */
284void gfs2_glock_queue_put(struct gfs2_glock *gl)
285{
286	gfs2_glock_queue_work(gl, 0);
287}
288
289/**
290 * gfs2_glock_put() - Decrement reference count on glock
291 * @gl: The glock to put
292 *
293 */
294
295void gfs2_glock_put(struct gfs2_glock *gl)
296{
297	if (lockref_put_or_lock(&gl->gl_lockref))
298		return;
299
300	__gfs2_glock_put(gl);
301}
302
303/**
304 * may_grant - check if its ok to grant a new lock
305 * @gl: The glock
306 * @gh: The lock request which we wish to grant
307 *
308 * Returns: true if its ok to grant the lock
309 */
310
311static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)
312{
313	const struct gfs2_holder *gh_head = list_first_entry(&gl->gl_holders, const struct gfs2_holder, gh_list);
314
315	if (gh != gh_head) {
316		/**
317		 * Here we make a special exception to grant holders who agree
318		 * to share the EX lock with other holders who also have the
319		 * bit set. If the original holder has the LM_FLAG_NODE_SCOPE bit
320		 * is set, we grant more holders with the bit set.
321		 */
322		if (gh_head->gh_state == LM_ST_EXCLUSIVE &&
323		    (gh_head->gh_flags & LM_FLAG_NODE_SCOPE) &&
324		    gh->gh_state == LM_ST_EXCLUSIVE &&
325		    (gh->gh_flags & LM_FLAG_NODE_SCOPE))
326			return 1;
327		if ((gh->gh_state == LM_ST_EXCLUSIVE ||
328		     gh_head->gh_state == LM_ST_EXCLUSIVE))
329			return 0;
330	}
331	if (gl->gl_state == gh->gh_state)
332		return 1;
333	if (gh->gh_flags & GL_EXACT)
334		return 0;
335	if (gl->gl_state == LM_ST_EXCLUSIVE) {
336		if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
337			return 1;
338		if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
339			return 1;
340	}
341	if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))
342		return 1;
343	return 0;
344}
345
346static void gfs2_holder_wake(struct gfs2_holder *gh)
347{
348	clear_bit(HIF_WAIT, &gh->gh_iflags);
349	smp_mb__after_atomic();
350	wake_up_bit(&gh->gh_iflags, HIF_WAIT);
351	if (gh->gh_flags & GL_ASYNC) {
352		struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
353
354		wake_up(&sdp->sd_async_glock_wait);
355	}
356}
357
358/**
359 * do_error - Something unexpected has happened during a lock request
360 * @gl: The glock
361 * @ret: The status from the DLM
362 */
363
364static void do_error(struct gfs2_glock *gl, const int ret)
365{
366	struct gfs2_holder *gh, *tmp;
367
368	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
369		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
370			continue;
371		if (ret & LM_OUT_ERROR)
372			gh->gh_error = -EIO;
373		else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
374			gh->gh_error = GLR_TRYFAILED;
375		else
376			continue;
377		list_del_init(&gh->gh_list);
378		trace_gfs2_glock_queue(gh, 0);
379		gfs2_holder_wake(gh);
380	}
381}
382
383/**
384 * do_promote - promote as many requests as possible on the current queue
385 * @gl: The glock
386 *
387 * Returns: 1 if there is a blocked holder at the head of the list, or 2
388 *          if a type specific operation is underway.
389 */
390
391static int do_promote(struct gfs2_glock *gl)
392__releases(&gl->gl_lockref.lock)
393__acquires(&gl->gl_lockref.lock)
394{
395	const struct gfs2_glock_operations *glops = gl->gl_ops;
396	struct gfs2_holder *gh, *tmp;
397	int ret;
398
399restart:
400	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
401		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
402			continue;
403		if (may_grant(gl, gh)) {
404			if (gh->gh_list.prev == &gl->gl_holders &&
405			    glops->go_lock) {
406				spin_unlock(&gl->gl_lockref.lock);
407				/* FIXME: eliminate this eventually */
408				ret = glops->go_lock(gh);
409				spin_lock(&gl->gl_lockref.lock);
410				if (ret) {
411					if (ret == 1)
412						return 2;
413					gh->gh_error = ret;
414					list_del_init(&gh->gh_list);
415					trace_gfs2_glock_queue(gh, 0);
416					gfs2_holder_wake(gh);
417					goto restart;
418				}
419				set_bit(HIF_HOLDER, &gh->gh_iflags);
420				trace_gfs2_promote(gh, 1);
421				gfs2_holder_wake(gh);
422				goto restart;
423			}
424			set_bit(HIF_HOLDER, &gh->gh_iflags);
425			trace_gfs2_promote(gh, 0);
426			gfs2_holder_wake(gh);
427			continue;
428		}
429		if (gh->gh_list.prev == &gl->gl_holders)
430			return 1;
431		do_error(gl, 0);
432		break;
433	}
434	return 0;
435}
436
437/**
438 * find_first_waiter - find the first gh that's waiting for the glock
439 * @gl: the glock
440 */
441
442static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
443{
444	struct gfs2_holder *gh;
445
446	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
447		if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
448			return gh;
449	}
450	return NULL;
451}
452
453/**
454 * state_change - record that the glock is now in a different state
455 * @gl: the glock
456 * @new_state: the new state
457 */
458
459static void state_change(struct gfs2_glock *gl, unsigned int new_state)
460{
461	int held1, held2;
462
463	held1 = (gl->gl_state != LM_ST_UNLOCKED);
464	held2 = (new_state != LM_ST_UNLOCKED);
465
466	if (held1 != held2) {
467		GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
468		if (held2)
469			gl->gl_lockref.count++;
470		else
471			gl->gl_lockref.count--;
472	}
473	if (new_state != gl->gl_target)
474		/* shorten our minimum hold time */
475		gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
476				       GL_GLOCK_MIN_HOLD);
477	gl->gl_state = new_state;
478	gl->gl_tchange = jiffies;
479}
480
481static void gfs2_set_demote(struct gfs2_glock *gl)
482{
483	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
484
485	set_bit(GLF_DEMOTE, &gl->gl_flags);
486	smp_mb();
487	wake_up(&sdp->sd_async_glock_wait);
488}
489
490static void gfs2_demote_wake(struct gfs2_glock *gl)
491{
492	gl->gl_demote_state = LM_ST_EXCLUSIVE;
493	clear_bit(GLF_DEMOTE, &gl->gl_flags);
494	smp_mb__after_atomic();
495	wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
496}
497
498/**
499 * finish_xmote - The DLM has replied to one of our lock requests
500 * @gl: The glock
501 * @ret: The status from the DLM
502 *
503 */
504
505static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
506{
507	const struct gfs2_glock_operations *glops = gl->gl_ops;
508	struct gfs2_holder *gh;
509	unsigned state = ret & LM_OUT_ST_MASK;
510	int rv;
511
512	spin_lock(&gl->gl_lockref.lock);
513	trace_gfs2_glock_state_change(gl, state);
514	state_change(gl, state);
515	gh = find_first_waiter(gl);
516
517	/* Demote to UN request arrived during demote to SH or DF */
518	if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
519	    state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
520		gl->gl_target = LM_ST_UNLOCKED;
521
522	/* Check for state != intended state */
523	if (unlikely(state != gl->gl_target)) {
524		if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
525			/* move to back of queue and try next entry */
526			if (ret & LM_OUT_CANCELED) {
527				if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
528					list_move_tail(&gh->gh_list, &gl->gl_holders);
529				gh = find_first_waiter(gl);
530				gl->gl_target = gh->gh_state;
531				goto retry;
532			}
533			/* Some error or failed "try lock" - report it */
534			if ((ret & LM_OUT_ERROR) ||
535			    (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
536				gl->gl_target = gl->gl_state;
537				do_error(gl, ret);
538				goto out;
539			}
540		}
541		switch(state) {
542		/* Unlocked due to conversion deadlock, try again */
543		case LM_ST_UNLOCKED:
544retry:
545			do_xmote(gl, gh, gl->gl_target);
546			break;
547		/* Conversion fails, unlock and try again */
548		case LM_ST_SHARED:
549		case LM_ST_DEFERRED:
550			do_xmote(gl, gh, LM_ST_UNLOCKED);
551			break;
552		default: /* Everything else */
553			fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
554			       gl->gl_target, state);
555			GLOCK_BUG_ON(gl, 1);
556		}
557		spin_unlock(&gl->gl_lockref.lock);
558		return;
559	}
560
561	/* Fast path - we got what we asked for */
562	if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
563		gfs2_demote_wake(gl);
564	if (state != LM_ST_UNLOCKED) {
565		if (glops->go_xmote_bh) {
566			spin_unlock(&gl->gl_lockref.lock);
567			rv = glops->go_xmote_bh(gl);
568			spin_lock(&gl->gl_lockref.lock);
569			if (rv) {
570				do_error(gl, rv);
571				goto out;
572			}
573		}
574		rv = do_promote(gl);
575		if (rv == 2)
576			goto out_locked;
577	}
578out:
579	clear_bit(GLF_LOCK, &gl->gl_flags);
580out_locked:
581	spin_unlock(&gl->gl_lockref.lock);
582}
583
584static bool is_system_glock(struct gfs2_glock *gl)
585{
586	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
587	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
588
589	if (gl == m_ip->i_gl)
590		return true;
591	return false;
592}
593
594/**
595 * do_xmote - Calls the DLM to change the state of a lock
596 * @gl: The lock state
597 * @gh: The holder (only for promotes)
598 * @target: The target lock state
599 *
600 */
601
602static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
603__releases(&gl->gl_lockref.lock)
604__acquires(&gl->gl_lockref.lock)
605{
606	const struct gfs2_glock_operations *glops = gl->gl_ops;
607	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
608	unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
609	int ret;
610
611	if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
612	    gh && !(gh->gh_flags & LM_FLAG_NOEXP))
613		return;
614	lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
615		      LM_FLAG_PRIORITY);
616	GLOCK_BUG_ON(gl, gl->gl_state == target);
617	GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
618	if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
619	    glops->go_inval) {
620		/*
621		 * If another process is already doing the invalidate, let that
622		 * finish first.  The glock state machine will get back to this
623		 * holder again later.
624		 */
625		if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
626				     &gl->gl_flags))
627			return;
628		do_error(gl, 0); /* Fail queued try locks */
629	}
630	gl->gl_req = target;
631	set_bit(GLF_BLOCKING, &gl->gl_flags);
632	if ((gl->gl_req == LM_ST_UNLOCKED) ||
633	    (gl->gl_state == LM_ST_EXCLUSIVE) ||
634	    (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
635		clear_bit(GLF_BLOCKING, &gl->gl_flags);
636	spin_unlock(&gl->gl_lockref.lock);
637	if (glops->go_sync) {
638		ret = glops->go_sync(gl);
639		/* If we had a problem syncing (due to io errors or whatever,
640		 * we should not invalidate the metadata or tell dlm to
641		 * release the glock to other nodes.
642		 */
643		if (ret) {
644			if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
645				fs_err(sdp, "Error %d syncing glock \n", ret);
646				gfs2_dump_glock(NULL, gl, true);
647			}
648			goto skip_inval;
649		}
650	}
651	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
652		/*
653		 * The call to go_sync should have cleared out the ail list.
654		 * If there are still items, we have a problem. We ought to
655		 * withdraw, but we can't because the withdraw code also uses
656		 * glocks. Warn about the error, dump the glock, then fall
657		 * through and wait for logd to do the withdraw for us.
658		 */
659		if ((atomic_read(&gl->gl_ail_count) != 0) &&
660		    (!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
661			gfs2_glock_assert_warn(gl,
662					       !atomic_read(&gl->gl_ail_count));
663			gfs2_dump_glock(NULL, gl, true);
664		}
665		glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
666		clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
667	}
668
669skip_inval:
670	gfs2_glock_hold(gl);
671	/*
672	 * Check for an error encountered since we called go_sync and go_inval.
673	 * If so, we can't withdraw from the glock code because the withdraw
674	 * code itself uses glocks (see function signal_our_withdraw) to
675	 * change the mount to read-only. Most importantly, we must not call
676	 * dlm to unlock the glock until the journal is in a known good state
677	 * (after journal replay) otherwise other nodes may use the object
678	 * (rgrp or dinode) and then later, journal replay will corrupt the
679	 * file system. The best we can do here is wait for the logd daemon
680	 * to see sd_log_error and withdraw, and in the meantime, requeue the
681	 * work for later.
682	 *
683	 * We make a special exception for some system glocks, such as the
684	 * system statfs inode glock, which needs to be granted before the
685	 * gfs2_quotad daemon can exit, and that exit needs to finish before
686	 * we can unmount the withdrawn file system.
687	 *
688	 * However, if we're just unlocking the lock (say, for unmount, when
689	 * gfs2_gl_hash_clear calls clear_glock) and recovery is complete
690	 * then it's okay to tell dlm to unlock it.
691	 */
692	if (unlikely(sdp->sd_log_error && !gfs2_withdrawn(sdp)))
693		gfs2_withdraw_delayed(sdp);
694	if (glock_blocked_by_withdraw(gl) &&
695	    (target != LM_ST_UNLOCKED ||
696	     test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
697		if (!is_system_glock(gl)) {
698			gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
699			goto out;
700		} else {
701			clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
702		}
703	}
704
705	if (sdp->sd_lockstruct.ls_ops->lm_lock)	{
706		/* lock_dlm */
707		ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
708		if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
709		    target == LM_ST_UNLOCKED &&
710		    test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
711			finish_xmote(gl, target);
712			gfs2_glock_queue_work(gl, 0);
713		} else if (ret) {
714			fs_err(sdp, "lm_lock ret %d\n", ret);
715			GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
716		}
717	} else { /* lock_nolock */
718		finish_xmote(gl, target);
719		gfs2_glock_queue_work(gl, 0);
720	}
721out:
722	spin_lock(&gl->gl_lockref.lock);
723}
724
725/**
726 * find_first_holder - find the first "holder" gh
727 * @gl: the glock
728 */
729
730static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
731{
732	struct gfs2_holder *gh;
733
734	if (!list_empty(&gl->gl_holders)) {
735		gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
736		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
737			return gh;
738	}
739	return NULL;
740}
741
742/**
743 * run_queue - do all outstanding tasks related to a glock
744 * @gl: The glock in question
745 * @nonblock: True if we must not block in run_queue
746 *
747 */
748
749static void run_queue(struct gfs2_glock *gl, const int nonblock)
750__releases(&gl->gl_lockref.lock)
751__acquires(&gl->gl_lockref.lock)
752{
753	struct gfs2_holder *gh = NULL;
754	int ret;
755
756	if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
757		return;
758
759	GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
760
761	if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
762	    gl->gl_demote_state != gl->gl_state) {
763		if (find_first_holder(gl))
764			goto out_unlock;
765		if (nonblock)
766			goto out_sched;
767		set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
768		GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
769		gl->gl_target = gl->gl_demote_state;
770	} else {
771		if (test_bit(GLF_DEMOTE, &gl->gl_flags))
772			gfs2_demote_wake(gl);
773		ret = do_promote(gl);
774		if (ret == 0)
775			goto out_unlock;
776		if (ret == 2)
777			goto out;
778		gh = find_first_waiter(gl);
779		gl->gl_target = gh->gh_state;
780		if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
781			do_error(gl, 0); /* Fail queued try locks */
782	}
783	do_xmote(gl, gh, gl->gl_target);
784out:
785	return;
786
787out_sched:
788	clear_bit(GLF_LOCK, &gl->gl_flags);
789	smp_mb__after_atomic();
790	gl->gl_lockref.count++;
791	__gfs2_glock_queue_work(gl, 0);
792	return;
793
794out_unlock:
795	clear_bit(GLF_LOCK, &gl->gl_flags);
796	smp_mb__after_atomic();
797	return;
798}
799
800void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
801{
802	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
803
804	if (ri->ri_magic == 0)
805		ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
806	if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
807		ri->ri_generation_deleted = cpu_to_be64(generation);
808}
809
810bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
811{
812	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
813
814	if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
815		return false;
816	return generation <= be64_to_cpu(ri->ri_generation_deleted);
817}
818
819static void gfs2_glock_poke(struct gfs2_glock *gl)
820{
821	int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
822	struct gfs2_holder gh;
823	int error;
824
825	gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh);
826	error = gfs2_glock_nq(&gh);
827	if (!error)
828		gfs2_glock_dq(&gh);
829	gfs2_holder_uninit(&gh);
830}
831
832static bool gfs2_try_evict(struct gfs2_glock *gl)
833{
834	struct gfs2_inode *ip;
835	bool evicted = false;
836
837	/*
838	 * If there is contention on the iopen glock and we have an inode, try
839	 * to grab and release the inode so that it can be evicted.  This will
840	 * allow the remote node to go ahead and delete the inode without us
841	 * having to do it, which will avoid rgrp glock thrashing.
842	 *
843	 * The remote node is likely still holding the corresponding inode
844	 * glock, so it will run before we get to verify that the delete has
845	 * happened below.
846	 */
847	spin_lock(&gl->gl_lockref.lock);
848	ip = gl->gl_object;
849	if (ip && !igrab(&ip->i_inode))
850		ip = NULL;
851	spin_unlock(&gl->gl_lockref.lock);
852	if (ip) {
853		struct gfs2_glock *inode_gl = NULL;
854
855		gl->gl_no_formal_ino = ip->i_no_formal_ino;
856		set_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
857		d_prune_aliases(&ip->i_inode);
858		iput(&ip->i_inode);
859
860		/* If the inode was evicted, gl->gl_object will now be NULL. */
861		spin_lock(&gl->gl_lockref.lock);
862		ip = gl->gl_object;
863		if (ip) {
864			inode_gl = ip->i_gl;
865			lockref_get(&inode_gl->gl_lockref);
866			clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
867		}
868		spin_unlock(&gl->gl_lockref.lock);
869		if (inode_gl) {
870			gfs2_glock_poke(inode_gl);
871			gfs2_glock_put(inode_gl);
872		}
873		evicted = !ip;
874	}
875	return evicted;
876}
877
878static void delete_work_func(struct work_struct *work)
879{
880	struct delayed_work *dwork = to_delayed_work(work);
881	struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
882	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
883	struct inode *inode;
884	u64 no_addr = gl->gl_name.ln_number;
885
886	spin_lock(&gl->gl_lockref.lock);
887	clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
888	spin_unlock(&gl->gl_lockref.lock);
889
890	if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
891		/*
892		 * If we can evict the inode, give the remote node trying to
893		 * delete the inode some time before verifying that the delete
894		 * has happened.  Otherwise, if we cause contention on the inode glock
895		 * immediately, the remote node will think that we still have
896		 * the inode in use, and so it will give up waiting.
897		 *
898		 * If we can't evict the inode, signal to the remote node that
899		 * the inode is still in use.  We'll later try to delete the
900		 * inode locally in gfs2_evict_inode.
901		 *
902		 * FIXME: We only need to verify that the remote node has
903		 * deleted the inode because nodes before this remote delete
904		 * rework won't cooperate.  At a later time, when we no longer
905		 * care about compatibility with such nodes, we can skip this
906		 * step entirely.
907		 */
908		if (gfs2_try_evict(gl)) {
909			if (gfs2_queue_delete_work(gl, 5 * HZ))
910				return;
911		}
912		goto out;
913	}
914
915	inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
916				    GFS2_BLKST_UNLINKED);
917	if (!IS_ERR_OR_NULL(inode)) {
918		d_prune_aliases(inode);
919		iput(inode);
920	}
921out:
922	gfs2_glock_put(gl);
923}
924
925static void glock_work_func(struct work_struct *work)
926{
927	unsigned long delay = 0;
928	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
929	unsigned int drop_refs = 1;
930
931	if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
932		finish_xmote(gl, gl->gl_reply);
933		drop_refs++;
934	}
935	spin_lock(&gl->gl_lockref.lock);
936	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
937	    gl->gl_state != LM_ST_UNLOCKED &&
938	    gl->gl_demote_state != LM_ST_EXCLUSIVE) {
939		unsigned long holdtime, now = jiffies;
940
941		holdtime = gl->gl_tchange + gl->gl_hold_time;
942		if (time_before(now, holdtime))
943			delay = holdtime - now;
944
945		if (!delay) {
946			clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
947			gfs2_set_demote(gl);
948		}
949	}
950	run_queue(gl, 0);
951	if (delay) {
952		/* Keep one glock reference for the work we requeue. */
953		drop_refs--;
954		if (gl->gl_name.ln_type != LM_TYPE_INODE)
955			delay = 0;
956		__gfs2_glock_queue_work(gl, delay);
957	}
958
959	/*
960	 * Drop the remaining glock references manually here. (Mind that
961	 * __gfs2_glock_queue_work depends on the lockref spinlock begin held
962	 * here as well.)
963	 */
964	gl->gl_lockref.count -= drop_refs;
965	if (!gl->gl_lockref.count) {
966		__gfs2_glock_put(gl);
967		return;
968	}
969	spin_unlock(&gl->gl_lockref.lock);
970}
971
972static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
973					    struct gfs2_glock *new)
974{
975	struct wait_glock_queue wait;
976	wait_queue_head_t *wq = glock_waitqueue(name);
977	struct gfs2_glock *gl;
978
979	wait.name = name;
980	init_wait(&wait.wait);
981	wait.wait.func = glock_wake_function;
982
983again:
984	prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
985	rcu_read_lock();
986	if (new) {
987		gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
988			&new->gl_node, ht_parms);
989		if (IS_ERR(gl))
990			goto out;
991	} else {
992		gl = rhashtable_lookup_fast(&gl_hash_table,
993			name, ht_parms);
994	}
995	if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
996		rcu_read_unlock();
997		schedule();
998		goto again;
999	}
1000out:
1001	rcu_read_unlock();
1002	finish_wait(wq, &wait.wait);
1003	return gl;
1004}
1005
1006/**
1007 * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
1008 * @sdp: The GFS2 superblock
1009 * @number: the lock number
1010 * @glops: The glock_operations to use
1011 * @create: If 0, don't create the glock if it doesn't exist
1012 * @glp: the glock is returned here
1013 *
1014 * This does not lock a glock, just finds/creates structures for one.
1015 *
1016 * Returns: errno
1017 */
1018
1019int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1020		   const struct gfs2_glock_operations *glops, int create,
1021		   struct gfs2_glock **glp)
1022{
1023	struct super_block *s = sdp->sd_vfs;
1024	struct lm_lockname name = { .ln_number = number,
1025				    .ln_type = glops->go_type,
1026				    .ln_sbd = sdp };
1027	struct gfs2_glock *gl, *tmp;
1028	struct address_space *mapping;
1029	struct kmem_cache *cachep;
1030	int ret = 0;
1031
1032	gl = find_insert_glock(&name, NULL);
1033	if (gl) {
1034		*glp = gl;
1035		return 0;
1036	}
1037	if (!create)
1038		return -ENOENT;
1039
1040	if (glops->go_flags & GLOF_ASPACE)
1041		cachep = gfs2_glock_aspace_cachep;
1042	else
1043		cachep = gfs2_glock_cachep;
1044	gl = kmem_cache_alloc(cachep, GFP_NOFS);
1045	if (!gl)
1046		return -ENOMEM;
1047
1048	memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1049
1050	if (glops->go_flags & GLOF_LVB) {
1051		gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1052		if (!gl->gl_lksb.sb_lvbptr) {
1053			kmem_cache_free(cachep, gl);
1054			return -ENOMEM;
1055		}
1056	}
1057
1058	atomic_inc(&sdp->sd_glock_disposal);
1059	gl->gl_node.next = NULL;
1060	gl->gl_flags = 0;
1061	gl->gl_name = name;
1062	lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1063	gl->gl_lockref.count = 1;
1064	gl->gl_state = LM_ST_UNLOCKED;
1065	gl->gl_target = LM_ST_UNLOCKED;
1066	gl->gl_demote_state = LM_ST_EXCLUSIVE;
1067	gl->gl_ops = glops;
1068	gl->gl_dstamp = 0;
1069	preempt_disable();
1070	/* We use the global stats to estimate the initial per-glock stats */
1071	gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1072	preempt_enable();
1073	gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1074	gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1075	gl->gl_tchange = jiffies;
1076	gl->gl_object = NULL;
1077	gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1078	INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1079	if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1080		INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1081
1082	mapping = gfs2_glock2aspace(gl);
1083	if (mapping) {
1084                mapping->a_ops = &gfs2_meta_aops;
1085		mapping->host = s->s_bdev->bd_inode;
1086		mapping->flags = 0;
1087		mapping_set_gfp_mask(mapping, GFP_NOFS);
1088		mapping->private_data = NULL;
1089		mapping->writeback_index = 0;
1090	}
1091
1092	tmp = find_insert_glock(&name, gl);
1093	if (!tmp) {
1094		*glp = gl;
1095		goto out;
1096	}
1097	if (IS_ERR(tmp)) {
1098		ret = PTR_ERR(tmp);
1099		goto out_free;
1100	}
1101	*glp = tmp;
1102
1103out_free:
1104	kfree(gl->gl_lksb.sb_lvbptr);
1105	kmem_cache_free(cachep, gl);
1106	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
1107		wake_up(&sdp->sd_glock_wait);
1108
1109out:
1110	return ret;
1111}
1112
1113/**
1114 * gfs2_holder_init - initialize a struct gfs2_holder in the default way
1115 * @gl: the glock
1116 * @state: the state we're requesting
1117 * @flags: the modifier flags
1118 * @gh: the holder structure
1119 *
1120 */
1121
1122void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1123		      struct gfs2_holder *gh)
1124{
1125	INIT_LIST_HEAD(&gh->gh_list);
1126	gh->gh_gl = gl;
1127	gh->gh_ip = _RET_IP_;
1128	gh->gh_owner_pid = get_pid(task_pid(current));
1129	gh->gh_state = state;
1130	gh->gh_flags = flags;
1131	gh->gh_error = 0;
1132	gh->gh_iflags = 0;
1133	gfs2_glock_hold(gl);
1134}
1135
1136/**
1137 * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
1138 * @state: the state we're requesting
1139 * @flags: the modifier flags
1140 * @gh: the holder structure
1141 *
1142 * Don't mess with the glock.
1143 *
1144 */
1145
1146void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1147{
1148	gh->gh_state = state;
1149	gh->gh_flags = flags;
1150	gh->gh_iflags = 0;
1151	gh->gh_ip = _RET_IP_;
1152	put_pid(gh->gh_owner_pid);
1153	gh->gh_owner_pid = get_pid(task_pid(current));
1154}
1155
1156/**
1157 * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
1158 * @gh: the holder structure
1159 *
1160 */
1161
1162void gfs2_holder_uninit(struct gfs2_holder *gh)
1163{
1164	put_pid(gh->gh_owner_pid);
1165	gfs2_glock_put(gh->gh_gl);
1166	gfs2_holder_mark_uninitialized(gh);
1167	gh->gh_ip = 0;
1168}
1169
1170static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1171					unsigned long start_time)
1172{
1173	/* Have we waited longer that a second? */
1174	if (time_after(jiffies, start_time + HZ)) {
1175		/* Lengthen the minimum hold time. */
1176		gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1177				       GL_GLOCK_MAX_HOLD);
1178	}
1179}
1180
1181/**
1182 * gfs2_glock_wait - wait on a glock acquisition
1183 * @gh: the glock holder
1184 *
1185 * Returns: 0 on success
1186 */
1187
1188int gfs2_glock_wait(struct gfs2_holder *gh)
1189{
1190	unsigned long start_time = jiffies;
1191
1192	might_sleep();
1193	wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1194	gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1195	return gh->gh_error;
1196}
1197
1198static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1199{
1200	int i;
1201
1202	for (i = 0; i < num_gh; i++)
1203		if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1204			return 1;
1205	return 0;
1206}
1207
1208/**
1209 * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
1210 * @num_gh: the number of holders in the array
1211 * @ghs: the glock holder array
1212 *
1213 * Returns: 0 on success, meaning all glocks have been granted and are held.
1214 *          -ESTALE if the request timed out, meaning all glocks were released,
1215 *          and the caller should retry the operation.
1216 */
1217
1218int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1219{
1220	struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1221	int i, ret = 0, timeout = 0;
1222	unsigned long start_time = jiffies;
1223	bool keep_waiting;
1224
1225	might_sleep();
1226	/*
1227	 * Total up the (minimum hold time * 2) of all glocks and use that to
1228	 * determine the max amount of time we should wait.
1229	 */
1230	for (i = 0; i < num_gh; i++)
1231		timeout += ghs[i].gh_gl->gl_hold_time << 1;
1232
1233wait_for_dlm:
1234	if (!wait_event_timeout(sdp->sd_async_glock_wait,
1235				!glocks_pending(num_gh, ghs), timeout))
1236		ret = -ESTALE; /* request timed out. */
1237
1238	/*
1239	 * If dlm granted all our requests, we need to adjust the glock
1240	 * minimum hold time values according to how long we waited.
1241	 *
1242	 * If our request timed out, we need to repeatedly release any held
1243	 * glocks we acquired thus far to allow dlm to acquire the remaining
1244	 * glocks without deadlocking.  We cannot currently cancel outstanding
1245	 * glock acquisitions.
1246	 *
1247	 * The HIF_WAIT bit tells us which requests still need a response from
1248	 * dlm.
1249	 *
1250	 * If dlm sent us any errors, we return the first error we find.
1251	 */
1252	keep_waiting = false;
1253	for (i = 0; i < num_gh; i++) {
1254		/* Skip holders we have already dequeued below. */
1255		if (!gfs2_holder_queued(&ghs[i]))
1256			continue;
1257		/* Skip holders with a pending DLM response. */
1258		if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) {
1259			keep_waiting = true;
1260			continue;
1261		}
1262
1263		if (test_bit(HIF_HOLDER, &ghs[i].gh_iflags)) {
1264			if (ret == -ESTALE)
1265				gfs2_glock_dq(&ghs[i]);
1266			else
1267				gfs2_glock_update_hold_time(ghs[i].gh_gl,
1268							    start_time);
1269		}
1270		if (!ret)
1271			ret = ghs[i].gh_error;
1272	}
1273
1274	if (keep_waiting)
1275		goto wait_for_dlm;
1276
1277	/*
1278	 * At this point, we've either acquired all locks or released them all.
1279	 */
1280	return ret;
1281}
1282
1283/**
1284 * handle_callback - process a demote request
1285 * @gl: the glock
1286 * @state: the state the caller wants us to change to
1287 * @delay: zero to demote immediately; otherwise pending demote
1288 * @remote: true if this came from a different cluster node
1289 *
1290 * There are only two requests that we are going to see in actual
1291 * practise: LM_ST_SHARED and LM_ST_UNLOCKED
1292 */
1293
1294static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1295			    unsigned long delay, bool remote)
1296{
1297	if (delay)
1298		set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1299	else
1300		gfs2_set_demote(gl);
1301	if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1302		gl->gl_demote_state = state;
1303		gl->gl_demote_time = jiffies;
1304	} else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1305			gl->gl_demote_state != state) {
1306		gl->gl_demote_state = LM_ST_UNLOCKED;
1307	}
1308	if (gl->gl_ops->go_callback)
1309		gl->gl_ops->go_callback(gl, remote);
1310	trace_gfs2_demote_rq(gl, remote);
1311}
1312
1313void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1314{
1315	struct va_format vaf;
1316	va_list args;
1317
1318	va_start(args, fmt);
1319
1320	if (seq) {
1321		seq_vprintf(seq, fmt, args);
1322	} else {
1323		vaf.fmt = fmt;
1324		vaf.va = &args;
1325
1326		pr_err("%pV", &vaf);
1327	}
1328
1329	va_end(args);
1330}
1331
1332/**
1333 * add_to_queue - Add a holder to the wait queue (but look for recursion)
1334 * @gh: the holder structure to add
1335 *
1336 * Eventually we should move the recursive locking trap to a
1337 * debugging option or something like that. This is the fast
1338 * path and needs to have the minimum number of distractions.
1339 *
1340 */
1341
1342static inline void add_to_queue(struct gfs2_holder *gh)
1343__releases(&gl->gl_lockref.lock)
1344__acquires(&gl->gl_lockref.lock)
1345{
1346	struct gfs2_glock *gl = gh->gh_gl;
1347	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1348	struct list_head *insert_pt = NULL;
1349	struct gfs2_holder *gh2;
1350	int try_futile = 0;
1351
1352	GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1353	if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1354		GLOCK_BUG_ON(gl, true);
1355
1356	if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1357		if (test_bit(GLF_LOCK, &gl->gl_flags))
1358			try_futile = !may_grant(gl, gh);
1359		if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1360			goto fail;
1361	}
1362
1363	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1364		if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1365		    (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))
1366			goto trap_recursive;
1367		if (try_futile &&
1368		    !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1369fail:
1370			gh->gh_error = GLR_TRYFAILED;
1371			gfs2_holder_wake(gh);
1372			return;
1373		}
1374		if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1375			continue;
1376		if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1377			insert_pt = &gh2->gh_list;
1378	}
1379	trace_gfs2_glock_queue(gh, 1);
1380	gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1381	gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1382	if (likely(insert_pt == NULL)) {
1383		list_add_tail(&gh->gh_list, &gl->gl_holders);
1384		if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1385			goto do_cancel;
1386		return;
1387	}
1388	list_add_tail(&gh->gh_list, insert_pt);
1389do_cancel:
1390	gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
1391	if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1392		spin_unlock(&gl->gl_lockref.lock);
1393		if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1394			sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1395		spin_lock(&gl->gl_lockref.lock);
1396	}
1397	return;
1398
1399trap_recursive:
1400	fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1401	fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1402	fs_err(sdp, "lock type: %d req lock state : %d\n",
1403	       gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1404	fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1405	fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1406	fs_err(sdp, "lock type: %d req lock state : %d\n",
1407	       gh->gh_gl->gl_name.ln_type, gh->gh_state);
1408	gfs2_dump_glock(NULL, gl, true);
1409	BUG();
1410}
1411
1412/**
1413 * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1414 * @gh: the holder structure
1415 *
1416 * if (gh->gh_flags & GL_ASYNC), this never returns an error
1417 *
1418 * Returns: 0, GLR_TRYFAILED, or errno on failure
1419 */
1420
1421int gfs2_glock_nq(struct gfs2_holder *gh)
1422{
1423	struct gfs2_glock *gl = gh->gh_gl;
1424	int error = 0;
1425
1426	if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1427		return -EIO;
1428
1429	if (test_bit(GLF_LRU, &gl->gl_flags))
1430		gfs2_glock_remove_from_lru(gl);
1431
1432	spin_lock(&gl->gl_lockref.lock);
1433	add_to_queue(gh);
1434	if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1435		     test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1436		set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1437		gl->gl_lockref.count++;
1438		__gfs2_glock_queue_work(gl, 0);
1439	}
1440	run_queue(gl, 1);
1441	spin_unlock(&gl->gl_lockref.lock);
1442
1443	if (!(gh->gh_flags & GL_ASYNC))
1444		error = gfs2_glock_wait(gh);
1445
1446	return error;
1447}
1448
1449/**
1450 * gfs2_glock_poll - poll to see if an async request has been completed
1451 * @gh: the holder
1452 *
1453 * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1454 */
1455
1456int gfs2_glock_poll(struct gfs2_holder *gh)
1457{
1458	return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1459}
1460
1461/**
1462 * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1463 * @gh: the glock holder
1464 *
1465 */
1466
1467void gfs2_glock_dq(struct gfs2_holder *gh)
1468{
1469	struct gfs2_glock *gl = gh->gh_gl;
1470	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1471	unsigned delay = 0;
1472	int fast_path = 0;
1473
1474	spin_lock(&gl->gl_lockref.lock);
1475	/*
1476	 * If we're in the process of file system withdraw, we cannot just
1477	 * dequeue any glocks until our journal is recovered, lest we
1478	 * introduce file system corruption. We need two exceptions to this
1479	 * rule: We need to allow unlocking of nondisk glocks and the glock
1480	 * for our own journal that needs recovery.
1481	 */
1482	if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1483	    glock_blocked_by_withdraw(gl) &&
1484	    gh->gh_gl != sdp->sd_jinode_gl) {
1485		sdp->sd_glock_dqs_held++;
1486		spin_unlock(&gl->gl_lockref.lock);
1487		might_sleep();
1488		wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
1489			    TASK_UNINTERRUPTIBLE);
1490		spin_lock(&gl->gl_lockref.lock);
1491	}
1492	if (gh->gh_flags & GL_NOCACHE)
1493		handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1494
1495	list_del_init(&gh->gh_list);
1496	clear_bit(HIF_HOLDER, &gh->gh_iflags);
1497	if (list_empty(&gl->gl_holders) &&
1498	    !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1499	    !test_bit(GLF_DEMOTE, &gl->gl_flags))
1500		fast_path = 1;
1501
1502	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1503		gfs2_glock_add_to_lru(gl);
1504
1505	trace_gfs2_glock_queue(gh, 0);
1506	if (unlikely(!fast_path)) {
1507		gl->gl_lockref.count++;
1508		if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1509		    !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1510		    gl->gl_name.ln_type == LM_TYPE_INODE)
1511			delay = gl->gl_hold_time;
1512		__gfs2_glock_queue_work(gl, delay);
1513	}
1514	spin_unlock(&gl->gl_lockref.lock);
1515}
1516
1517void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1518{
1519	struct gfs2_glock *gl = gh->gh_gl;
1520	gfs2_glock_dq(gh);
1521	might_sleep();
1522	wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1523}
1524
1525/**
1526 * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1527 * @gh: the holder structure
1528 *
1529 */
1530
1531void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1532{
1533	gfs2_glock_dq(gh);
1534	gfs2_holder_uninit(gh);
1535}
1536
1537/**
1538 * gfs2_glock_nq_num - acquire a glock based on lock number
1539 * @sdp: the filesystem
1540 * @number: the lock number
1541 * @glops: the glock operations for the type of glock
1542 * @state: the state to acquire the glock in
1543 * @flags: modifier flags for the acquisition
1544 * @gh: the struct gfs2_holder
1545 *
1546 * Returns: errno
1547 */
1548
1549int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1550		      const struct gfs2_glock_operations *glops,
1551		      unsigned int state, u16 flags, struct gfs2_holder *gh)
1552{
1553	struct gfs2_glock *gl;
1554	int error;
1555
1556	error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1557	if (!error) {
1558		error = gfs2_glock_nq_init(gl, state, flags, gh);
1559		gfs2_glock_put(gl);
1560	}
1561
1562	return error;
1563}
1564
1565/**
1566 * glock_compare - Compare two struct gfs2_glock structures for sorting
1567 * @arg_a: the first structure
1568 * @arg_b: the second structure
1569 *
1570 */
1571
1572static int glock_compare(const void *arg_a, const void *arg_b)
1573{
1574	const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1575	const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1576	const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1577	const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1578
1579	if (a->ln_number > b->ln_number)
1580		return 1;
1581	if (a->ln_number < b->ln_number)
1582		return -1;
1583	BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1584	return 0;
1585}
1586
1587/**
1588 * nq_m_sync - synchonously acquire more than one glock in deadlock free order
1589 * @num_gh: the number of structures
1590 * @ghs: an array of struct gfs2_holder structures
1591 * @p: placeholder for the holder structure to pass back
1592 *
1593 * Returns: 0 on success (all glocks acquired),
1594 *          errno on failure (no glocks acquired)
1595 */
1596
1597static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1598		     struct gfs2_holder **p)
1599{
1600	unsigned int x;
1601	int error = 0;
1602
1603	for (x = 0; x < num_gh; x++)
1604		p[x] = &ghs[x];
1605
1606	sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1607
1608	for (x = 0; x < num_gh; x++) {
1609		p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1610
1611		error = gfs2_glock_nq(p[x]);
1612		if (error) {
1613			while (x--)
1614				gfs2_glock_dq(p[x]);
1615			break;
1616		}
1617	}
1618
1619	return error;
1620}
1621
1622/**
1623 * gfs2_glock_nq_m - acquire multiple glocks
1624 * @num_gh: the number of structures
1625 * @ghs: an array of struct gfs2_holder structures
1626 *
1627 *
1628 * Returns: 0 on success (all glocks acquired),
1629 *          errno on failure (no glocks acquired)
1630 */
1631
1632int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1633{
1634	struct gfs2_holder *tmp[4];
1635	struct gfs2_holder **pph = tmp;
1636	int error = 0;
1637
1638	switch(num_gh) {
1639	case 0:
1640		return 0;
1641	case 1:
1642		ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1643		return gfs2_glock_nq(ghs);
1644	default:
1645		if (num_gh <= 4)
1646			break;
1647		pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1648				    GFP_NOFS);
1649		if (!pph)
1650			return -ENOMEM;
1651	}
1652
1653	error = nq_m_sync(num_gh, ghs, pph);
1654
1655	if (pph != tmp)
1656		kfree(pph);
1657
1658	return error;
1659}
1660
1661/**
1662 * gfs2_glock_dq_m - release multiple glocks
1663 * @num_gh: the number of structures
1664 * @ghs: an array of struct gfs2_holder structures
1665 *
1666 */
1667
1668void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1669{
1670	while (num_gh--)
1671		gfs2_glock_dq(&ghs[num_gh]);
1672}
1673
1674void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1675{
1676	unsigned long delay = 0;
1677	unsigned long holdtime;
1678	unsigned long now = jiffies;
1679
1680	gfs2_glock_hold(gl);
1681	spin_lock(&gl->gl_lockref.lock);
1682	holdtime = gl->gl_tchange + gl->gl_hold_time;
1683	if (!list_empty(&gl->gl_holders) &&
1684	    gl->gl_name.ln_type == LM_TYPE_INODE) {
1685		if (time_before(now, holdtime))
1686			delay = holdtime - now;
1687		if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1688			delay = gl->gl_hold_time;
1689	}
1690	handle_callback(gl, state, delay, true);
1691	__gfs2_glock_queue_work(gl, delay);
1692	spin_unlock(&gl->gl_lockref.lock);
1693}
1694
1695/**
1696 * gfs2_should_freeze - Figure out if glock should be frozen
1697 * @gl: The glock in question
1698 *
1699 * Glocks are not frozen if (a) the result of the dlm operation is
1700 * an error, (b) the locking operation was an unlock operation or
1701 * (c) if there is a "noexp" flagged request anywhere in the queue
1702 *
1703 * Returns: 1 if freezing should occur, 0 otherwise
1704 */
1705
1706static int gfs2_should_freeze(const struct gfs2_glock *gl)
1707{
1708	const struct gfs2_holder *gh;
1709
1710	if (gl->gl_reply & ~LM_OUT_ST_MASK)
1711		return 0;
1712	if (gl->gl_target == LM_ST_UNLOCKED)
1713		return 0;
1714
1715	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1716		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1717			continue;
1718		if (LM_FLAG_NOEXP & gh->gh_flags)
1719			return 0;
1720	}
1721
1722	return 1;
1723}
1724
1725/**
1726 * gfs2_glock_complete - Callback used by locking
1727 * @gl: Pointer to the glock
1728 * @ret: The return value from the dlm
1729 *
1730 * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1731 * to use a bitfield shared with other glock state fields.
1732 */
1733
1734void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1735{
1736	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1737
1738	spin_lock(&gl->gl_lockref.lock);
1739	gl->gl_reply = ret;
1740
1741	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1742		if (gfs2_should_freeze(gl)) {
1743			set_bit(GLF_FROZEN, &gl->gl_flags);
1744			spin_unlock(&gl->gl_lockref.lock);
1745			return;
1746		}
1747	}
1748
1749	gl->gl_lockref.count++;
1750	set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1751	__gfs2_glock_queue_work(gl, 0);
1752	spin_unlock(&gl->gl_lockref.lock);
1753}
1754
1755static int glock_cmp(void *priv, const struct list_head *a,
1756		     const struct list_head *b)
1757{
1758	struct gfs2_glock *gla, *glb;
1759
1760	gla = list_entry(a, struct gfs2_glock, gl_lru);
1761	glb = list_entry(b, struct gfs2_glock, gl_lru);
1762
1763	if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1764		return 1;
1765	if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1766		return -1;
1767
1768	return 0;
1769}
1770
1771/**
1772 * gfs2_dispose_glock_lru - Demote a list of glocks
1773 * @list: The list to dispose of
1774 *
1775 * Disposing of glocks may involve disk accesses, so that here we sort
1776 * the glocks by number (i.e. disk location of the inodes) so that if
1777 * there are any such accesses, they'll be sent in order (mostly).
1778 *
1779 * Must be called under the lru_lock, but may drop and retake this
1780 * lock. While the lru_lock is dropped, entries may vanish from the
1781 * list, but no new entries will appear on the list (since it is
1782 * private)
1783 */
1784
1785static void gfs2_dispose_glock_lru(struct list_head *list)
1786__releases(&lru_lock)
1787__acquires(&lru_lock)
1788{
1789	struct gfs2_glock *gl;
1790
1791	list_sort(NULL, list, glock_cmp);
1792
1793	while(!list_empty(list)) {
1794		gl = list_first_entry(list, struct gfs2_glock, gl_lru);
1795		list_del_init(&gl->gl_lru);
1796		clear_bit(GLF_LRU, &gl->gl_flags);
1797		if (!spin_trylock(&gl->gl_lockref.lock)) {
1798add_back_to_lru:
1799			list_add(&gl->gl_lru, &lru_list);
1800			set_bit(GLF_LRU, &gl->gl_flags);
1801			atomic_inc(&lru_count);
1802			continue;
1803		}
1804		if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
1805			spin_unlock(&gl->gl_lockref.lock);
1806			goto add_back_to_lru;
1807		}
1808		gl->gl_lockref.count++;
1809		if (demote_ok(gl))
1810			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1811		WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
1812		__gfs2_glock_queue_work(gl, 0);
1813		spin_unlock(&gl->gl_lockref.lock);
1814		cond_resched_lock(&lru_lock);
1815	}
1816}
1817
1818/**
1819 * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
1820 * @nr: The number of entries to scan
1821 *
1822 * This function selects the entries on the LRU which are able to
1823 * be demoted, and then kicks off the process by calling
1824 * gfs2_dispose_glock_lru() above.
1825 */
1826
1827static long gfs2_scan_glock_lru(int nr)
1828{
1829	struct gfs2_glock *gl;
1830	LIST_HEAD(skipped);
1831	LIST_HEAD(dispose);
1832	long freed = 0;
1833
1834	spin_lock(&lru_lock);
1835	while ((nr-- >= 0) && !list_empty(&lru_list)) {
1836		gl = list_first_entry(&lru_list, struct gfs2_glock, gl_lru);
1837
1838		/* Test for being demotable */
1839		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
1840			list_move(&gl->gl_lru, &dispose);
1841			atomic_dec(&lru_count);
1842			freed++;
1843			continue;
1844		}
1845
1846		list_move(&gl->gl_lru, &skipped);
1847	}
1848	list_splice(&skipped, &lru_list);
1849	if (!list_empty(&dispose))
1850		gfs2_dispose_glock_lru(&dispose);
1851	spin_unlock(&lru_lock);
1852
1853	return freed;
1854}
1855
1856static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
1857					    struct shrink_control *sc)
1858{
1859	if (!(sc->gfp_mask & __GFP_FS))
1860		return SHRINK_STOP;
1861	return gfs2_scan_glock_lru(sc->nr_to_scan);
1862}
1863
1864static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
1865					     struct shrink_control *sc)
1866{
1867	return vfs_pressure_ratio(atomic_read(&lru_count));
1868}
1869
1870static struct shrinker glock_shrinker = {
1871	.seeks = DEFAULT_SEEKS,
1872	.count_objects = gfs2_glock_shrink_count,
1873	.scan_objects = gfs2_glock_shrink_scan,
1874};
1875
1876/**
1877 * glock_hash_walk - Call a function for glock in a hash bucket
1878 * @examiner: the function
1879 * @sdp: the filesystem
1880 *
1881 * Note that the function can be called multiple times on the same
1882 * object.  So the user must ensure that the function can cope with
1883 * that.
1884 */
1885
1886static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1887{
1888	struct gfs2_glock *gl;
1889	struct rhashtable_iter iter;
1890
1891	rhashtable_walk_enter(&gl_hash_table, &iter);
1892
1893	do {
1894		rhashtable_walk_start(&iter);
1895
1896		while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl))
1897			if (gl->gl_name.ln_sbd == sdp &&
1898			    lockref_get_not_dead(&gl->gl_lockref))
1899				examiner(gl);
1900
1901		rhashtable_walk_stop(&iter);
1902	} while (cond_resched(), gl == ERR_PTR(-EAGAIN));
1903
1904	rhashtable_walk_exit(&iter);
1905}
1906
1907bool gfs2_queue_delete_work(struct gfs2_glock *gl, unsigned long delay)
1908{
1909	bool queued;
1910
1911	spin_lock(&gl->gl_lockref.lock);
1912	queued = queue_delayed_work(gfs2_delete_workqueue,
1913				    &gl->gl_delete, delay);
1914	if (queued)
1915		set_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1916	spin_unlock(&gl->gl_lockref.lock);
1917	return queued;
1918}
1919
1920void gfs2_cancel_delete_work(struct gfs2_glock *gl)
1921{
1922	if (cancel_delayed_work_sync(&gl->gl_delete)) {
1923		clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1924		gfs2_glock_put(gl);
1925	}
1926}
1927
1928bool gfs2_delete_work_queued(const struct gfs2_glock *gl)
1929{
1930	return test_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1931}
1932
1933static void flush_delete_work(struct gfs2_glock *gl)
1934{
1935	if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
1936		if (cancel_delayed_work(&gl->gl_delete)) {
1937			queue_delayed_work(gfs2_delete_workqueue,
1938					   &gl->gl_delete, 0);
1939		}
1940	}
1941	gfs2_glock_queue_work(gl, 0);
1942}
1943
1944void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
1945{
1946	glock_hash_walk(flush_delete_work, sdp);
1947	flush_workqueue(gfs2_delete_workqueue);
1948}
1949
1950/**
1951 * thaw_glock - thaw out a glock which has an unprocessed reply waiting
1952 * @gl: The glock to thaw
1953 *
1954 */
1955
1956static void thaw_glock(struct gfs2_glock *gl)
1957{
1958	if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) {
1959		gfs2_glock_put(gl);
1960		return;
1961	}
1962	set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1963	gfs2_glock_queue_work(gl, 0);
1964}
1965
1966/**
1967 * clear_glock - look at a glock and see if we can free it from glock cache
1968 * @gl: the glock to look at
1969 *
1970 */
1971
1972static void clear_glock(struct gfs2_glock *gl)
1973{
1974	gfs2_glock_remove_from_lru(gl);
1975
1976	spin_lock(&gl->gl_lockref.lock);
1977	if (gl->gl_state != LM_ST_UNLOCKED)
1978		handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1979	__gfs2_glock_queue_work(gl, 0);
1980	spin_unlock(&gl->gl_lockref.lock);
1981}
1982
1983/**
1984 * gfs2_glock_thaw - Thaw any frozen glocks
1985 * @sdp: The super block
1986 *
1987 */
1988
1989void gfs2_glock_thaw(struct gfs2_sbd *sdp)
1990{
1991	glock_hash_walk(thaw_glock, sdp);
1992}
1993
1994static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1995{
1996	spin_lock(&gl->gl_lockref.lock);
1997	gfs2_dump_glock(seq, gl, fsid);
1998	spin_unlock(&gl->gl_lockref.lock);
1999}
2000
2001static void dump_glock_func(struct gfs2_glock *gl)
2002{
2003	dump_glock(NULL, gl, true);
2004}
2005
2006/**
2007 * gfs2_gl_hash_clear - Empty out the glock hash table
2008 * @sdp: the filesystem
2009 *
2010 * Called when unmounting the filesystem.
2011 */
2012
2013void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2014{
2015	set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
2016	flush_workqueue(glock_workqueue);
2017	glock_hash_walk(clear_glock, sdp);
2018	flush_workqueue(glock_workqueue);
2019	wait_event_timeout(sdp->sd_glock_wait,
2020			   atomic_read(&sdp->sd_glock_disposal) == 0,
2021			   HZ * 600);
2022	glock_hash_walk(dump_glock_func, sdp);
2023}
2024
2025void gfs2_glock_finish_truncate(struct gfs2_inode *ip)
2026{
2027	struct gfs2_glock *gl = ip->i_gl;
2028	int ret;
2029
2030	ret = gfs2_truncatei_resume(ip);
2031	gfs2_glock_assert_withdraw(gl, ret == 0);
2032
2033	spin_lock(&gl->gl_lockref.lock);
2034	clear_bit(GLF_LOCK, &gl->gl_flags);
2035	run_queue(gl, 1);
2036	spin_unlock(&gl->gl_lockref.lock);
2037}
2038
2039static const char *state2str(unsigned state)
2040{
2041	switch(state) {
2042	case LM_ST_UNLOCKED:
2043		return "UN";
2044	case LM_ST_SHARED:
2045		return "SH";
2046	case LM_ST_DEFERRED:
2047		return "DF";
2048	case LM_ST_EXCLUSIVE:
2049		return "EX";
2050	}
2051	return "??";
2052}
2053
2054static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2055{
2056	char *p = buf;
2057	if (flags & LM_FLAG_TRY)
2058		*p++ = 't';
2059	if (flags & LM_FLAG_TRY_1CB)
2060		*p++ = 'T';
2061	if (flags & LM_FLAG_NOEXP)
2062		*p++ = 'e';
2063	if (flags & LM_FLAG_ANY)
2064		*p++ = 'A';
2065	if (flags & LM_FLAG_PRIORITY)
2066		*p++ = 'p';
2067	if (flags & LM_FLAG_NODE_SCOPE)
2068		*p++ = 'n';
2069	if (flags & GL_ASYNC)
2070		*p++ = 'a';
2071	if (flags & GL_EXACT)
2072		*p++ = 'E';
2073	if (flags & GL_NOCACHE)
2074		*p++ = 'c';
2075	if (test_bit(HIF_HOLDER, &iflags))
2076		*p++ = 'H';
2077	if (test_bit(HIF_WAIT, &iflags))
2078		*p++ = 'W';
2079	*p = 0;
2080	return buf;
2081}
2082
2083/**
2084 * dump_holder - print information about a glock holder
2085 * @seq: the seq_file struct
2086 * @gh: the glock holder
2087 * @fs_id_buf: pointer to file system id (if requested)
2088 *
2089 */
2090
2091static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2092			const char *fs_id_buf)
2093{
2094	struct task_struct *gh_owner = NULL;
2095	char flags_buf[32];
2096
2097	rcu_read_lock();
2098	if (gh->gh_owner_pid)
2099		gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2100	gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2101		       fs_id_buf, state2str(gh->gh_state),
2102		       hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2103		       gh->gh_error,
2104		       gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
2105		       gh_owner ? gh_owner->comm : "(ended)",
2106		       (void *)gh->gh_ip);
2107	rcu_read_unlock();
2108}
2109
2110static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2111{
2112	const unsigned long *gflags = &gl->gl_flags;
2113	char *p = buf;
2114
2115	if (test_bit(GLF_LOCK, gflags))
2116		*p++ = 'l';
2117	if (test_bit(GLF_DEMOTE, gflags))
2118		*p++ = 'D';
2119	if (test_bit(GLF_PENDING_DEMOTE, gflags))
2120		*p++ = 'd';
2121	if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2122		*p++ = 'p';
2123	if (test_bit(GLF_DIRTY, gflags))
2124		*p++ = 'y';
2125	if (test_bit(GLF_LFLUSH, gflags))
2126		*p++ = 'f';
2127	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2128		*p++ = 'i';
2129	if (test_bit(GLF_REPLY_PENDING, gflags))
2130		*p++ = 'r';
2131	if (test_bit(GLF_INITIAL, gflags))
2132		*p++ = 'I';
2133	if (test_bit(GLF_FROZEN, gflags))
2134		*p++ = 'F';
2135	if (!list_empty(&gl->gl_holders))
2136		*p++ = 'q';
2137	if (test_bit(GLF_LRU, gflags))
2138		*p++ = 'L';
2139	if (gl->gl_object)
2140		*p++ = 'o';
2141	if (test_bit(GLF_BLOCKING, gflags))
2142		*p++ = 'b';
2143	if (test_bit(GLF_PENDING_DELETE, gflags))
2144		*p++ = 'P';
2145	if (test_bit(GLF_FREEING, gflags))
2146		*p++ = 'x';
2147	*p = 0;
2148	return buf;
2149}
2150
2151/**
2152 * gfs2_dump_glock - print information about a glock
2153 * @seq: The seq_file struct
2154 * @gl: the glock
2155 * @fsid: If true, also dump the file system id
2156 *
2157 * The file format is as follows:
2158 * One line per object, capital letters are used to indicate objects
2159 * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
2160 * other objects are indented by a single space and follow the glock to
2161 * which they are related. Fields are indicated by lower case letters
2162 * followed by a colon and the field value, except for strings which are in
2163 * [] so that its possible to see if they are composed of spaces for
2164 * example. The field's are n = number (id of the object), f = flags,
2165 * t = type, s = state, r = refcount, e = error, p = pid.
2166 *
2167 */
2168
2169void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2170{
2171	const struct gfs2_glock_operations *glops = gl->gl_ops;
2172	unsigned long long dtime;
2173	const struct gfs2_holder *gh;
2174	char gflags_buf[32];
2175	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2176	char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2177	unsigned long nrpages = 0;
2178
2179	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2180		struct address_space *mapping = gfs2_glock2aspace(gl);
2181
2182		nrpages = mapping->nrpages;
2183	}
2184	memset(fs_id_buf, 0, sizeof(fs_id_buf));
2185	if (fsid && sdp) /* safety precaution */
2186		sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2187	dtime = jiffies - gl->gl_demote_time;
2188	dtime *= 1000000/HZ; /* demote time in uSec */
2189	if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2190		dtime = 0;
2191	gfs2_print_dbg(seq, "%sG:  s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2192		       "v:%d r:%d m:%ld p:%lu\n",
2193		       fs_id_buf, state2str(gl->gl_state),
2194		       gl->gl_name.ln_type,
2195		       (unsigned long long)gl->gl_name.ln_number,
2196		       gflags2str(gflags_buf, gl),
2197		       state2str(gl->gl_target),
2198		       state2str(gl->gl_demote_state), dtime,
2199		       atomic_read(&gl->gl_ail_count),
2200		       atomic_read(&gl->gl_revokes),
2201		       (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2202
2203	list_for_each_entry(gh, &gl->gl_holders, gh_list)
2204		dump_holder(seq, gh, fs_id_buf);
2205
2206	if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2207		glops->go_dump(seq, gl, fs_id_buf);
2208}
2209
2210static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2211{
2212	struct gfs2_glock *gl = iter_ptr;
2213
2214	seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2215		   gl->gl_name.ln_type,
2216		   (unsigned long long)gl->gl_name.ln_number,
2217		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2218		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2219		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2220		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2221		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2222		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2223		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2224		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2225	return 0;
2226}
2227
2228static const char *gfs2_gltype[] = {
2229	"type",
2230	"reserved",
2231	"nondisk",
2232	"inode",
2233	"rgrp",
2234	"meta",
2235	"iopen",
2236	"flock",
2237	"plock",
2238	"quota",
2239	"journal",
2240};
2241
2242static const char *gfs2_stype[] = {
2243	[GFS2_LKS_SRTT]		= "srtt",
2244	[GFS2_LKS_SRTTVAR]	= "srttvar",
2245	[GFS2_LKS_SRTTB]	= "srttb",
2246	[GFS2_LKS_SRTTVARB]	= "srttvarb",
2247	[GFS2_LKS_SIRT]		= "sirt",
2248	[GFS2_LKS_SIRTVAR]	= "sirtvar",
2249	[GFS2_LKS_DCOUNT]	= "dlm",
2250	[GFS2_LKS_QCOUNT]	= "queue",
2251};
2252
2253#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2254
2255static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2256{
2257	struct gfs2_sbd *sdp = seq->private;
2258	loff_t pos = *(loff_t *)iter_ptr;
2259	unsigned index = pos >> 3;
2260	unsigned subindex = pos & 0x07;
2261	int i;
2262
2263	if (index == 0 && subindex != 0)
2264		return 0;
2265
2266	seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2267		   (index == 0) ? "cpu": gfs2_stype[subindex]);
2268
2269	for_each_possible_cpu(i) {
2270                const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2271
2272		if (index == 0)
2273			seq_printf(seq, " %15u", i);
2274		else
2275			seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2276				   lkstats[index - 1].stats[subindex]);
2277	}
2278	seq_putc(seq, '\n');
2279	return 0;
2280}
2281
2282int __init gfs2_glock_init(void)
2283{
2284	int i, ret;
2285
2286	ret = rhashtable_init(&gl_hash_table, &ht_parms);
2287	if (ret < 0)
2288		return ret;
2289
2290	glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
2291					  WQ_HIGHPRI | WQ_FREEZABLE, 0);
2292	if (!glock_workqueue) {
2293		rhashtable_destroy(&gl_hash_table);
2294		return -ENOMEM;
2295	}
2296	gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
2297						WQ_MEM_RECLAIM | WQ_FREEZABLE,
2298						0);
2299	if (!gfs2_delete_workqueue) {
2300		destroy_workqueue(glock_workqueue);
2301		rhashtable_destroy(&gl_hash_table);
2302		return -ENOMEM;
2303	}
2304
2305	ret = register_shrinker(&glock_shrinker);
2306	if (ret) {
2307		destroy_workqueue(gfs2_delete_workqueue);
2308		destroy_workqueue(glock_workqueue);
2309		rhashtable_destroy(&gl_hash_table);
2310		return ret;
2311	}
2312
2313	for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2314		init_waitqueue_head(glock_wait_table + i);
2315
2316	return 0;
2317}
2318
2319void gfs2_glock_exit(void)
2320{
2321	unregister_shrinker(&glock_shrinker);
2322	rhashtable_destroy(&gl_hash_table);
2323	destroy_workqueue(glock_workqueue);
2324	destroy_workqueue(gfs2_delete_workqueue);
2325}
2326
2327static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2328{
2329	struct gfs2_glock *gl = gi->gl;
2330
2331	if (gl) {
2332		if (n == 0)
2333			return;
2334		if (!lockref_put_not_zero(&gl->gl_lockref))
2335			gfs2_glock_queue_put(gl);
2336	}
2337	for (;;) {
2338		gl = rhashtable_walk_next(&gi->hti);
2339		if (IS_ERR_OR_NULL(gl)) {
2340			if (gl == ERR_PTR(-EAGAIN)) {
2341				n = 1;
2342				continue;
2343			}
2344			gl = NULL;
2345			break;
2346		}
2347		if (gl->gl_name.ln_sbd != gi->sdp)
2348			continue;
2349		if (n <= 1) {
2350			if (!lockref_get_not_dead(&gl->gl_lockref))
2351				continue;
2352			break;
2353		} else {
2354			if (__lockref_is_dead(&gl->gl_lockref))
2355				continue;
2356			n--;
2357		}
2358	}
2359	gi->gl = gl;
2360}
2361
2362static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2363	__acquires(RCU)
2364{
2365	struct gfs2_glock_iter *gi = seq->private;
2366	loff_t n;
2367
2368	/*
2369	 * We can either stay where we are, skip to the next hash table
2370	 * entry, or start from the beginning.
2371	 */
2372	if (*pos < gi->last_pos) {
2373		rhashtable_walk_exit(&gi->hti);
2374		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2375		n = *pos + 1;
2376	} else {
2377		n = *pos - gi->last_pos;
2378	}
2379
2380	rhashtable_walk_start(&gi->hti);
2381
2382	gfs2_glock_iter_next(gi, n);
2383	gi->last_pos = *pos;
2384	return gi->gl;
2385}
2386
2387static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2388				 loff_t *pos)
2389{
2390	struct gfs2_glock_iter *gi = seq->private;
2391
2392	(*pos)++;
2393	gi->last_pos = *pos;
2394	gfs2_glock_iter_next(gi, 1);
2395	return gi->gl;
2396}
2397
2398static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2399	__releases(RCU)
2400{
2401	struct gfs2_glock_iter *gi = seq->private;
2402
2403	rhashtable_walk_stop(&gi->hti);
2404}
2405
2406static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2407{
2408	dump_glock(seq, iter_ptr, false);
2409	return 0;
2410}
2411
2412static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2413{
2414	preempt_disable();
2415	if (*pos >= GFS2_NR_SBSTATS)
2416		return NULL;
2417	return pos;
2418}
2419
2420static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2421				   loff_t *pos)
2422{
2423	(*pos)++;
2424	if (*pos >= GFS2_NR_SBSTATS)
2425		return NULL;
2426	return pos;
2427}
2428
2429static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2430{
2431	preempt_enable();
2432}
2433
2434static const struct seq_operations gfs2_glock_seq_ops = {
2435	.start = gfs2_glock_seq_start,
2436	.next  = gfs2_glock_seq_next,
2437	.stop  = gfs2_glock_seq_stop,
2438	.show  = gfs2_glock_seq_show,
2439};
2440
2441static const struct seq_operations gfs2_glstats_seq_ops = {
2442	.start = gfs2_glock_seq_start,
2443	.next  = gfs2_glock_seq_next,
2444	.stop  = gfs2_glock_seq_stop,
2445	.show  = gfs2_glstats_seq_show,
2446};
2447
2448static const struct seq_operations gfs2_sbstats_sops = {
2449	.start = gfs2_sbstats_seq_start,
2450	.next  = gfs2_sbstats_seq_next,
2451	.stop  = gfs2_sbstats_seq_stop,
2452	.show  = gfs2_sbstats_seq_show,
2453};
2454
2455#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2456
2457static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2458			      const struct seq_operations *ops)
2459{
2460	int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2461	if (ret == 0) {
2462		struct seq_file *seq = file->private_data;
2463		struct gfs2_glock_iter *gi = seq->private;
2464
2465		gi->sdp = inode->i_private;
2466		seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2467		if (seq->buf)
2468			seq->size = GFS2_SEQ_GOODSIZE;
2469		/*
2470		 * Initially, we are "before" the first hash table entry; the
2471		 * first call to rhashtable_walk_next gets us the first entry.
2472		 */
2473		gi->last_pos = -1;
2474		gi->gl = NULL;
2475		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2476	}
2477	return ret;
2478}
2479
2480static int gfs2_glocks_open(struct inode *inode, struct file *file)
2481{
2482	return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2483}
2484
2485static int gfs2_glocks_release(struct inode *inode, struct file *file)
2486{
2487	struct seq_file *seq = file->private_data;
2488	struct gfs2_glock_iter *gi = seq->private;
2489
2490	if (gi->gl)
2491		gfs2_glock_put(gi->gl);
2492	rhashtable_walk_exit(&gi->hti);
2493	return seq_release_private(inode, file);
2494}
2495
2496static int gfs2_glstats_open(struct inode *inode, struct file *file)
2497{
2498	return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2499}
2500
2501static const struct file_operations gfs2_glocks_fops = {
2502	.owner   = THIS_MODULE,
2503	.open    = gfs2_glocks_open,
2504	.read    = seq_read,
2505	.llseek  = seq_lseek,
2506	.release = gfs2_glocks_release,
2507};
2508
2509static const struct file_operations gfs2_glstats_fops = {
2510	.owner   = THIS_MODULE,
2511	.open    = gfs2_glstats_open,
2512	.read    = seq_read,
2513	.llseek  = seq_lseek,
2514	.release = gfs2_glocks_release,
2515};
2516
2517DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2518
2519void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2520{
2521	sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2522
2523	debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2524			    &gfs2_glocks_fops);
2525
2526	debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2527			    &gfs2_glstats_fops);
2528
2529	debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2530			    &gfs2_sbstats_fops);
2531}
2532
2533void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2534{
2535	debugfs_remove_recursive(sdp->debugfs_dir);
2536	sdp->debugfs_dir = NULL;
2537}
2538
2539void gfs2_register_debugfs(void)
2540{
2541	gfs2_root = debugfs_create_dir("gfs2", NULL);
2542}
2543
2544void gfs2_unregister_debugfs(void)
2545{
2546	debugfs_remove(gfs2_root);
2547	gfs2_root = NULL;
2548}
2549