1/*
2 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
3 *  Copyright (C) 2007 The Regents of the University of California.
4 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
5 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
6 *  UCRL-CODE-235197
7 *
8 *  This file is part of the SPL, Solaris Porting Layer.
9 *
10 *  The SPL is free software; you can redistribute it and/or modify it
11 *  under the terms of the GNU General Public License as published by the
12 *  Free Software Foundation; either version 2 of the License, or (at your
13 *  option) any later version.
14 *
15 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
16 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
18 *  for more details.
19 *
20 *  You should have received a copy of the GNU General Public License along
21 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
22 */
23
24#ifndef _SPL_MUTEX_H
25#define	_SPL_MUTEX_H
26
27#include <sys/types.h>
28#include <linux/sched.h>
29#include <linux/mutex.h>
30#include <linux/lockdep.h>
31#include <linux/compiler_compat.h>
32
33typedef enum {
34	MUTEX_DEFAULT	= 0,
35	MUTEX_SPIN	= 1,
36	MUTEX_ADAPTIVE	= 2,
37	MUTEX_NOLOCKDEP	= 3
38} kmutex_type_t;
39
40typedef struct {
41	struct mutex		m_mutex;
42	spinlock_t		m_lock;	/* used for serializing mutex_exit */
43	kthread_t		*m_owner;
44#ifdef CONFIG_LOCKDEP
45	kmutex_type_t		m_type;
46#endif /* CONFIG_LOCKDEP */
47} kmutex_t;
48
49#define	MUTEX(mp)		(&((mp)->m_mutex))
50
51static inline void
52spl_mutex_set_owner(kmutex_t *mp)
53{
54	mp->m_owner = current;
55}
56
57static inline void
58spl_mutex_clear_owner(kmutex_t *mp)
59{
60	mp->m_owner = NULL;
61}
62
63#define	mutex_owner(mp)		(READ_ONCE((mp)->m_owner))
64#define	mutex_owned(mp)		(mutex_owner(mp) == current)
65#define	MUTEX_HELD(mp)		mutex_owned(mp)
66#define	MUTEX_NOT_HELD(mp)	(!MUTEX_HELD(mp))
67
68#ifdef CONFIG_LOCKDEP
69static inline void
70spl_mutex_set_type(kmutex_t *mp, kmutex_type_t type)
71{
72	mp->m_type = type;
73}
74static inline void
75spl_mutex_lockdep_off_maybe(kmutex_t *mp)			\
76{								\
77	if (mp && mp->m_type == MUTEX_NOLOCKDEP)		\
78		lockdep_off();					\
79}
80static inline void
81spl_mutex_lockdep_on_maybe(kmutex_t *mp)			\
82{								\
83	if (mp && mp->m_type == MUTEX_NOLOCKDEP)		\
84		lockdep_on();					\
85}
86#else  /* CONFIG_LOCKDEP */
87#define	spl_mutex_set_type(mp, type)
88#define	spl_mutex_lockdep_off_maybe(mp)
89#define	spl_mutex_lockdep_on_maybe(mp)
90#endif /* CONFIG_LOCKDEP */
91
92/*
93 * The following functions must be a #define	and not static inline.
94 * This ensures that the native linux mutex functions (lock/unlock)
95 * will be correctly located in the users code which is important
96 * for the built in kernel lock analysis tools
97 */
98#undef mutex_init
99#define	mutex_init(mp, name, type, ibc)				\
100{								\
101	static struct lock_class_key __key;			\
102	ASSERT(type == MUTEX_DEFAULT || type == MUTEX_NOLOCKDEP); \
103								\
104	__mutex_init(MUTEX(mp), (name) ? (#name) : (#mp), &__key); \
105	spin_lock_init(&(mp)->m_lock);				\
106	spl_mutex_clear_owner(mp);				\
107	spl_mutex_set_type(mp, type);				\
108}
109
110#undef mutex_destroy
111#define	mutex_destroy(mp)					\
112{								\
113	VERIFY3P(mutex_owner(mp), ==, NULL);			\
114}
115
116#define	mutex_tryenter(mp)					\
117/* CSTYLED */								\
118({								\
119	int _rc_;						\
120								\
121	spl_mutex_lockdep_off_maybe(mp);			\
122	if ((_rc_ = mutex_trylock(MUTEX(mp))) == 1)		\
123		spl_mutex_set_owner(mp);			\
124	spl_mutex_lockdep_on_maybe(mp);				\
125								\
126	_rc_;							\
127})
128
129#define	NESTED_SINGLE 1
130
131#define	mutex_enter_nested(mp, subclass)			\
132{								\
133	ASSERT3P(mutex_owner(mp), !=, current);			\
134	spl_mutex_lockdep_off_maybe(mp);			\
135	mutex_lock_nested(MUTEX(mp), (subclass));		\
136	spl_mutex_lockdep_on_maybe(mp);				\
137	spl_mutex_set_owner(mp);				\
138}
139
140#define	mutex_enter_interruptible(mp)				\
141/* CSTYLED */							\
142({								\
143	int _rc_;						\
144								\
145	ASSERT3P(mutex_owner(mp), !=, current);			\
146	spl_mutex_lockdep_off_maybe(mp);			\
147	_rc_ = mutex_lock_interruptible(MUTEX(mp));		\
148	spl_mutex_lockdep_on_maybe(mp);				\
149	if (!_rc_) {						\
150		spl_mutex_set_owner(mp);			\
151	}							\
152								\
153	_rc_;							\
154})
155
156#define	mutex_enter(mp) mutex_enter_nested((mp), 0)
157
158/*
159 * The reason for the spinlock:
160 *
161 * The Linux mutex is designed with a fast-path/slow-path design such that it
162 * does not guarantee serialization upon itself, allowing a race where latter
163 * acquirers finish mutex_unlock before former ones.
164 *
165 * The race renders it unsafe to be used for serializing the freeing of an
166 * object in which the mutex is embedded, where the latter acquirer could go
167 * on to free the object while the former one is still doing mutex_unlock and
168 * causing memory corruption.
169 *
170 * However, there are many places in ZFS where the mutex is used for
171 * serializing object freeing, and the code is shared among other OSes without
172 * this issue. Thus, we need the spinlock to force the serialization on
173 * mutex_exit().
174 *
175 * See http://lwn.net/Articles/575477/ for the information about the race.
176 */
177#define	mutex_exit(mp)						\
178{								\
179	ASSERT3P(mutex_owner(mp), ==, current);			\
180	spl_mutex_clear_owner(mp);				\
181	spin_lock(&(mp)->m_lock);				\
182	spl_mutex_lockdep_off_maybe(mp);			\
183	mutex_unlock(MUTEX(mp));				\
184	spl_mutex_lockdep_on_maybe(mp);				\
185	spin_unlock(&(mp)->m_lock);				\
186	/* NOTE: do not dereference mp after this point */	\
187}
188
189#endif /* _SPL_MUTEX_H */
190