kern_umtx.c revision 115310 
1/*
2 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice unmodified, this list of conditions, and the following
10 *    disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: head/sys/kern/kern_umtx.c 115310 2003-05-25 18:18:32Z jeff $
27 *
28 */
29
30#include <sys/param.h>
31#include <sys/kernel.h>
32#include <sys/lock.h>
33#include <sys/mutex.h>
34#include <sys/proc.h>
35#include <sys/signalvar.h>
36#include <sys/sysent.h>
37#include <sys/systm.h>
38#include <sys/sysproto.h>
39#include <sys/thr.h>
40#include <sys/umtx.h>
41
42#define	UMTX_LOCK()	mtx_lock(&umtx_lock);
43#define	UMTX_UNLOCK()	mtx_unlock(&umtx_lock);
44
45struct mtx umtx_lock;
46
47MTX_SYSINIT(umtx, &umtx_lock, "User-land mutex lock", MTX_DEF);
48
49int
50_umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
51    /* struct umtx *umtx */
52{
53	struct umtx *umtx;
54	struct thread *blocked;
55	intptr_t owner;
56	intptr_t old;
57	int error;
58
59	error = 0;
60
61	/*
62	 * Care must be exercised when dealing with this structure.  It
63	 * can fault on any access.
64	 */
65	umtx = uap->umtx;
66
67	UMTX_LOCK();
68
69	for (;;) {
70		/*
71		 * Try the uncontested case.  This should be done in userland.
72		 */
73		owner = casuptr((intptr_t *)&umtx->u_owner,
74		    UMTX_UNOWNED, (intptr_t)td);
75
76		/* The acquire succeeded. */
77		if (owner == UMTX_UNOWNED) {
78			error = 0;
79			goto out;
80		}
81
82		/* The address was invalid. */
83		if (owner == -1) {
84			error = EFAULT;
85			goto out;
86		}
87
88		if (owner & UMTX_CONTESTED)
89			break;
90
91		/*
92		 * Set the contested bit so that a release in user space
93		 * knows to use the system call for unlock.  If this fails
94		 * either some one else has acquired the lock or it has been
95		 * released.
96		 */
97		old = casuptr((intptr_t *)&umtx->u_owner, owner,
98		    owner | UMTX_CONTESTED);
99
100		/* We set the contested bit. */
101		if (old == owner)
102			break;
103
104		/* The address was invalid. */
105		if (old == -1) {
106			error = EFAULT;
107			goto out;
108		}
109		/* We didn't set the contested bit, try again. */
110	}
111
112	/*
113	 * We are now protected from further races via umtx_lock.
114	 * If userland messes with their mutex without using cmpset
115	 * they will deadlock themselves but they will still be
116	 * killable via signals.
117	 */
118
119	if ((owner = fuword(&umtx->u_blocked)) == -1) {
120		error = EFAULT;
121		goto out;
122	}
123
124	if (owner == UMTX_UNOWNED) {
125		if (suword(&umtx->u_blocked, (long)td) == -1) {
126			error = EFAULT;
127			goto out;
128		}
129		/*
130		 * Other blocked threads will reside here.
131		 */
132		STAILQ_INIT(&td->td_umtxq);
133	} else {
134		FOREACH_THREAD_IN_PROC(td->td_proc, blocked)
135			if (blocked == (struct thread *)(owner))
136				break;
137
138		if (blocked == NULL) {
139			error = EINVAL;
140			goto out;
141		}
142		/*
143		 * Insert us onto the end of the TAILQ.
144		 */
145		STAILQ_INSERT_TAIL(&blocked->td_umtxq, td, td_umtx);
146	}
147
148	for (;;) {
149		/*
150		 * Sleep until we can acquire the lock.  We must still deliver
151		 * signals so that they are not deferred until we acquire the
152		 * lock which may be never.  The threads actual priority is
153		 * used to maintain proper ordering.
154		 */
155
156		error = msleep(&td->td_umtx, &umtx_lock,
157		    td->td_priority | PCATCH, "umtx", 0);
158
159		/*
160		 * When we are woken up we need to see if we now own the lock
161		 * even if a signal was delivered.
162		 */
163		if ((owner = fuword(&umtx->u_owner)) == -1) {
164			error = EFAULT;
165			break;
166		}
167		owner &= ~UMTX_CONTESTED;
168		if ((struct thread *)owner == td) {
169			error = 0;
170			break;
171		}
172
173		/*
174		 * We may have signals to deliver.
175		 */
176		if (error)
177			break;
178	}
179
180out:
181	UMTX_UNLOCK();
182
183	return (error);
184}
185
186int
187_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
188    /* struct umtx *umtx */
189{
190	struct thread *td0;
191	struct umtx *umtx;
192	intptr_t owner;
193	intptr_t blocked;
194	intptr_t old;
195	int error;
196
197	error = 0;
198	umtx = uap->umtx;
199
200	UMTX_LOCK();
201
202	/*
203	 * Make sure we own this mtx.
204	 *
205	 * XXX Need a {fu,su}ptr this is not correct on arch where
206	 * sizeof(intptr_t) != sizeof(long).
207	 */
208	if ((owner = fuword(&umtx->u_owner)) == -1) {
209		error = EFAULT;
210		goto out;
211	}
212	if ((struct thread *)(owner & ~UMTX_CONTESTED) != td) {
213		error = EPERM;
214		goto out;
215	}
216	/*
217	 * If we own it but it isn't contested then we can just release and
218	 * return.
219	 */
220	if ((owner & UMTX_CONTESTED) == 0) {
221		owner = casuptr((intptr_t *)&umtx->u_owner,
222		    (intptr_t)td, UMTX_UNOWNED);
223
224		if (owner == -1)
225			error = EFAULT;
226		/*
227		 * If this failed someone modified the memory without going
228		 * through this api.
229		 */
230		else if (owner != (intptr_t)td)
231			error = EINVAL;
232		else
233			error = 0;
234
235		goto out;
236	}
237
238	/*
239	 * Since we own the mutex and the proc lock we are free to inspect
240	 * the blocked queue.  It must have one valid entry since the
241	 * CONTESTED bit was set.
242	 */
243	blocked = fuword(&umtx->u_blocked);
244	if (blocked == -1) {
245		error = EFAULT;
246		goto out;
247	}
248	if (blocked == 0) {
249		error = EINVAL;
250		goto out;
251	}
252
253	FOREACH_THREAD_IN_PROC(td->td_proc, td0)
254		if (td0 == (struct thread *)blocked)
255			break;
256
257	if (td0 == NULL) {
258		error = EINVAL;
259		goto out;
260	}
261
262	if (!STAILQ_EMPTY(&td0->td_umtxq)) {
263		struct thread *next;
264
265		blocked |= UMTX_CONTESTED;
266		next = STAILQ_FIRST(&td0->td_umtxq);
267		if (suword(&umtx->u_blocked, (long)next) == -1) {
268			error = EFAULT;
269			goto out;
270		}
271		STAILQ_REMOVE_HEAD(&td0->td_umtxq, td_umtx);
272
273		/*
274		 * Switch the queue over to the next blocked thread.
275		 */
276		if (!STAILQ_EMPTY(&td0->td_umtxq)) {
277			next->td_umtxq = td0->td_umtxq;
278			STAILQ_INIT(&td0->td_umtxq);
279		} else
280			STAILQ_INIT(&next->td_umtxq);
281	} else {
282		if (suword(&umtx->u_blocked, UMTX_UNOWNED) == -1) {
283			error = EFAULT;
284			goto out;
285		}
286	}
287	/*
288	 * Now directly assign this mutex to the first thread that was
289	 * blocked on it.
290	 */
291	old = casuptr((intptr_t *)&umtx->u_owner, owner, blocked);
292
293	/*
294	 * This will only happen if someone modifies the lock without going
295	 * through this api.
296	 */
297	if (old != owner) {
298		error = EINVAL;
299		goto out;
300	}
301	if (old == -1) {
302		error = EFAULT;
303		goto out;
304	}
305	/* Success. */
306	error = 0;
307	wakeup(&td0->td_umtx);
308
309out:
310	UMTX_UNLOCK();
311
312	return (error);
313}
314