Deleted Added
sdiff udiff text old ( 216463 ) new ( 216641 )
full compact
kern_umtx.c (216463) kern_umtx.c (216641)
1/*-
2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 216463 2010-12-15 19:30:44Z mdf $");
29__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 216641 2010-12-22 05:01:52Z davidxu $");
30
31#include "opt_compat.h"
32#include <sys/param.h>
33#include <sys/kernel.h>
34#include <sys/limits.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mutex.h>
38#include <sys/priv.h>
39#include <sys/proc.h>
40#include <sys/sched.h>
41#include <sys/smp.h>
42#include <sys/sysctl.h>
43#include <sys/sysent.h>
44#include <sys/systm.h>
45#include <sys/sysproto.h>
30
31#include "opt_compat.h"
32#include <sys/param.h>
33#include <sys/kernel.h>
34#include <sys/limits.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mutex.h>
38#include <sys/priv.h>
39#include <sys/proc.h>
40#include <sys/sched.h>
41#include <sys/smp.h>
42#include <sys/sysctl.h>
43#include <sys/sysent.h>
44#include <sys/systm.h>
45#include <sys/sysproto.h>
46#include <sys/syscallsubr.h>
46#include <sys/eventhandler.h>
47#include <sys/umtx.h>
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/pmap.h>
52#include <vm/vm_map.h>
53#include <vm/vm_object.h>

--- 2292 unchanged lines hidden (view full) ---

2346static int
2347do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2348 struct timespec *timeout, u_long wflags)
2349{
2350 struct umtx_q *uq;
2351 struct timeval tv;
2352 struct timespec cts, ets, tts;
2353 uint32_t flags;
47#include <sys/eventhandler.h>
48#include <sys/umtx.h>
49
50#include <vm/vm.h>
51#include <vm/vm_param.h>
52#include <vm/pmap.h>
53#include <vm/vm_map.h>
54#include <vm/vm_object.h>

--- 2292 unchanged lines hidden (view full) ---

2347static int
2348do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2349 struct timespec *timeout, u_long wflags)
2350{
2351 struct umtx_q *uq;
2352 struct timeval tv;
2353 struct timespec cts, ets, tts;
2354 uint32_t flags;
2355 uint32_t clockid;
2354 int error;
2355
2356 uq = td->td_umtxq;
2357 flags = fuword32(&cv->c_flags);
2358 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2359 if (error != 0)
2360 return (error);
2356 int error;
2357
2358 uq = td->td_umtxq;
2359 flags = fuword32(&cv->c_flags);
2360 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2361 if (error != 0)
2362 return (error);
2363
2364 if ((wflags & CVWAIT_CLOCKID) != 0) {
2365 clockid = fuword32(&cv->c_clockid);
2366 if (clockid < CLOCK_REALTIME ||
2367 clockid >= CLOCK_THREAD_CPUTIME_ID) {
2368 /* hmm, only HW clock id will work. */
2369 return (EINVAL);
2370 }
2371 } else {
2372 clockid = CLOCK_REALTIME;
2373 }
2374
2361 umtxq_lock(&uq->uq_key);
2362 umtxq_busy(&uq->uq_key);
2363 umtxq_insert(uq);
2364 umtxq_unlock(&uq->uq_key);
2365
2366 /*
2375 umtxq_lock(&uq->uq_key);
2376 umtxq_busy(&uq->uq_key);
2377 umtxq_insert(uq);
2378 umtxq_unlock(&uq->uq_key);
2379
2380 /*
2367 * The magic thing is we should set c_has_waiters to 1 before
2368 * releasing user mutex.
2381 * Set c_has_waiters to 1 before releasing user mutex, also
2382 * don't modify cache line when unnecessary.
2369 */
2383 */
2370 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2384 if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0)
2385 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
2371
2372 umtxq_lock(&uq->uq_key);
2373 umtxq_unbusy(&uq->uq_key);
2374 umtxq_unlock(&uq->uq_key);
2375
2376 error = do_unlock_umutex(td, m);
2377
2378 umtxq_lock(&uq->uq_key);
2379 if (error == 0) {
2386
2387 umtxq_lock(&uq->uq_key);
2388 umtxq_unbusy(&uq->uq_key);
2389 umtxq_unlock(&uq->uq_key);
2390
2391 error = do_unlock_umutex(td, m);
2392
2393 umtxq_lock(&uq->uq_key);
2394 if (error == 0) {
2380 if ((wflags & UMTX_CHECK_UNPARKING) &&
2381 (td->td_pflags & TDP_WAKEUP)) {
2382 td->td_pflags &= ~TDP_WAKEUP;
2383 error = EINTR;
2384 } else if (timeout == NULL) {
2395 if (timeout == NULL) {
2385 error = umtxq_sleep(uq, "ucond", 0);
2386 } else {
2396 error = umtxq_sleep(uq, "ucond", 0);
2397 } else {
2387 getnanouptime(&ets);
2388 timespecadd(&ets, timeout);
2389 TIMESPEC_TO_TIMEVAL(&tv, timeout);
2398 if ((wflags & CVWAIT_ABSTIME) == 0) {
2399 kern_clock_gettime(td, clockid, &ets);
2400 timespecadd(&ets, timeout);
2401 tts = *timeout;
2402 } else { /* absolute time */
2403 ets = *timeout;
2404 tts = *timeout;
2405 kern_clock_gettime(td, clockid, &cts);
2406 timespecsub(&tts, &cts);
2407 }
2408 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2390 for (;;) {
2391 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2392 if (error != ETIMEDOUT)
2393 break;
2409 for (;;) {
2410 error = umtxq_sleep(uq, "ucond", tvtohz(&tv));
2411 if (error != ETIMEDOUT)
2412 break;
2394 getnanouptime(&cts);
2413 kern_clock_gettime(td, clockid, &cts);
2395 if (timespeccmp(&cts, &ets, >=)) {
2396 error = ETIMEDOUT;
2397 break;
2398 }
2399 tts = ets;
2400 timespecsub(&tts, &cts);
2401 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2402 }
2403 }
2404 }
2405
2406 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2407 error = 0;
2408 else {
2414 if (timespeccmp(&cts, &ets, >=)) {
2415 error = ETIMEDOUT;
2416 break;
2417 }
2418 tts = ets;
2419 timespecsub(&tts, &cts);
2420 TIMESPEC_TO_TIMEVAL(&tv, &tts);
2421 }
2422 }
2423 }
2424
2425 if ((uq->uq_flags & UQF_UMTXQ) == 0)
2426 error = 0;
2427 else {
2409 umtxq_remove(uq);
2428 /*
2429 * This must be timeout,interrupted by signal or
2430 * surprious wakeup, clear c_has_waiter flag when
2431 * necessary.
2432 */
2433 umtxq_busy(&uq->uq_key);
2434 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
2435 int oldlen = uq->uq_cur_queue->length;
2436 umtxq_remove(uq);
2437 if (oldlen == 1) {
2438 umtxq_unlock(&uq->uq_key);
2439 suword32(
2440 __DEVOLATILE(uint32_t *,
2441 &cv->c_has_waiters), 0);
2442 umtxq_lock(&uq->uq_key);
2443 }
2444 }
2445 umtxq_unbusy(&uq->uq_key);
2410 if (error == ERESTART)
2411 error = EINTR;
2412 }
2413
2414 umtxq_unlock(&uq->uq_key);
2415 umtx_key_release(&uq->uq_key);
2416 return (error);
2417}

--- 606 unchanged lines hidden (view full) ---

3024}
3025
3026static int
3027__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3028{
3029 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3030}
3031
2446 if (error == ERESTART)
2447 error = EINTR;
2448 }
2449
2450 umtxq_unlock(&uq->uq_key);
2451 umtx_key_release(&uq->uq_key);
2452 return (error);
2453}

--- 606 unchanged lines hidden (view full) ---

3060}
3061
3062static int
3063__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3064{
3065 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3066}
3067
3068#define BATCH_SIZE 128
3032static int
3069static int
3070__umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap)
3071{
3072 int count = uap->val;
3073 void *uaddrs[BATCH_SIZE];
3074 char **upp = (char **)uap->obj;
3075 int tocopy;
3076 int error = 0;
3077 int i, pos = 0;
3078
3079 while (count > 0) {
3080 tocopy = count;
3081 if (tocopy > BATCH_SIZE)
3082 tocopy = BATCH_SIZE;
3083 error = copyin(upp+pos, uaddrs, tocopy * sizeof(char *));
3084 if (error != 0)
3085 break;
3086 for (i = 0; i < tocopy; ++i)
3087 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
3088 count -= tocopy;
3089 pos += tocopy;
3090 }
3091 return (error);
3092}
3093
3094static int
3033__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3034{
3035 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3036}
3037
3038static int
3039__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3040{

--- 199 unchanged lines hidden (view full) ---

3240 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3241 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3242 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3243 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3244 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3245 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3246 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3247 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */
3095__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3096{
3097 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3098}
3099
3100static int
3101__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3102{

--- 199 unchanged lines hidden (view full) ---

3302 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */
3303 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */
3304 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3305 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */
3306 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3307 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */
3308 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3309 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */
3248 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3310 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */
3311 __umtx_op_nwake_private /* UMTX_OP_NWAKE_PRIVATE */
3249};
3250
3251int
3252_umtx_op(struct thread *td, struct _umtx_op_args *uap)
3253{
3254 if ((unsigned)uap->op < UMTX_OP_MAX)
3255 return (*op_table[uap->op])(td, uap);
3256 return (EINVAL);

--- 225 unchanged lines hidden (view full) ---

3482 if (timeout.tv_nsec >= 1000000000 ||
3483 timeout.tv_nsec < 0)
3484 return (EINVAL);
3485 ts = &timeout;
3486 }
3487 return (do_sem_wait(td, uap->obj, ts));
3488}
3489
3312};
3313
3314int
3315_umtx_op(struct thread *td, struct _umtx_op_args *uap)
3316{
3317 if ((unsigned)uap->op < UMTX_OP_MAX)
3318 return (*op_table[uap->op])(td, uap);
3319 return (EINVAL);

--- 225 unchanged lines hidden (view full) ---

3545 if (timeout.tv_nsec >= 1000000000 ||
3546 timeout.tv_nsec < 0)
3547 return (EINVAL);
3548 ts = &timeout;
3549 }
3550 return (do_sem_wait(td, uap->obj, ts));
3551}
3552
3553static int
3554__umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap)
3555{
3556 int count = uap->val;
3557 uint32_t uaddrs[BATCH_SIZE];
3558 uint32_t **upp = (uint32_t **)uap->obj;
3559 int tocopy;
3560 int error = 0;
3561 int i, pos = 0;
3562
3563 while (count > 0) {
3564 tocopy = count;
3565 if (tocopy > BATCH_SIZE)
3566 tocopy = BATCH_SIZE;
3567 error = copyin(upp+pos, uaddrs, tocopy * sizeof(uint32_t));
3568 if (error != 0)
3569 break;
3570 for (i = 0; i < tocopy; ++i)
3571 kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i],
3572 INT_MAX, 1);
3573 count -= tocopy;
3574 pos += tocopy;
3575 }
3576 return (error);
3577}
3578
3490static _umtx_op_func op_table_compat32[] = {
3491 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3492 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3493 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3494 __umtx_op_wake, /* UMTX_OP_WAKE */
3495 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3496 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3497 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */

--- 5 unchanged lines hidden (view full) ---

3503 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3504 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3505 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3506 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3507 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3508 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3509 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3510 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */
3579static _umtx_op_func op_table_compat32[] = {
3580 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */
3581 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */
3582 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */
3583 __umtx_op_wake, /* UMTX_OP_WAKE */
3584 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */
3585 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */
3586 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */

--- 5 unchanged lines hidden (view full) ---

3592 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */
3593 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */
3594 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */
3595 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */
3596 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */
3597 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
3598 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */
3599 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */
3511 __umtx_op_sem_wake /* UMTX_OP_SEM_WAKE */
3600 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */
3601 __umtx_op_nwake_private32 /* UMTX_OP_NWAKE_PRIVATE */
3512};
3513
3514int
3515freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3516{
3517 if ((unsigned)uap->op < UMTX_OP_MAX)
3518 return (*op_table_compat32[uap->op])(td,
3519 (struct _umtx_op_args *)uap);

--- 76 unchanged lines hidden --- 		3602};
3603
3604int
3605freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap)
3606{
3607 if ((unsigned)uap->op < UMTX_OP_MAX)
3608 return (*op_table_compat32[uap->op])(td,
3609 (struct _umtx_op_args *)uap);

--- 76 unchanged lines hidden ---