1/*- 2 * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 216641 2010-12-22 05:01:52Z davidxu $"); |
30 31#include "opt_compat.h" 32#include <sys/param.h> 33#include <sys/kernel.h> 34#include <sys/limits.h> 35#include <sys/lock.h> 36#include <sys/malloc.h> 37#include <sys/mutex.h> 38#include <sys/priv.h> 39#include <sys/proc.h> 40#include <sys/sched.h> 41#include <sys/smp.h> 42#include <sys/sysctl.h> 43#include <sys/sysent.h> 44#include <sys/systm.h> 45#include <sys/sysproto.h> |
46#include <sys/syscallsubr.h> |
47#include <sys/eventhandler.h> 48#include <sys/umtx.h> 49 50#include <vm/vm.h> 51#include <vm/vm_param.h> 52#include <vm/pmap.h> 53#include <vm/vm_map.h> 54#include <vm/vm_object.h> --- 2292 unchanged lines hidden (view full) --- 2347static int 2348do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2349 struct timespec *timeout, u_long wflags) 2350{ 2351 struct umtx_q *uq; 2352 struct timeval tv; 2353 struct timespec cts, ets, tts; 2354 uint32_t flags; |
2355 uint32_t clockid; |
2356 int error; 2357 2358 uq = td->td_umtxq; 2359 flags = fuword32(&cv->c_flags); 2360 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2361 if (error != 0) 2362 return (error); |
2363 2364 if ((wflags & CVWAIT_CLOCKID) != 0) { 2365 clockid = fuword32(&cv->c_clockid); 2366 if (clockid < CLOCK_REALTIME || 2367 clockid >= CLOCK_THREAD_CPUTIME_ID) { 2368 /* hmm, only HW clock id will work. */ 2369 return (EINVAL); 2370 } 2371 } else { 2372 clockid = CLOCK_REALTIME; 2373 } 2374 |
2375 umtxq_lock(&uq->uq_key); 2376 umtxq_busy(&uq->uq_key); 2377 umtxq_insert(uq); 2378 umtxq_unlock(&uq->uq_key); 2379 2380 /* |
2381 * Set c_has_waiters to 1 before releasing user mutex, also 2382 * don't modify cache line when unnecessary. |
2383 */ |
2384 if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0) 2385 suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); |
2386 2387 umtxq_lock(&uq->uq_key); 2388 umtxq_unbusy(&uq->uq_key); 2389 umtxq_unlock(&uq->uq_key); 2390 2391 error = do_unlock_umutex(td, m); 2392 2393 umtxq_lock(&uq->uq_key); 2394 if (error == 0) { |
2395 if (timeout == NULL) { |
2396 error = umtxq_sleep(uq, "ucond", 0); 2397 } else { |
2398 if ((wflags & CVWAIT_ABSTIME) == 0) { 2399 kern_clock_gettime(td, clockid, &ets); 2400 timespecadd(&ets, timeout); 2401 tts = *timeout; 2402 } else { /* absolute time */ 2403 ets = *timeout; 2404 tts = *timeout; 2405 kern_clock_gettime(td, clockid, &cts); 2406 timespecsub(&tts, &cts); 2407 } 2408 TIMESPEC_TO_TIMEVAL(&tv, &tts); |
2409 for (;;) { 2410 error = umtxq_sleep(uq, "ucond", tvtohz(&tv)); 2411 if (error != ETIMEDOUT) 2412 break; |
2413 kern_clock_gettime(td, clockid, &cts); |
2414 if (timespeccmp(&cts, &ets, >=)) { 2415 error = ETIMEDOUT; 2416 break; 2417 } 2418 tts = ets; 2419 timespecsub(&tts, &cts); 2420 TIMESPEC_TO_TIMEVAL(&tv, &tts); 2421 } 2422 } 2423 } 2424 2425 if ((uq->uq_flags & UQF_UMTXQ) == 0) 2426 error = 0; 2427 else { |
2428 /* 2429 * This must be timeout,interrupted by signal or 2430 * surprious wakeup, clear c_has_waiter flag when 2431 * necessary. 2432 */ 2433 umtxq_busy(&uq->uq_key); 2434 if ((uq->uq_flags & UQF_UMTXQ) != 0) { 2435 int oldlen = uq->uq_cur_queue->length; 2436 umtxq_remove(uq); 2437 if (oldlen == 1) { 2438 umtxq_unlock(&uq->uq_key); 2439 suword32( 2440 __DEVOLATILE(uint32_t *, 2441 &cv->c_has_waiters), 0); 2442 umtxq_lock(&uq->uq_key); 2443 } 2444 } 2445 umtxq_unbusy(&uq->uq_key); |
2446 if (error == ERESTART) 2447 error = EINTR; 2448 } 2449 2450 umtxq_unlock(&uq->uq_key); 2451 umtx_key_release(&uq->uq_key); 2452 return (error); 2453} --- 606 unchanged lines hidden (view full) --- 3060} 3061 3062static int 3063__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 3064{ 3065 return (kern_umtx_wake(td, uap->obj, uap->val, 0)); 3066} 3067 |
3068#define BATCH_SIZE 128 |
3069static int |
3070__umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) 3071{ 3072 int count = uap->val; 3073 void *uaddrs[BATCH_SIZE]; 3074 char **upp = (char **)uap->obj; 3075 int tocopy; 3076 int error = 0; 3077 int i, pos = 0; 3078 3079 while (count > 0) { 3080 tocopy = count; 3081 if (tocopy > BATCH_SIZE) 3082 tocopy = BATCH_SIZE; 3083 error = copyin(upp+pos, uaddrs, tocopy * sizeof(char *)); 3084 if (error != 0) 3085 break; 3086 for (i = 0; i < tocopy; ++i) 3087 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); 3088 count -= tocopy; 3089 pos += tocopy; 3090 } 3091 return (error); 3092} 3093 3094static int |
3095__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) 3096{ 3097 return (kern_umtx_wake(td, uap->obj, uap->val, 1)); 3098} 3099 3100static int 3101__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 3102{ --- 199 unchanged lines hidden (view full) --- 3302 __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */ 3303 __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */ 3304 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3305 __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3306 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3307 __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */ 3308 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */ 3309 __umtx_op_sem_wait, /* UMTX_OP_SEM_WAIT */ |
3310 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */ 3311 __umtx_op_nwake_private /* UMTX_OP_NWAKE_PRIVATE */ |
3312}; 3313 3314int 3315_umtx_op(struct thread *td, struct _umtx_op_args *uap) 3316{ 3317 if ((unsigned)uap->op < UMTX_OP_MAX) 3318 return (*op_table[uap->op])(td, uap); 3319 return (EINVAL); --- 225 unchanged lines hidden (view full) --- 3545 if (timeout.tv_nsec >= 1000000000 || 3546 timeout.tv_nsec < 0) 3547 return (EINVAL); 3548 ts = &timeout; 3549 } 3550 return (do_sem_wait(td, uap->obj, ts)); 3551} 3552 |
3553static int 3554__umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) 3555{ 3556 int count = uap->val; 3557 uint32_t uaddrs[BATCH_SIZE]; 3558 uint32_t **upp = (uint32_t **)uap->obj; 3559 int tocopy; 3560 int error = 0; 3561 int i, pos = 0; 3562 3563 while (count > 0) { 3564 tocopy = count; 3565 if (tocopy > BATCH_SIZE) 3566 tocopy = BATCH_SIZE; 3567 error = copyin(upp+pos, uaddrs, tocopy * sizeof(uint32_t)); 3568 if (error != 0) 3569 break; 3570 for (i = 0; i < tocopy; ++i) 3571 kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i], 3572 INT_MAX, 1); 3573 count -= tocopy; 3574 pos += tocopy; 3575 } 3576 return (error); 3577} 3578 |
3579static _umtx_op_func op_table_compat32[] = { 3580 __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ 3581 __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ 3582 __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ 3583 __umtx_op_wake, /* UMTX_OP_WAKE */ 3584 __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ 3585 __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ 3586 __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ --- 5 unchanged lines hidden (view full) --- 3592 __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */ 3593 __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */ 3594 __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3595 __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3596 __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3597 __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */ 3598 __umtx_op_wake_umutex, /* UMTX_OP_UMUTEX_WAKE */ 3599 __umtx_op_sem_wait_compat32, /* UMTX_OP_SEM_WAIT */ |
3600 __umtx_op_sem_wake, /* UMTX_OP_SEM_WAKE */ 3601 __umtx_op_nwake_private32 /* UMTX_OP_NWAKE_PRIVATE */ |
3602}; 3603 3604int 3605freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) 3606{ 3607 if ((unsigned)uap->op < UMTX_OP_MAX) 3608 return (*op_table_compat32[uap->op])(td, 3609 (struct _umtx_op_args *)uap); --- 76 unchanged lines hidden --- |