Deleted Added
sdiff udiff text old ( 251522 ) new ( 251526 )
full compact
vfs_aio.c (251522) vfs_aio.c (251526)
1/*-
2 * Copyright (c) 1997 John S. Dyson. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 5 unchanged lines hidden (view full) ---

14 * of the author. This software is distributed AS-IS.
15 */
16
17/*
18 * This file contains support for the POSIX 1003.1B AIO/LIO facility.
19 */
20
21#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1997 John S. Dyson. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 5 unchanged lines hidden (view full) ---

14 * of the author. This software is distributed AS-IS.
15 */
16
17/*
18 * This file contains support for the POSIX 1003.1B AIO/LIO facility.
19 */
20
21#include <sys/cdefs.h>
22__FBSDID("$FreeBSD: head/sys/kern/vfs_aio.c 251522 2013-06-08 13:02:43Z glebius $");
22__FBSDID("$FreeBSD: head/sys/kern/vfs_aio.c 251526 2013-06-08 13:27:57Z glebius $");
23
24#include "opt_compat.h"
25
26#include <sys/param.h>
27#include <sys/systm.h>
28#include <sys/malloc.h>
29#include <sys/bio.h>
30#include <sys/buf.h>

--- 304 unchanged lines hidden (view full) ---

335static TAILQ_HEAD(,aiocblist) aio_jobs; /* (c) Async job list */
336static struct unrhdr *aiod_unr;
337
338void aio_init_aioinfo(struct proc *p);
339static int aio_onceonly(void);
340static int aio_free_entry(struct aiocblist *aiocbe);
341static void aio_process_rw(struct aiocblist *aiocbe);
342static void aio_process_sync(struct aiocblist *aiocbe);
23
24#include "opt_compat.h"
25
26#include <sys/param.h>
27#include <sys/systm.h>
28#include <sys/malloc.h>
29#include <sys/bio.h>
30#include <sys/buf.h>

--- 304 unchanged lines hidden (view full) ---

335static TAILQ_HEAD(,aiocblist) aio_jobs; /* (c) Async job list */
336static struct unrhdr *aiod_unr;
337
338void aio_init_aioinfo(struct proc *p);
339static int aio_onceonly(void);
340static int aio_free_entry(struct aiocblist *aiocbe);
341static void aio_process_rw(struct aiocblist *aiocbe);
342static void aio_process_sync(struct aiocblist *aiocbe);
343static void aio_process_mlock(struct aiocblist *aiocbe);
343static int aio_newproc(int *);
344int aio_aqueue(struct thread *td, struct aiocb *job,
345 struct aioliojob *lio, int type, struct aiocb_ops *ops);
346static void aio_physwakeup(struct buf *bp);
347static void aio_proc_rundown(void *arg, struct proc *p);
348static void aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp);
349static int aio_qphysio(struct proc *p, struct aiocblist *iocb);
350static void biohelper(void *, int);

--- 70 unchanged lines hidden (view full) ---

421 &aio_modload,
422 NULL
423};
424
425static struct syscall_helper_data aio_syscalls[] = {
426 SYSCALL_INIT_HELPER(aio_cancel),
427 SYSCALL_INIT_HELPER(aio_error),
428 SYSCALL_INIT_HELPER(aio_fsync),
344static int aio_newproc(int *);
345int aio_aqueue(struct thread *td, struct aiocb *job,
346 struct aioliojob *lio, int type, struct aiocb_ops *ops);
347static void aio_physwakeup(struct buf *bp);
348static void aio_proc_rundown(void *arg, struct proc *p);
349static void aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp);
350static int aio_qphysio(struct proc *p, struct aiocblist *iocb);
351static void biohelper(void *, int);

--- 70 unchanged lines hidden (view full) ---

422 &aio_modload,
423 NULL
424};
425
426static struct syscall_helper_data aio_syscalls[] = {
427 SYSCALL_INIT_HELPER(aio_cancel),
428 SYSCALL_INIT_HELPER(aio_error),
429 SYSCALL_INIT_HELPER(aio_fsync),
430 SYSCALL_INIT_HELPER(aio_mlock),
429 SYSCALL_INIT_HELPER(aio_read),
430 SYSCALL_INIT_HELPER(aio_return),
431 SYSCALL_INIT_HELPER(aio_suspend),
432 SYSCALL_INIT_HELPER(aio_waitcomplete),
433 SYSCALL_INIT_HELPER(aio_write),
434 SYSCALL_INIT_HELPER(lio_listio),
435 SYSCALL_INIT_HELPER(oaio_read),
436 SYSCALL_INIT_HELPER(oaio_write),

--- 11 unchanged lines hidden (view full) ---

448#include <compat/freebsd32/freebsd32_util.h>
449
450static struct syscall_helper_data aio32_syscalls[] = {
451 SYSCALL32_INIT_HELPER(freebsd32_aio_return),
452 SYSCALL32_INIT_HELPER(freebsd32_aio_suspend),
453 SYSCALL32_INIT_HELPER(freebsd32_aio_cancel),
454 SYSCALL32_INIT_HELPER(freebsd32_aio_error),
455 SYSCALL32_INIT_HELPER(freebsd32_aio_fsync),
431 SYSCALL_INIT_HELPER(aio_read),
432 SYSCALL_INIT_HELPER(aio_return),
433 SYSCALL_INIT_HELPER(aio_suspend),
434 SYSCALL_INIT_HELPER(aio_waitcomplete),
435 SYSCALL_INIT_HELPER(aio_write),
436 SYSCALL_INIT_HELPER(lio_listio),
437 SYSCALL_INIT_HELPER(oaio_read),
438 SYSCALL_INIT_HELPER(oaio_write),

--- 11 unchanged lines hidden (view full) ---

450#include <compat/freebsd32/freebsd32_util.h>
451
452static struct syscall_helper_data aio32_syscalls[] = {
453 SYSCALL32_INIT_HELPER(freebsd32_aio_return),
454 SYSCALL32_INIT_HELPER(freebsd32_aio_suspend),
455 SYSCALL32_INIT_HELPER(freebsd32_aio_cancel),
456 SYSCALL32_INIT_HELPER(freebsd32_aio_error),
457 SYSCALL32_INIT_HELPER(freebsd32_aio_fsync),
458 SYSCALL32_INIT_HELPER(freebsd32_aio_mlock),
456 SYSCALL32_INIT_HELPER(freebsd32_aio_read),
457 SYSCALL32_INIT_HELPER(freebsd32_aio_write),
458 SYSCALL32_INIT_HELPER(freebsd32_aio_waitcomplete),
459 SYSCALL32_INIT_HELPER(freebsd32_lio_listio),
460 SYSCALL32_INIT_HELPER(freebsd32_oaio_read),
461 SYSCALL32_INIT_HELPER(freebsd32_oaio_write),
462 SYSCALL32_INIT_HELPER(freebsd32_olio_listio),
463 SYSCALL_INIT_LAST

--- 233 unchanged lines hidden (view full) ---

697 * curthread can't exit since we are curthread.
698 *
699 * Therefore, we use curthread as the thread to pass to
700 * knlist_delete(). This does mean that it is possible for the
701 * thread pointer at close time to differ from the thread pointer
702 * at open time, but this is already true of file descriptors in
703 * a multithreaded process.
704 */
459 SYSCALL32_INIT_HELPER(freebsd32_aio_read),
460 SYSCALL32_INIT_HELPER(freebsd32_aio_write),
461 SYSCALL32_INIT_HELPER(freebsd32_aio_waitcomplete),
462 SYSCALL32_INIT_HELPER(freebsd32_lio_listio),
463 SYSCALL32_INIT_HELPER(freebsd32_oaio_read),
464 SYSCALL32_INIT_HELPER(freebsd32_oaio_write),
465 SYSCALL32_INIT_HELPER(freebsd32_olio_listio),
466 SYSCALL_INIT_LAST

--- 233 unchanged lines hidden (view full) ---

700 * curthread can't exit since we are curthread.
701 *
702 * Therefore, we use curthread as the thread to pass to
703 * knlist_delete(). This does mean that it is possible for the
704 * thread pointer at close time to differ from the thread pointer
705 * at open time, but this is already true of file descriptors in
706 * a multithreaded process.
707 */
705 fdrop(aiocbe->fd_file, curthread);
708 if (aiocbe->fd_file)
709 fdrop(aiocbe->fd_file, curthread);
706 crfree(aiocbe->cred);
707 uma_zfree(aiocb_zone, aiocbe);
708 AIO_LOCK(ki);
709
710 return (0);
711}
712
713static void

--- 249 unchanged lines hidden (view full) ---

963 if (fp->f_vnode != NULL)
964 error = aio_fsync_vnode(td, fp->f_vnode);
965 cb->_aiocb_private.error = error;
966 cb->_aiocb_private.status = 0;
967 td->td_ucred = td_savedcred;
968}
969
970static void
710 crfree(aiocbe->cred);
711 uma_zfree(aiocb_zone, aiocbe);
712 AIO_LOCK(ki);
713
714 return (0);
715}
716
717static void

--- 249 unchanged lines hidden (view full) ---

967 if (fp->f_vnode != NULL)
968 error = aio_fsync_vnode(td, fp->f_vnode);
969 cb->_aiocb_private.error = error;
970 cb->_aiocb_private.status = 0;
971 td->td_ucred = td_savedcred;
972}
973
974static void
975aio_process_mlock(struct aiocblist *aiocbe)
976{
977 struct aiocb *cb = &aiocbe->uaiocb;
978 int error;
979
980 KASSERT(aiocbe->uaiocb.aio_lio_opcode == LIO_MLOCK,
981 ("%s: opcode %d", __func__, aiocbe->uaiocb.aio_lio_opcode));
982
983 error = vm_mlock(aiocbe->userproc, aiocbe->cred,
984 __DEVOLATILE(void *, cb->aio_buf), cb->aio_nbytes);
985 cb->_aiocb_private.error = error;
986 cb->_aiocb_private.status = 0;
987}
988
989static void
971aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type)
972{
973 struct aioliojob *lj;
974 struct kaioinfo *ki;
975 struct aiocblist *scb, *scbn;
976 int lj_done;
977
978 ki = userp->p_aioinfo;

--- 159 unchanged lines hidden (view full) ---

1138 switch(aiocbe->uaiocb.aio_lio_opcode) {
1139 case LIO_READ:
1140 case LIO_WRITE:
1141 aio_process_rw(aiocbe);
1142 break;
1143 case LIO_SYNC:
1144 aio_process_sync(aiocbe);
1145 break;
990aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type)
991{
992 struct aioliojob *lj;
993 struct kaioinfo *ki;
994 struct aiocblist *scb, *scbn;
995 int lj_done;
996
997 ki = userp->p_aioinfo;

--- 159 unchanged lines hidden (view full) ---

1157 switch(aiocbe->uaiocb.aio_lio_opcode) {
1158 case LIO_READ:
1159 case LIO_WRITE:
1160 aio_process_rw(aiocbe);
1161 break;
1162 case LIO_SYNC:
1163 aio_process_sync(aiocbe);
1164 break;
1165 case LIO_MLOCK:
1166 aio_process_mlock(aiocbe);
1167 break;
1146 }
1147
1148 mtx_lock(&aio_job_mtx);
1149 /* Decrement the active job count. */
1150 ki->kaio_active_count--;
1151 mtx_unlock(&aio_job_mtx);
1152
1153 AIO_LOCK(ki);

--- 124 unchanged lines hidden (view full) ---

1278 struct cdev *dev;
1279 struct kaioinfo *ki;
1280 struct aioliojob *lj;
1281 int error, ref;
1282
1283 cb = &aiocbe->uaiocb;
1284 fp = aiocbe->fd_file;
1285
1168 }
1169
1170 mtx_lock(&aio_job_mtx);
1171 /* Decrement the active job count. */
1172 ki->kaio_active_count--;
1173 mtx_unlock(&aio_job_mtx);
1174
1175 AIO_LOCK(ki);

--- 124 unchanged lines hidden (view full) ---

1300 struct cdev *dev;
1301 struct kaioinfo *ki;
1302 struct aioliojob *lj;
1303 int error, ref;
1304
1305 cb = &aiocbe->uaiocb;
1306 fp = aiocbe->fd_file;
1307
1286 if (fp->f_type != DTYPE_VNODE)
1308 if (fp == NULL || fp->f_type != DTYPE_VNODE)
1287 return (-1);
1288
1289 vp = fp->f_vnode;
1290
1291 /*
1292 * If its not a disk, we don't want to return a positive error.
1293 * It causes the aio code to not fall through to try the thread
1294 * way when you're talking to a regular file.

--- 335 unchanged lines hidden (view full) ---

1630 error = fget_write(td, fd, CAP_PWRITE, &fp);
1631 break;
1632 case LIO_READ:
1633 error = fget_read(td, fd, CAP_PREAD, &fp);
1634 break;
1635 case LIO_SYNC:
1636 error = fget(td, fd, CAP_FSYNC, &fp);
1637 break;
1309 return (-1);
1310
1311 vp = fp->f_vnode;
1312
1313 /*
1314 * If its not a disk, we don't want to return a positive error.
1315 * It causes the aio code to not fall through to try the thread
1316 * way when you're talking to a regular file.

--- 335 unchanged lines hidden (view full) ---

1652 error = fget_write(td, fd, CAP_PWRITE, &fp);
1653 break;
1654 case LIO_READ:
1655 error = fget_read(td, fd, CAP_PREAD, &fp);
1656 break;
1657 case LIO_SYNC:
1658 error = fget(td, fd, CAP_FSYNC, &fp);
1659 break;
1660 case LIO_MLOCK:
1661 fp = NULL;
1662 break;
1638 case LIO_NOP:
1639 error = fget(td, fd, CAP_NONE, &fp);
1640 break;
1641 default:
1642 error = EINVAL;
1643 }
1644 if (error) {
1645 uma_zfree(aiocb_zone, aiocbe);

--- 41 unchanged lines hidden (view full) ---

1687 kev.ident = (uintptr_t)aiocbe->uuaiocb;
1688 kev.filter = EVFILT_AIO;
1689 kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | evflags;
1690 kev.data = (intptr_t)aiocbe;
1691 kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sival_ptr;
1692 error = kqfd_register(kqfd, &kev, td, 1);
1693aqueue_fail:
1694 if (error) {
1663 case LIO_NOP:
1664 error = fget(td, fd, CAP_NONE, &fp);
1665 break;
1666 default:
1667 error = EINVAL;
1668 }
1669 if (error) {
1670 uma_zfree(aiocb_zone, aiocbe);

--- 41 unchanged lines hidden (view full) ---

1712 kev.ident = (uintptr_t)aiocbe->uuaiocb;
1713 kev.filter = EVFILT_AIO;
1714 kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | evflags;
1715 kev.data = (intptr_t)aiocbe;
1716 kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sival_ptr;
1717 error = kqfd_register(kqfd, &kev, td, 1);
1718aqueue_fail:
1719 if (error) {
1695 fdrop(fp, td);
1720 if (fp)
1721 fdrop(fp, td);
1696 uma_zfree(aiocb_zone, aiocbe);
1697 ops->store_error(job, error);
1698 goto done;
1699 }
1700no_kqueue:
1701
1702 ops->store_error(job, EINPROGRESS);
1703 aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
1704 aiocbe->userproc = p;
1705 aiocbe->cred = crhold(td->td_ucred);
1706 aiocbe->jobflags = 0;
1707 aiocbe->lio = lj;
1708
1709 if (opcode == LIO_SYNC)
1710 goto queueit;
1711
1722 uma_zfree(aiocb_zone, aiocbe);
1723 ops->store_error(job, error);
1724 goto done;
1725 }
1726no_kqueue:
1727
1728 ops->store_error(job, EINPROGRESS);
1729 aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
1730 aiocbe->userproc = p;
1731 aiocbe->cred = crhold(td->td_ucred);
1732 aiocbe->jobflags = 0;
1733 aiocbe->lio = lj;
1734
1735 if (opcode == LIO_SYNC)
1736 goto queueit;
1737
1712 if (fp->f_type == DTYPE_SOCKET) {
1738 if (fp && fp->f_type == DTYPE_SOCKET) {
1713 /*
1714 * Alternate queueing for socket ops: Reach down into the
1715 * descriptor to get the socket data. Then check to see if the
1716 * socket is ready to be read or written (based on the requested
1717 * operation).
1718 *
1719 * If it is not ready for io, then queue the aiocbe on the
1720 * socket, and set the flags so we get a call when sbnotify()

--- 461 unchanged lines hidden (view full) ---

2182
2183int
2184sys_aio_write(struct thread *td, struct aio_write_args *uap)
2185{
2186
2187 return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
2188}
2189
1739 /*
1740 * Alternate queueing for socket ops: Reach down into the
1741 * descriptor to get the socket data. Then check to see if the
1742 * socket is ready to be read or written (based on the requested
1743 * operation).
1744 *
1745 * If it is not ready for io, then queue the aiocbe on the
1746 * socket, and set the flags so we get a call when sbnotify()

--- 461 unchanged lines hidden (view full) ---

2208
2209int
2210sys_aio_write(struct thread *td, struct aio_write_args *uap)
2211{
2212
2213 return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
2214}
2215
2216int
2217sys_aio_mlock(struct thread *td, struct aio_mlock_args *uap)
2218{
2219
2220 return (aio_aqueue(td, uap->aiocbp, NULL, LIO_MLOCK, &aiocb_ops));
2221}
2222
2190static int
2191kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
2192 struct aiocb **acb_list, int nent, struct sigevent *sig,
2193 struct aiocb_ops *ops)
2194{
2195 struct proc *p = td->td_proc;
2196 struct aiocb *iocb;
2197 struct kaioinfo *ki;

--- 726 unchanged lines hidden (view full) ---

2924freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
2925{
2926
2927 return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
2928 &aiocb32_ops));
2929}
2930
2931int
2223static int
2224kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
2225 struct aiocb **acb_list, int nent, struct sigevent *sig,
2226 struct aiocb_ops *ops)
2227{
2228 struct proc *p = td->td_proc;
2229 struct aiocb *iocb;
2230 struct kaioinfo *ki;

--- 726 unchanged lines hidden (view full) ---

2957freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
2958{
2959
2960 return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
2961 &aiocb32_ops));
2962}
2963
2964int
2965freebsd32_aio_mlock(struct thread *td, struct freebsd32_aio_mlock_args *uap)
2966{
2967
2968 return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_MLOCK,
2969 &aiocb32_ops));
2970}
2971
2972int
2932freebsd32_aio_waitcomplete(struct thread *td,
2933 struct freebsd32_aio_waitcomplete_args *uap)
2934{
2935 struct timespec32 ts32;
2936 struct timespec ts, *tsp;
2937 int error;
2938
2939 if (uap->timeout) {

--- 113 unchanged lines hidden --- 		2973freebsd32_aio_waitcomplete(struct thread *td,
2974 struct freebsd32_aio_waitcomplete_args *uap)
2975{
2976 struct timespec32 ts32;
2977 struct timespec ts, *tsp;
2978 int error;
2979
2980 if (uap->timeout) {

--- 113 unchanged lines hidden ---