Deleted Added
sdiff udiff text old ( 98499 ) new ( 98849 )
full compact
uipc_socket.c (98499) uipc_socket.c (98849)
1/*
2 * Copyright (c) 1982, 1986, 1988, 1990, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 17 unchanged lines hidden (view full) ---

26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
1/*
2 * Copyright (c) 1982, 1986, 1988, 1990, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 17 unchanged lines hidden (view full) ---

26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
34 * $FreeBSD: head/sys/kern/uipc_socket.c 98499 2002-06-20 18:52:54Z alfred $
34 * $FreeBSD: head/sys/kern/uipc_socket.c 98849 2002-06-26 03:37:47Z ken $
35 */
36
37#include "opt_inet.h"
35 */
36
37#include "opt_inet.h"
38#include "opt_zero.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/fcntl.h>
42#include <sys/lock.h>
43#include <sys/malloc.h>
44#include <sys/mbuf.h>
45#include <sys/mutex.h>

--- 43 unchanged lines hidden (view full) ---

89SYSCTL_DECL(_kern_ipc);
90
91static int somaxconn = SOMAXCONN;
92SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
93 &somaxconn, 0, "Maximum pending socket connection queue size");
94static int numopensockets;
95SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
96 &numopensockets, 0, "Number of open sockets");
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/fcntl.h>
43#include <sys/lock.h>
44#include <sys/malloc.h>
45#include <sys/mbuf.h>
46#include <sys/mutex.h>

--- 43 unchanged lines hidden (view full) ---

90SYSCTL_DECL(_kern_ipc);
91
92static int somaxconn = SOMAXCONN;
93SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
94 &somaxconn, 0, "Maximum pending socket connection queue size");
95static int numopensockets;
96SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
97 &numopensockets, 0, "Number of open sockets");
98#ifdef ZERO_COPY_SOCKETS
99/* These aren't static because they're used in other files. */
100int so_zero_copy_send = 1;
101int so_zero_copy_receive = 1;
102SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0,
103 "Zero copy controls");
104SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW,
105 &so_zero_copy_receive, 0, "Enable zero copy receive");
106SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW,
107 &so_zero_copy_send, 0, "Enable zero copy send");
108#endif /* ZERO_COPY_SOCKETS */
97
98
99/*
100 * Socket operation routines.
101 * These routines are called by the routines in
102 * sys_socket.c or from a system process, and
103 * implement the semantics of socket operations by
104 * switching out to the protocol specific routines.

--- 361 unchanged lines hidden (view full) ---

466 * otherwise by the mbuf chain "top" (which must be null
467 * if uio is not). Data provided in mbuf chain must be small
468 * enough to send all at once.
469 *
470 * Returns nonzero on error, timeout or signal; callers
471 * must check for short counts if EINTR/ERESTART are returned.
472 * Data and control buffers are freed on return.
473 */
109
110
111/*
112 * Socket operation routines.
113 * These routines are called by the routines in
114 * sys_socket.c or from a system process, and
115 * implement the semantics of socket operations by
116 * switching out to the protocol specific routines.

--- 361 unchanged lines hidden (view full) ---

478 * otherwise by the mbuf chain "top" (which must be null
479 * if uio is not). Data provided in mbuf chain must be small
480 * enough to send all at once.
481 *
482 * Returns nonzero on error, timeout or signal; callers
483 * must check for short counts if EINTR/ERESTART are returned.
484 * Data and control buffers are freed on return.
485 */
486
487#ifdef ZERO_COPY_SOCKETS
488struct so_zerocopy_stats{
489 int size_ok;
490 int align_ok;
491 int found_ifp;
492};
493struct so_zerocopy_stats so_zerocp_stats = {0,0,0};
494#include <netinet/in.h>
495#include <net/route.h>
496#include <netinet/in_pcb.h>
497#include <vm/vm.h>
498#include <vm/vm_page.h>
499#include <vm/vm_object.h>
500#endif /*ZERO_COPY_SOCKETS*/
501
474int
475sosend(so, addr, uio, top, control, flags, td)
476 register struct socket *so;
477 struct sockaddr *addr;
478 struct uio *uio;
479 struct mbuf *top;
480 struct mbuf *control;
481 int flags;
482 struct thread *td;
483{
484 struct mbuf **mp;
485 register struct mbuf *m;
486 register long space, len, resid;
487 int clen = 0, error, s, dontroute, mlen;
488 int atomic = sosendallatonce(so) || top;
502int
503sosend(so, addr, uio, top, control, flags, td)
504 register struct socket *so;
505 struct sockaddr *addr;
506 struct uio *uio;
507 struct mbuf *top;
508 struct mbuf *control;
509 int flags;
510 struct thread *td;
511{
512 struct mbuf **mp;
513 register struct mbuf *m;
514 register long space, len, resid;
515 int clen = 0, error, s, dontroute, mlen;
516 int atomic = sosendallatonce(so) || top;
517#ifdef ZERO_COPY_SOCKETS
518 int cow_send;
519#endif /* ZERO_COPY_SOCKETS */
489
490 if (uio)
491 resid = uio->uio_resid;
492 else
493 resid = top->m_pkthdr.len;
494 /*
495 * In theory resid should be unsigned.
496 * However, space must be signed, as it might be less than 0

--- 72 unchanged lines hidden (view full) ---

569 if (uio == NULL) {
570 /*
571 * Data is prepackaged in "top".
572 */
573 resid = 0;
574 if (flags & MSG_EOR)
575 top->m_flags |= M_EOR;
576 } else do {
520
521 if (uio)
522 resid = uio->uio_resid;
523 else
524 resid = top->m_pkthdr.len;
525 /*
526 * In theory resid should be unsigned.
527 * However, space must be signed, as it might be less than 0

--- 72 unchanged lines hidden (view full) ---

600 if (uio == NULL) {
601 /*
602 * Data is prepackaged in "top".
603 */
604 resid = 0;
605 if (flags & MSG_EOR)
606 top->m_flags |= M_EOR;
607 } else do {
608#ifdef ZERO_COPY_SOCKETS
609 cow_send = 0;
610#endif /* ZERO_COPY_SOCKETS */
577 if (top == 0) {
578 MGETHDR(m, M_TRYWAIT, MT_DATA);
579 if (m == NULL) {
580 error = ENOBUFS;
581 goto release;
582 }
583 mlen = MHLEN;
584 m->m_pkthdr.len = 0;
585 m->m_pkthdr.rcvif = (struct ifnet *)0;
586 } else {
587 MGET(m, M_TRYWAIT, MT_DATA);
588 if (m == NULL) {
589 error = ENOBUFS;
590 goto release;
591 }
592 mlen = MLEN;
593 }
594 if (resid >= MINCLSIZE) {
611 if (top == 0) {
612 MGETHDR(m, M_TRYWAIT, MT_DATA);
613 if (m == NULL) {
614 error = ENOBUFS;
615 goto release;
616 }
617 mlen = MHLEN;
618 m->m_pkthdr.len = 0;
619 m->m_pkthdr.rcvif = (struct ifnet *)0;
620 } else {
621 MGET(m, M_TRYWAIT, MT_DATA);
622 if (m == NULL) {
623 error = ENOBUFS;
624 goto release;
625 }
626 mlen = MLEN;
627 }
628 if (resid >= MINCLSIZE) {
629#ifdef ZERO_COPY_SOCKETS
630 if (so_zero_copy_send &&
631 resid>=PAGE_SIZE &&
632 space>=PAGE_SIZE &&
633 uio->uio_iov->iov_len>=PAGE_SIZE) {
634 so_zerocp_stats.size_ok++;
635 if (!((vm_offset_t)
636 uio->uio_iov->iov_base & PAGE_MASK)){
637 so_zerocp_stats.align_ok++;
638 cow_send = socow_setup(m, uio);
639 }
640 }
641 if (!cow_send){
642#endif /* ZERO_COPY_SOCKETS */
595 MCLGET(m, M_TRYWAIT);
596 if ((m->m_flags & M_EXT) == 0)
597 goto nopages;
598 mlen = MCLBYTES;
599 len = min(min(mlen, resid), space);
600 } else {
643 MCLGET(m, M_TRYWAIT);
644 if ((m->m_flags & M_EXT) == 0)
645 goto nopages;
646 mlen = MCLBYTES;
647 len = min(min(mlen, resid), space);
648 } else {
649#ifdef ZERO_COPY_SOCKETS
650 len = PAGE_SIZE;
651 }
652
653 } else {
654#endif /* ZERO_COPY_SOCKETS */
601nopages:
602 len = min(min(mlen, resid), space);
603 /*
604 * For datagram protocols, leave room
605 * for protocol headers in first mbuf.
606 */
607 if (atomic && top == 0 && len < mlen)
608 MH_ALIGN(m, len);
609 }
610 space -= len;
655nopages:
656 len = min(min(mlen, resid), space);
657 /*
658 * For datagram protocols, leave room
659 * for protocol headers in first mbuf.
660 */
661 if (atomic && top == 0 && len < mlen)
662 MH_ALIGN(m, len);
663 }
664 space -= len;
665#ifdef ZERO_COPY_SOCKETS
666 if (cow_send)
667 error = 0;
668 else
669#endif /* ZERO_COPY_SOCKETS */
611 error = uiomove(mtod(m, caddr_t), (int)len, uio);
612 resid = uio->uio_resid;
613 m->m_len = len;
614 *mp = m;
615 top->m_pkthdr.len += len;
616 if (error)
617 goto release;
618 mp = &m->m_next;

--- 95 unchanged lines hidden (view full) ---

714 if (flags & MSG_OOB) {
715 m = m_get(M_TRYWAIT, MT_DATA);
716 if (m == NULL)
717 return (ENOBUFS);
718 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
719 if (error)
720 goto bad;
721 do {
670 error = uiomove(mtod(m, caddr_t), (int)len, uio);
671 resid = uio->uio_resid;
672 m->m_len = len;
673 *mp = m;
674 top->m_pkthdr.len += len;
675 if (error)
676 goto release;
677 mp = &m->m_next;

--- 95 unchanged lines hidden (view full) ---

773 if (flags & MSG_OOB) {
774 m = m_get(M_TRYWAIT, MT_DATA);
775 if (m == NULL)
776 return (ENOBUFS);
777 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
778 if (error)
779 goto bad;
780 do {
781#ifdef ZERO_COPY_SOCKETS
782 if (so_zero_copy_receive) {
783 vm_page_t pg;
784 int disposable;
785
786 if ((m->m_flags & M_EXT)
787 && (m->m_ext.ext_type == EXT_DISPOSABLE))
788 disposable = 1;
789 else
790 disposable = 0;
791
792 pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t)));
793 if (uio->uio_offset == -1)
794 uio->uio_offset =IDX_TO_OFF(pg->pindex);
795
796 error = uiomoveco(mtod(m, caddr_t),
797 min(uio->uio_resid, m->m_len),
798 uio, pg->object,
799 disposable);
800 } else
801#endif /* ZERO_COPY_SOCKETS */
722 error = uiomove(mtod(m, caddr_t),
723 (int) min(uio->uio_resid, m->m_len), uio);
724 m = m_free(m);
725 } while (uio->uio_resid && error == 0 && m);
726bad:
727 if (m)
728 m_freem(m);
729 return (error);

--- 139 unchanged lines hidden (view full) ---

869 * Otherwise copy them out via the uio, then free.
870 * Sockbuf must be consistent here (points to current mbuf,
871 * it points to next record) when we drop priority;
872 * we must note any additions to the sockbuf when we
873 * block interrupts again.
874 */
875 if (mp == 0) {
876 splx(s);
802 error = uiomove(mtod(m, caddr_t),
803 (int) min(uio->uio_resid, m->m_len), uio);
804 m = m_free(m);
805 } while (uio->uio_resid && error == 0 && m);
806bad:
807 if (m)
808 m_freem(m);
809 return (error);

--- 139 unchanged lines hidden (view full) ---

949 * Otherwise copy them out via the uio, then free.
950 * Sockbuf must be consistent here (points to current mbuf,
951 * it points to next record) when we drop priority;
952 * we must note any additions to the sockbuf when we
953 * block interrupts again.
954 */
955 if (mp == 0) {
956 splx(s);
957#ifdef ZERO_COPY_SOCKETS
958 if (so_zero_copy_receive) {
959 vm_page_t pg;
960 int disposable;
961
962 if ((m->m_flags & M_EXT)
963 && (m->m_ext.ext_type == EXT_DISPOSABLE))
964 disposable = 1;
965 else
966 disposable = 0;
967
968 pg = PHYS_TO_VM_PAGE(vtophys(mtod(m, caddr_t) +
969 moff));
970
971 if (uio->uio_offset == -1)
972 uio->uio_offset =IDX_TO_OFF(pg->pindex);
973
974 error = uiomoveco(mtod(m, caddr_t) + moff,
975 (int)len, uio,pg->object,
976 disposable);
977 } else
978#endif /* ZERO_COPY_SOCKETS */
877 error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio);
878 s = splnet();
879 if (error)
880 goto release;
881 } else
882 uio->uio_resid -= len;
883 if (len == m->m_len - moff) {
884 if (m->m_flags & M_EOR)

--- 806 unchanged lines hidden --- 		979 error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio);
980 s = splnet();
981 if (error)
982 goto release;
983 } else
984 uio->uio_resid -= len;
985 if (len == m->m_len - moff) {
986 if (m->m_flags & M_EOR)

--- 806 unchanged lines hidden ---