93 94#include "opt_inet.h" 95#include "opt_mac.h" 96#include "opt_zero.h" 97#include "opt_compat.h" 98 99#include <sys/param.h> 100#include <sys/systm.h> 101#include <sys/fcntl.h> 102#include <sys/limits.h> 103#include <sys/lock.h> 104#include <sys/mac.h> 105#include <sys/malloc.h> 106#include <sys/mbuf.h> 107#include <sys/mutex.h> 108#include <sys/domain.h> 109#include <sys/file.h> /* for struct knote */ 110#include <sys/kernel.h> 111#include <sys/event.h> 112#include <sys/poll.h> 113#include <sys/proc.h> 114#include <sys/protosw.h> 115#include <sys/socket.h> 116#include <sys/socketvar.h> 117#include <sys/resourcevar.h> 118#include <sys/signalvar.h> 119#include <sys/sysctl.h> 120#include <sys/uio.h> 121#include <sys/jail.h> 122 123#include <vm/uma.h> 124 125#ifdef COMPAT_IA32 126#include <sys/mount.h> 127#include <compat/freebsd32/freebsd32.h> 128 129extern struct sysentvec ia32_freebsd_sysvec; 130#endif 131 132static int soreceive_rcvoob(struct socket *so, struct uio *uio, 133 int flags); 134 135static void filt_sordetach(struct knote *kn); 136static int filt_soread(struct knote *kn, long hint); 137static void filt_sowdetach(struct knote *kn); 138static int filt_sowrite(struct knote *kn, long hint); 139static int filt_solisten(struct knote *kn, long hint); 140 141static struct filterops solisten_filtops = 142 { 1, NULL, filt_sordetach, filt_solisten }; 143static struct filterops soread_filtops = 144 { 1, NULL, filt_sordetach, filt_soread }; 145static struct filterops sowrite_filtops = 146 { 1, NULL, filt_sowdetach, filt_sowrite }; 147 148uma_zone_t socket_zone; 149so_gen_t so_gencnt; /* generation count for sockets */ 150 151MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 152MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 153 154SYSCTL_DECL(_kern_ipc); 155 156static int somaxconn = SOMAXCONN; 157static int somaxconn_sysctl(SYSCTL_HANDLER_ARGS); 158/* XXX: we dont have SYSCTL_USHORT */ 159SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLTYPE_UINT | CTLFLAG_RW, 160 0, sizeof(int), somaxconn_sysctl, "I", "Maximum pending socket connection " 161 "queue size"); 162static int numopensockets; 163SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 164 &numopensockets, 0, "Number of open sockets"); 165#ifdef ZERO_COPY_SOCKETS 166/* These aren't static because they're used in other files. */ 167int so_zero_copy_send = 1; 168int so_zero_copy_receive = 1; 169SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0, 170 "Zero copy controls"); 171SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW, 172 &so_zero_copy_receive, 0, "Enable zero copy receive"); 173SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW, 174 &so_zero_copy_send, 0, "Enable zero copy send"); 175#endif /* ZERO_COPY_SOCKETS */ 176 177/* 178 * accept_mtx locks down per-socket fields relating to accept queues. See 179 * socketvar.h for an annotation of the protected fields of struct socket. 180 */ 181struct mtx accept_mtx; 182MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 183 184/* 185 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 186 * so_gencnt field. 187 */ 188static struct mtx so_global_mtx; 189MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 190 191/* 192 * Socket operation routines. 193 * These routines are called by the routines in 194 * sys_socket.c or from a system process, and 195 * implement the semantics of socket operations by 196 * switching out to the protocol specific routines. 197 */ 198 199/* 200 * Get a socket structure from our zone, and initialize it. 201 * Note that it would probably be better to allocate socket 202 * and PCB at the same time, but I'm not convinced that all 203 * the protocols can be easily modified to do this. 204 * 205 * soalloc() returns a socket with a ref count of 0. 206 */ 207struct socket * 208soalloc(int mflags) 209{ 210 struct socket *so; 211 212 so = uma_zalloc(socket_zone, mflags | M_ZERO); 213 if (so != NULL) { 214#ifdef MAC 215 if (mac_init_socket(so, mflags) != 0) { 216 uma_zfree(socket_zone, so); 217 return (NULL); 218 } 219#endif 220 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 221 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 222 TAILQ_INIT(&so->so_aiojobq); 223 mtx_lock(&so_global_mtx); 224 so->so_gencnt = ++so_gencnt; 225 ++numopensockets; 226 mtx_unlock(&so_global_mtx); 227 } 228 return (so); 229} 230 231/* 232 * socreate returns a socket with a ref count of 1. The socket should be 233 * closed with soclose(). 234 */ 235int 236socreate(dom, aso, type, proto, cred, td) 237 int dom; 238 struct socket **aso; 239 int type; 240 int proto; 241 struct ucred *cred; 242 struct thread *td; 243{ 244 struct protosw *prp; 245 struct socket *so; 246 int error; 247 248 if (proto) 249 prp = pffindproto(dom, proto, type); 250 else 251 prp = pffindtype(dom, type); 252 253 if (prp == NULL || prp->pr_usrreqs->pru_attach == NULL || 254 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 255 return (EPROTONOSUPPORT); 256 257 if (jailed(cred) && jail_socket_unixiproute_only && 258 prp->pr_domain->dom_family != PF_LOCAL && 259 prp->pr_domain->dom_family != PF_INET && 260 prp->pr_domain->dom_family != PF_ROUTE) { 261 return (EPROTONOSUPPORT); 262 } 263 264 if (prp->pr_type != type) 265 return (EPROTOTYPE); 266 so = soalloc(M_WAITOK); 267 if (so == NULL) 268 return (ENOBUFS); 269 270 TAILQ_INIT(&so->so_incomp); 271 TAILQ_INIT(&so->so_comp); 272 so->so_type = type; 273 so->so_cred = crhold(cred); 274 so->so_proto = prp; 275#ifdef MAC 276 mac_create_socket(cred, so); 277#endif 278 knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv), 279 NULL, NULL, NULL); 280 knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd), 281 NULL, NULL, NULL); 282 so->so_count = 1; 283 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 284 if (error) { 285 ACCEPT_LOCK(); 286 SOCK_LOCK(so); 287 so->so_state |= SS_NOFDREF; 288 sorele(so); 289 return (error); 290 } 291 *aso = so; 292 return (0); 293} 294 295int 296sobind(so, nam, td) 297 struct socket *so; 298 struct sockaddr *nam; 299 struct thread *td; 300{ 301 302 return ((*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td)); 303} 304 305void 306sodealloc(struct socket *so) 307{ 308 309 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 310 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 311 312 mtx_lock(&so_global_mtx); 313 so->so_gencnt = ++so_gencnt; 314 mtx_unlock(&so_global_mtx); 315 if (so->so_rcv.sb_hiwat) 316 (void)chgsbsize(so->so_cred->cr_uidinfo, 317 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 318 if (so->so_snd.sb_hiwat) 319 (void)chgsbsize(so->so_cred->cr_uidinfo, 320 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 321#ifdef INET 322 /* remove acccept filter if one is present. */ 323 if (so->so_accf != NULL) 324 do_setopt_accept_filter(so, NULL); 325#endif 326#ifdef MAC 327 mac_destroy_socket(so); 328#endif 329 crfree(so->so_cred); 330 SOCKBUF_LOCK_DESTROY(&so->so_snd); 331 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 332 uma_zfree(socket_zone, so); 333 mtx_lock(&so_global_mtx); 334 --numopensockets; 335 mtx_unlock(&so_global_mtx); 336} 337 338/* 339 * solisten() transitions a socket from a non-listening state to a listening 340 * state, but can also be used to update the listen queue depth on an 341 * existing listen socket. The protocol will call back into the sockets 342 * layer using solisten_proto_check() and solisten_proto() to check and set 343 * socket-layer listen state. Call backs are used so that the protocol can 344 * acquire both protocol and socket layer locks in whatever order is required 345 * by the protocol. 346 * 347 * Protocol implementors are advised to hold the socket lock across the 348 * socket-layer test and set to avoid races at the socket layer. 349 */ 350int 351solisten(so, backlog, td) 352 struct socket *so; 353 int backlog; 354 struct thread *td; 355{ 356 357 return ((*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td)); 358} 359 360int 361solisten_proto_check(so) 362 struct socket *so; 363{ 364 365 SOCK_LOCK_ASSERT(so); 366 367 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 368 SS_ISDISCONNECTING)) 369 return (EINVAL); 370 return (0); 371} 372 373void 374solisten_proto(so, backlog) 375 struct socket *so; 376 int backlog; 377{ 378 379 SOCK_LOCK_ASSERT(so); 380 381 if (backlog < 0 || backlog > somaxconn) 382 backlog = somaxconn; 383 so->so_qlimit = backlog; 384 so->so_options |= SO_ACCEPTCONN; 385} 386 387/* 388 * Attempt to free a socket. This should really be sotryfree(). 389 * 390 * We free the socket if the protocol is no longer interested in the socket, 391 * there's no file descriptor reference, and the refcount is 0. While the 392 * calling macro sotryfree() tests the refcount, sofree() has to test it 393 * again as it's possible to race with an accept()ing thread if the socket is 394 * in an listen queue of a listen socket, as being in the listen queue 395 * doesn't elevate the reference count. sofree() acquires the accept mutex 396 * early for this test in order to avoid that race. 397 */ 398void 399sofree(so) 400 struct socket *so; 401{ 402 struct socket *head; 403 404 ACCEPT_LOCK_ASSERT(); 405 SOCK_LOCK_ASSERT(so); 406 407 if (so->so_pcb != NULL || (so->so_state & SS_NOFDREF) == 0 || 408 so->so_count != 0 || (so->so_state & SS_PROTOREF)) { 409 SOCK_UNLOCK(so); 410 ACCEPT_UNLOCK(); 411 return; 412 } 413 414 head = so->so_head; 415 if (head != NULL) { 416 KASSERT((so->so_qstate & SQ_COMP) != 0 || 417 (so->so_qstate & SQ_INCOMP) != 0, 418 ("sofree: so_head != NULL, but neither SQ_COMP nor " 419 "SQ_INCOMP")); 420 KASSERT((so->so_qstate & SQ_COMP) == 0 || 421 (so->so_qstate & SQ_INCOMP) == 0, 422 ("sofree: so->so_qstate is SQ_COMP and also SQ_INCOMP")); 423 /* 424 * accept(2) is responsible draining the completed 425 * connection queue and freeing those sockets, so 426 * we just return here if this socket is currently 427 * on the completed connection queue. Otherwise, 428 * accept(2) may hang after select(2) has indicating 429 * that a listening socket was ready. If it's an 430 * incomplete connection, we remove it from the queue 431 * and free it; otherwise, it won't be released until 432 * the listening socket is closed. 433 */ 434 if ((so->so_qstate & SQ_COMP) != 0) { 435 SOCK_UNLOCK(so); 436 ACCEPT_UNLOCK(); 437 return; 438 } 439 TAILQ_REMOVE(&head->so_incomp, so, so_list); 440 head->so_incqlen--; 441 so->so_qstate &= ~SQ_INCOMP; 442 so->so_head = NULL; 443 } 444 KASSERT((so->so_qstate & SQ_COMP) == 0 && 445 (so->so_qstate & SQ_INCOMP) == 0, 446 ("sofree: so_head == NULL, but still SQ_COMP(%d) or SQ_INCOMP(%d)", 447 so->so_qstate & SQ_COMP, so->so_qstate & SQ_INCOMP)); 448 SOCK_UNLOCK(so); 449 ACCEPT_UNLOCK(); 450 SOCKBUF_LOCK(&so->so_snd); 451 so->so_snd.sb_flags |= SB_NOINTR; 452 (void)sblock(&so->so_snd, M_WAITOK); 453 /* 454 * socantsendmore_locked() drops the socket buffer mutex so that it 455 * can safely perform wakeups. Re-acquire the mutex before 456 * continuing. 457 */ 458 socantsendmore_locked(so); 459 SOCKBUF_LOCK(&so->so_snd); 460 sbunlock(&so->so_snd); 461 sbrelease_locked(&so->so_snd, so); 462 SOCKBUF_UNLOCK(&so->so_snd); 463 sorflush(so); 464 knlist_destroy(&so->so_rcv.sb_sel.si_note); 465 knlist_destroy(&so->so_snd.sb_sel.si_note); 466 sodealloc(so); 467} 468 469/* 470 * Close a socket on last file table reference removal. 471 * Initiate disconnect if connected. 472 * Free socket when disconnect complete. 473 * 474 * This function will sorele() the socket. Note that soclose() may be 475 * called prior to the ref count reaching zero. The actual socket 476 * structure will not be freed until the ref count reaches zero. 477 */ 478int 479soclose(so) 480 struct socket *so; 481{ 482 int error = 0; 483 484 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 485 486 funsetown(&so->so_sigio); 487 if (so->so_options & SO_ACCEPTCONN) { 488 struct socket *sp; 489 ACCEPT_LOCK(); 490 while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { 491 TAILQ_REMOVE(&so->so_incomp, sp, so_list); 492 so->so_incqlen--; 493 sp->so_qstate &= ~SQ_INCOMP; 494 sp->so_head = NULL; 495 ACCEPT_UNLOCK(); 496 soabort(sp); 497 ACCEPT_LOCK(); 498 } 499 while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { 500 TAILQ_REMOVE(&so->so_comp, sp, so_list); 501 so->so_qlen--; 502 sp->so_qstate &= ~SQ_COMP; 503 sp->so_head = NULL; 504 ACCEPT_UNLOCK(); 505 soabort(sp); 506 ACCEPT_LOCK(); 507 } 508 ACCEPT_UNLOCK(); 509 } 510 if (so->so_pcb == NULL) 511 goto discard; 512 if (so->so_state & SS_ISCONNECTED) { 513 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 514 error = sodisconnect(so); 515 if (error) 516 goto drop; 517 } 518 if (so->so_options & SO_LINGER) { 519 if ((so->so_state & SS_ISDISCONNECTING) && 520 (so->so_state & SS_NBIO)) 521 goto drop; 522 while (so->so_state & SS_ISCONNECTED) { 523 error = tsleep(&so->so_timeo, 524 PSOCK | PCATCH, "soclos", so->so_linger * hz); 525 if (error) 526 break; 527 } 528 } 529 } 530drop: 531 if (so->so_pcb != NULL) { 532 int error2 = (*so->so_proto->pr_usrreqs->pru_detach)(so); 533 if (error == 0) 534 error = error2; 535 } 536discard: 537 ACCEPT_LOCK(); 538 SOCK_LOCK(so); 539 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 540 so->so_state |= SS_NOFDREF; 541 sorele(so); 542 return (error); 543} 544 545/*
|
581} 582 583int 584soaccept(so, nam) 585 struct socket *so; 586 struct sockaddr **nam; 587{ 588 int error; 589 590 SOCK_LOCK(so); 591 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 592 so->so_state &= ~SS_NOFDREF; 593 SOCK_UNLOCK(so); 594 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 595 return (error); 596} 597 598int 599soconnect(so, nam, td) 600 struct socket *so; 601 struct sockaddr *nam; 602 struct thread *td; 603{ 604 int error; 605 606 if (so->so_options & SO_ACCEPTCONN) 607 return (EOPNOTSUPP); 608 /* 609 * If protocol is connection-based, can only connect once. 610 * Otherwise, if connected, try to disconnect first. 611 * This allows user to disconnect by connecting to, e.g., 612 * a null address. 613 */ 614 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 615 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 616 (error = sodisconnect(so)))) { 617 error = EISCONN; 618 } else { 619 /* 620 * Prevent accumulated error from previous connection 621 * from biting us. 622 */ 623 so->so_error = 0; 624 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td); 625 } 626 627 return (error); 628} 629 630int 631soconnect2(so1, so2) 632 struct socket *so1; 633 struct socket *so2; 634{ 635 636 return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2)); 637} 638 639int 640sodisconnect(so) 641 struct socket *so; 642{ 643 int error; 644 645 if ((so->so_state & SS_ISCONNECTED) == 0) 646 return (ENOTCONN); 647 if (so->so_state & SS_ISDISCONNECTING) 648 return (EALREADY); 649 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 650 return (error); 651} 652 653#ifdef ZERO_COPY_SOCKETS 654struct so_zerocopy_stats{ 655 int size_ok; 656 int align_ok; 657 int found_ifp; 658}; 659struct so_zerocopy_stats so_zerocp_stats = {0,0,0}; 660#include <netinet/in.h> 661#include <net/route.h> 662#include <netinet/in_pcb.h> 663#include <vm/vm.h> 664#include <vm/vm_page.h> 665#include <vm/vm_object.h> 666#endif /*ZERO_COPY_SOCKETS*/ 667 668/* 669 * sosend_copyin() accepts a uio and prepares an mbuf chain holding part or 670 * all of the data referenced by the uio. If desired, it uses zero-copy. 671 * *space will be updated to reflect data copied in. 672 * 673 * NB: If atomic I/O is requested, the caller must already have checked that 674 * space can hold resid bytes. 675 * 676 * NB: In the event of an error, the caller may need to free the partial 677 * chain pointed to by *mpp. The contents of both *uio and *space may be 678 * modified even in the case of an error. 679 */ 680static int 681sosend_copyin(struct uio *uio, struct mbuf **retmp, int atomic, long *space, 682 int flags) 683{ 684 struct mbuf *m, **mp, *top; 685 long len, resid; 686 int error; 687#ifdef ZERO_COPY_SOCKETS 688 int cow_send; 689#endif 690 691 *retmp = top = NULL; 692 mp = ⊤ 693 len = 0; 694 resid = uio->uio_resid; 695 error = 0; 696 do { 697#ifdef ZERO_COPY_SOCKETS 698 cow_send = 0; 699#endif /* ZERO_COPY_SOCKETS */ 700 if (resid >= MINCLSIZE) { 701#ifdef ZERO_COPY_SOCKETS 702 if (top == NULL) { 703 MGETHDR(m, M_TRYWAIT, MT_DATA); 704 if (m == NULL) { 705 error = ENOBUFS; 706 goto out; 707 } 708 m->m_pkthdr.len = 0; 709 m->m_pkthdr.rcvif = NULL; 710 } else { 711 MGET(m, M_TRYWAIT, MT_DATA); 712 if (m == NULL) { 713 error = ENOBUFS; 714 goto out; 715 } 716 } 717 if (so_zero_copy_send && 718 resid>=PAGE_SIZE && 719 *space>=PAGE_SIZE && 720 uio->uio_iov->iov_len>=PAGE_SIZE) { 721 so_zerocp_stats.size_ok++; 722 so_zerocp_stats.align_ok++; 723 cow_send = socow_setup(m, uio); 724 len = cow_send; 725 } 726 if (!cow_send) { 727 MCLGET(m, M_TRYWAIT); 728 if ((m->m_flags & M_EXT) == 0) { 729 m_free(m); 730 m = NULL; 731 } else { 732 len = min(min(MCLBYTES, resid), 733 *space); 734 } 735 } 736#else /* ZERO_COPY_SOCKETS */ 737 if (top == NULL) { 738 m = m_getcl(M_TRYWAIT, MT_DATA, M_PKTHDR); 739 m->m_pkthdr.len = 0; 740 m->m_pkthdr.rcvif = NULL; 741 } else 742 m = m_getcl(M_TRYWAIT, MT_DATA, 0); 743 len = min(min(MCLBYTES, resid), *space); 744#endif /* ZERO_COPY_SOCKETS */ 745 } else { 746 if (top == NULL) { 747 m = m_gethdr(M_TRYWAIT, MT_DATA); 748 m->m_pkthdr.len = 0; 749 m->m_pkthdr.rcvif = NULL; 750 751 len = min(min(MHLEN, resid), *space); 752 /* 753 * For datagram protocols, leave room 754 * for protocol headers in first mbuf. 755 */ 756 if (atomic && m && len < MHLEN) 757 MH_ALIGN(m, len); 758 } else { 759 m = m_get(M_TRYWAIT, MT_DATA); 760 len = min(min(MLEN, resid), *space); 761 } 762 } 763 if (m == NULL) { 764 error = ENOBUFS; 765 goto out; 766 } 767 768 *space -= len; 769#ifdef ZERO_COPY_SOCKETS 770 if (cow_send) 771 error = 0; 772 else 773#endif /* ZERO_COPY_SOCKETS */ 774 error = uiomove(mtod(m, void *), (int)len, uio); 775 resid = uio->uio_resid; 776 m->m_len = len; 777 *mp = m; 778 top->m_pkthdr.len += len; 779 if (error) 780 goto out; 781 mp = &m->m_next; 782 if (resid <= 0) { 783 if (flags & MSG_EOR) 784 top->m_flags |= M_EOR; 785 break; 786 } 787 } while (*space > 0 && atomic); 788out: 789 *retmp = top; 790 return (error); 791} 792 793#define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK) 794 795int 796sosend_dgram(so, addr, uio, top, control, flags, td) 797 struct socket *so; 798 struct sockaddr *addr; 799 struct uio *uio; 800 struct mbuf *top; 801 struct mbuf *control; 802 int flags; 803 struct thread *td; 804{ 805 long space, resid; 806 int clen = 0, error, dontroute; 807 int atomic = sosendallatonce(so) || top; 808 809 KASSERT(so->so_type == SOCK_DGRAM, ("sodgram_send: !SOCK_DGRAM")); 810 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 811 ("sodgram_send: !PR_ATOMIC")); 812 813 if (uio != NULL) 814 resid = uio->uio_resid; 815 else 816 resid = top->m_pkthdr.len; 817 /* 818 * In theory resid should be unsigned. 819 * However, space must be signed, as it might be less than 0 820 * if we over-committed, and we must use a signed comparison 821 * of space and resid. On the other hand, a negative resid 822 * causes us to loop sending 0-length segments to the protocol. 823 * 824 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 825 * type sockets since that's an error. 826 */ 827 if (resid < 0) { 828 error = EINVAL; 829 goto out; 830 } 831 832 dontroute = 833 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 834 if (td != NULL) 835 td->td_proc->p_stats->p_ru.ru_msgsnd++; 836 if (control != NULL) 837 clen = control->m_len; 838 839 SOCKBUF_LOCK(&so->so_snd); 840 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 841 SOCKBUF_UNLOCK(&so->so_snd); 842 error = EPIPE; 843 goto out; 844 } 845 if (so->so_error) { 846 error = so->so_error; 847 so->so_error = 0; 848 SOCKBUF_UNLOCK(&so->so_snd); 849 goto out; 850 } 851 if ((so->so_state & SS_ISCONNECTED) == 0) { 852 /* 853 * `sendto' and `sendmsg' is allowed on a connection- 854 * based socket if it supports implied connect. 855 * Return ENOTCONN if not connected and no address is 856 * supplied. 857 */ 858 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 859 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 860 if ((so->so_state & SS_ISCONFIRMING) == 0 && 861 !(resid == 0 && clen != 0)) { 862 SOCKBUF_UNLOCK(&so->so_snd); 863 error = ENOTCONN; 864 goto out; 865 } 866 } else if (addr == NULL) { 867 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 868 error = ENOTCONN; 869 else 870 error = EDESTADDRREQ; 871 SOCKBUF_UNLOCK(&so->so_snd); 872 goto out; 873 } 874 } 875 876 /* 877 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 878 * problem and need fixing. 879 */ 880 space = sbspace(&so->so_snd); 881 if (flags & MSG_OOB) 882 space += 1024; 883 space -= clen; 884 if (resid > space) { 885 error = EMSGSIZE; 886 goto out; 887 } 888 SOCKBUF_UNLOCK(&so->so_snd); 889 if (uio == NULL) { 890 resid = 0; 891 if (flags & MSG_EOR) 892 top->m_flags |= M_EOR; 893 } else { 894 error = sosend_copyin(uio, &top, atomic, &space, flags); 895 if (error) 896 goto out; 897 resid = uio->uio_resid; 898 } 899 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 900 /* 901 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 902 * than with. 903 */ 904 if (dontroute) { 905 SOCK_LOCK(so); 906 so->so_options |= SO_DONTROUTE; 907 SOCK_UNLOCK(so); 908 } 909 /* 910 * XXX all the SBS_CANTSENDMORE checks previously 911 * done could be out of date. We could have recieved 912 * a reset packet in an interrupt or maybe we slept 913 * while doing page faults in uiomove() etc. We could 914 * probably recheck again inside the locking protection 915 * here, but there are probably other places that this 916 * also happens. We must rethink this. 917 */ 918 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 919 (flags & MSG_OOB) ? PRUS_OOB : 920 /* 921 * If the user set MSG_EOF, the protocol 922 * understands this flag and nothing left to 923 * send then use PRU_SEND_EOF instead of PRU_SEND. 924 */ 925 ((flags & MSG_EOF) && 926 (so->so_proto->pr_flags & PR_IMPLOPCL) && 927 (resid <= 0)) ? 928 PRUS_EOF : 929 /* If there is more to send set PRUS_MORETOCOME */ 930 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 931 top, addr, control, td); 932 if (dontroute) { 933 SOCK_LOCK(so); 934 so->so_options &= ~SO_DONTROUTE; 935 SOCK_UNLOCK(so); 936 } 937 clen = 0; 938 control = NULL; 939 top = NULL; 940out: 941 if (top != NULL) 942 m_freem(top); 943 if (control != NULL) 944 m_freem(control); 945 return (error); 946} 947 948/* 949 * Send on a socket. 950 * If send must go all at once and message is larger than 951 * send buffering, then hard error. 952 * Lock against other senders. 953 * If must go all at once and not enough room now, then 954 * inform user that this would block and do nothing. 955 * Otherwise, if nonblocking, send as much as possible. 956 * The data to be sent is described by "uio" if nonzero, 957 * otherwise by the mbuf chain "top" (which must be null 958 * if uio is not). Data provided in mbuf chain must be small 959 * enough to send all at once. 960 * 961 * Returns nonzero on error, timeout or signal; callers 962 * must check for short counts if EINTR/ERESTART are returned. 963 * Data and control buffers are freed on return. 964 */ 965#define snderr(errno) { error = (errno); goto release; } 966int 967sosend(so, addr, uio, top, control, flags, td) 968 struct socket *so; 969 struct sockaddr *addr; 970 struct uio *uio; 971 struct mbuf *top; 972 struct mbuf *control; 973 int flags; 974 struct thread *td; 975{ 976 long space, resid; 977 int clen = 0, error, dontroute; 978 int atomic = sosendallatonce(so) || top; 979 980 if (uio != NULL) 981 resid = uio->uio_resid; 982 else 983 resid = top->m_pkthdr.len; 984 /* 985 * In theory resid should be unsigned. 986 * However, space must be signed, as it might be less than 0 987 * if we over-committed, and we must use a signed comparison 988 * of space and resid. On the other hand, a negative resid 989 * causes us to loop sending 0-length segments to the protocol. 990 * 991 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 992 * type sockets since that's an error. 993 */ 994 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 995 error = EINVAL; 996 goto out; 997 } 998 999 dontroute = 1000 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1001 (so->so_proto->pr_flags & PR_ATOMIC); 1002 if (td != NULL) 1003 td->td_proc->p_stats->p_ru.ru_msgsnd++; 1004 if (control != NULL) 1005 clen = control->m_len; 1006 1007 SOCKBUF_LOCK(&so->so_snd); 1008restart: 1009 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1010 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1011 if (error) 1012 goto out_locked; 1013 do { 1014 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1015 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 1016 snderr(EPIPE); 1017 if (so->so_error) { 1018 error = so->so_error; 1019 so->so_error = 0; 1020 goto release; 1021 } 1022 if ((so->so_state & SS_ISCONNECTED) == 0) { 1023 /* 1024 * `sendto' and `sendmsg' is allowed on a connection- 1025 * based socket if it supports implied connect. 1026 * Return ENOTCONN if not connected and no address is 1027 * supplied. 1028 */ 1029 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1030 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1031 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1032 !(resid == 0 && clen != 0)) 1033 snderr(ENOTCONN); 1034 } else if (addr == NULL) 1035 snderr(so->so_proto->pr_flags & PR_CONNREQUIRED ? 1036 ENOTCONN : EDESTADDRREQ); 1037 } 1038 space = sbspace(&so->so_snd); 1039 if (flags & MSG_OOB) 1040 space += 1024; 1041 if ((atomic && resid > so->so_snd.sb_hiwat) || 1042 clen > so->so_snd.sb_hiwat) 1043 snderr(EMSGSIZE); 1044 if (space < resid + clen && 1045 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1046 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) 1047 snderr(EWOULDBLOCK); 1048 sbunlock(&so->so_snd); 1049 error = sbwait(&so->so_snd); 1050 if (error) 1051 goto out_locked; 1052 goto restart; 1053 } 1054 SOCKBUF_UNLOCK(&so->so_snd); 1055 space -= clen; 1056 do { 1057 if (uio == NULL) { 1058 resid = 0; 1059 if (flags & MSG_EOR) 1060 top->m_flags |= M_EOR; 1061 } else { 1062 error = sosend_copyin(uio, &top, atomic, 1063 &space, flags); 1064 if (error != 0) { 1065 SOCKBUF_LOCK(&so->so_snd); 1066 goto release; 1067 } 1068 resid = uio->uio_resid; 1069 } 1070 if (dontroute) { 1071 SOCK_LOCK(so); 1072 so->so_options |= SO_DONTROUTE; 1073 SOCK_UNLOCK(so); 1074 } 1075 /* 1076 * XXX all the SBS_CANTSENDMORE checks previously 1077 * done could be out of date. We could have recieved 1078 * a reset packet in an interrupt or maybe we slept 1079 * while doing page faults in uiomove() etc. We could 1080 * probably recheck again inside the locking protection 1081 * here, but there are probably other places that this 1082 * also happens. We must rethink this. 1083 */ 1084 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1085 (flags & MSG_OOB) ? PRUS_OOB : 1086 /* 1087 * If the user set MSG_EOF, the protocol 1088 * understands this flag and nothing left to 1089 * send then use PRU_SEND_EOF instead of PRU_SEND. 1090 */ 1091 ((flags & MSG_EOF) && 1092 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1093 (resid <= 0)) ? 1094 PRUS_EOF : 1095 /* If there is more to send set PRUS_MORETOCOME */ 1096 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1097 top, addr, control, td); 1098 if (dontroute) { 1099 SOCK_LOCK(so); 1100 so->so_options &= ~SO_DONTROUTE; 1101 SOCK_UNLOCK(so); 1102 } 1103 clen = 0; 1104 control = NULL; 1105 top = NULL; 1106 if (error) { 1107 SOCKBUF_LOCK(&so->so_snd); 1108 goto release; 1109 } 1110 } while (resid && space > 0); 1111 SOCKBUF_LOCK(&so->so_snd); 1112 } while (resid); 1113 1114release: 1115 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1116 sbunlock(&so->so_snd); 1117out_locked: 1118 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1119 SOCKBUF_UNLOCK(&so->so_snd); 1120out: 1121 if (top != NULL) 1122 m_freem(top); 1123 if (control != NULL) 1124 m_freem(control); 1125 return (error); 1126} 1127#undef snderr 1128 1129/* 1130 * The part of soreceive() that implements reading non-inline out-of-band 1131 * data from a socket. For more complete comments, see soreceive(), from 1132 * which this code originated. 1133 * 1134 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1135 * unable to return an mbuf chain to the caller. 1136 */ 1137static int 1138soreceive_rcvoob(so, uio, flags) 1139 struct socket *so; 1140 struct uio *uio; 1141 int flags; 1142{ 1143 struct protosw *pr = so->so_proto; 1144 struct mbuf *m; 1145 int error; 1146 1147 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1148 1149 m = m_get(M_TRYWAIT, MT_DATA); 1150 if (m == NULL) 1151 return (ENOBUFS); 1152 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1153 if (error) 1154 goto bad; 1155 do { 1156#ifdef ZERO_COPY_SOCKETS 1157 if (so_zero_copy_receive) { 1158 int disposable; 1159 1160 if ((m->m_flags & M_EXT) 1161 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1162 disposable = 1; 1163 else 1164 disposable = 0; 1165 1166 error = uiomoveco(mtod(m, void *), 1167 min(uio->uio_resid, m->m_len), 1168 uio, disposable); 1169 } else 1170#endif /* ZERO_COPY_SOCKETS */ 1171 error = uiomove(mtod(m, void *), 1172 (int) min(uio->uio_resid, m->m_len), uio); 1173 m = m_free(m); 1174 } while (uio->uio_resid && error == 0 && m); 1175bad: 1176 if (m != NULL) 1177 m_freem(m); 1178 return (error); 1179} 1180 1181/* 1182 * Following replacement or removal of the first mbuf on the first mbuf chain 1183 * of a socket buffer, push necessary state changes back into the socket 1184 * buffer so that other consumers see the values consistently. 'nextrecord' 1185 * is the callers locally stored value of the original value of 1186 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1187 * NOTE: 'nextrecord' may be NULL. 1188 */ 1189static __inline void 1190sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1191{ 1192 1193 SOCKBUF_LOCK_ASSERT(sb); 1194 /* 1195 * First, update for the new value of nextrecord. If necessary, make 1196 * it the first record. 1197 */ 1198 if (sb->sb_mb != NULL) 1199 sb->sb_mb->m_nextpkt = nextrecord; 1200 else 1201 sb->sb_mb = nextrecord; 1202 1203 /* 1204 * Now update any dependent socket buffer fields to reflect the new 1205 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1206 * addition of a second clause that takes care of the case where 1207 * sb_mb has been updated, but remains the last record. 1208 */ 1209 if (sb->sb_mb == NULL) { 1210 sb->sb_mbtail = NULL; 1211 sb->sb_lastrecord = NULL; 1212 } else if (sb->sb_mb->m_nextpkt == NULL) 1213 sb->sb_lastrecord = sb->sb_mb; 1214} 1215 1216 1217/* 1218 * Implement receive operations on a socket. 1219 * We depend on the way that records are added to the sockbuf 1220 * by sbappend*. In particular, each record (mbufs linked through m_next) 1221 * must begin with an address if the protocol so specifies, 1222 * followed by an optional mbuf or mbufs containing ancillary data, 1223 * and then zero or more mbufs of data. 1224 * In order to avoid blocking network interrupts for the entire time here, 1225 * we splx() while doing the actual copy to user space. 1226 * Although the sockbuf is locked, new data may still be appended, 1227 * and thus we must maintain consistency of the sockbuf during that time. 1228 * 1229 * The caller may receive the data as a single mbuf chain by supplying 1230 * an mbuf **mp0 for use in returning the chain. The uio is then used 1231 * only for the count in uio_resid. 1232 */ 1233int 1234soreceive(so, psa, uio, mp0, controlp, flagsp) 1235 struct socket *so; 1236 struct sockaddr **psa; 1237 struct uio *uio; 1238 struct mbuf **mp0; 1239 struct mbuf **controlp; 1240 int *flagsp; 1241{ 1242 struct mbuf *m, **mp; 1243 int flags, len, error, offset; 1244 struct protosw *pr = so->so_proto; 1245 struct mbuf *nextrecord; 1246 int moff, type = 0; 1247 int orig_resid = uio->uio_resid; 1248 1249 mp = mp0; 1250 if (psa != NULL) 1251 *psa = NULL; 1252 if (controlp != NULL) 1253 *controlp = NULL; 1254 if (flagsp != NULL) 1255 flags = *flagsp &~ MSG_EOR; 1256 else 1257 flags = 0; 1258 if (flags & MSG_OOB) 1259 return (soreceive_rcvoob(so, uio, flags)); 1260 if (mp != NULL) 1261 *mp = NULL; 1262 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1263 && uio->uio_resid) 1264 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1265 1266 SOCKBUF_LOCK(&so->so_rcv); 1267restart: 1268 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1269 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1270 if (error) 1271 goto out; 1272 1273 m = so->so_rcv.sb_mb; 1274 /* 1275 * If we have less data than requested, block awaiting more 1276 * (subject to any timeout) if: 1277 * 1. the current count is less than the low water mark, or 1278 * 2. MSG_WAITALL is set, and it is possible to do the entire 1279 * receive operation at once if we block (resid <= hiwat). 1280 * 3. MSG_DONTWAIT is not set 1281 * If MSG_WAITALL is set but resid is larger than the receive buffer, 1282 * we have to do the receive in sections, and thus risk returning 1283 * a short count if a timeout or signal occurs after we start. 1284 */ 1285 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1286 so->so_rcv.sb_cc < uio->uio_resid) && 1287 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat || 1288 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) && 1289 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1290 KASSERT(m != NULL || !so->so_rcv.sb_cc, 1291 ("receive: m == %p so->so_rcv.sb_cc == %u", 1292 m, so->so_rcv.sb_cc)); 1293 if (so->so_error) { 1294 if (m != NULL) 1295 goto dontblock; 1296 error = so->so_error; 1297 if ((flags & MSG_PEEK) == 0) 1298 so->so_error = 0; 1299 goto release; 1300 } 1301 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1302 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1303 if (m) 1304 goto dontblock; 1305 else 1306 goto release; 1307 } 1308 for (; m != NULL; m = m->m_next) 1309 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1310 m = so->so_rcv.sb_mb; 1311 goto dontblock; 1312 } 1313 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1314 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1315 error = ENOTCONN; 1316 goto release; 1317 } 1318 if (uio->uio_resid == 0) 1319 goto release; 1320 if ((so->so_state & SS_NBIO) || 1321 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1322 error = EWOULDBLOCK; 1323 goto release; 1324 } 1325 SBLASTRECORDCHK(&so->so_rcv); 1326 SBLASTMBUFCHK(&so->so_rcv); 1327 sbunlock(&so->so_rcv); 1328 error = sbwait(&so->so_rcv); 1329 if (error) 1330 goto out; 1331 goto restart; 1332 } 1333dontblock: 1334 /* 1335 * From this point onward, we maintain 'nextrecord' as a cache of the 1336 * pointer to the next record in the socket buffer. We must keep the 1337 * various socket buffer pointers and local stack versions of the 1338 * pointers in sync, pushing out modifications before dropping the 1339 * socket buffer mutex, and re-reading them when picking it up. 1340 * 1341 * Otherwise, we will race with the network stack appending new data 1342 * or records onto the socket buffer by using inconsistent/stale 1343 * versions of the field, possibly resulting in socket buffer 1344 * corruption. 1345 * 1346 * By holding the high-level sblock(), we prevent simultaneous 1347 * readers from pulling off the front of the socket buffer. 1348 */ 1349 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1350 if (uio->uio_td) 1351 uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++; 1352 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1353 SBLASTRECORDCHK(&so->so_rcv); 1354 SBLASTMBUFCHK(&so->so_rcv); 1355 nextrecord = m->m_nextpkt; 1356 if (pr->pr_flags & PR_ADDR) { 1357 KASSERT(m->m_type == MT_SONAME, 1358 ("m->m_type == %d", m->m_type)); 1359 orig_resid = 0; 1360 if (psa != NULL) 1361 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1362 M_NOWAIT); 1363 if (flags & MSG_PEEK) { 1364 m = m->m_next; 1365 } else { 1366 sbfree(&so->so_rcv, m); 1367 so->so_rcv.sb_mb = m_free(m); 1368 m = so->so_rcv.sb_mb; 1369 sockbuf_pushsync(&so->so_rcv, nextrecord); 1370 } 1371 } 1372 1373 /* 1374 * Process one or more MT_CONTROL mbufs present before any data mbufs 1375 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1376 * just copy the data; if !MSG_PEEK, we call into the protocol to 1377 * perform externalization (or freeing if controlp == NULL). 1378 */ 1379 if (m != NULL && m->m_type == MT_CONTROL) { 1380 struct mbuf *cm = NULL, *cmn; 1381 struct mbuf **cme = &cm; 1382 1383 do { 1384 if (flags & MSG_PEEK) { 1385 if (controlp != NULL) { 1386 *controlp = m_copy(m, 0, m->m_len); 1387 controlp = &(*controlp)->m_next; 1388 } 1389 m = m->m_next; 1390 } else { 1391 sbfree(&so->so_rcv, m); 1392 so->so_rcv.sb_mb = m->m_next; 1393 m->m_next = NULL; 1394 *cme = m; 1395 cme = &(*cme)->m_next; 1396 m = so->so_rcv.sb_mb; 1397 } 1398 } while (m != NULL && m->m_type == MT_CONTROL); 1399 if ((flags & MSG_PEEK) == 0) 1400 sockbuf_pushsync(&so->so_rcv, nextrecord); 1401 while (cm != NULL) { 1402 cmn = cm->m_next; 1403 cm->m_next = NULL; 1404 if (pr->pr_domain->dom_externalize != NULL) { 1405 SOCKBUF_UNLOCK(&so->so_rcv); 1406 error = (*pr->pr_domain->dom_externalize) 1407 (cm, controlp); 1408 SOCKBUF_LOCK(&so->so_rcv); 1409 } else if (controlp != NULL) 1410 *controlp = cm; 1411 else 1412 m_freem(cm); 1413 if (controlp != NULL) { 1414 orig_resid = 0; 1415 while (*controlp != NULL) 1416 controlp = &(*controlp)->m_next; 1417 } 1418 cm = cmn; 1419 } 1420 if (so->so_rcv.sb_mb) 1421 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1422 else 1423 nextrecord = NULL; 1424 orig_resid = 0; 1425 } 1426 if (m != NULL) { 1427 if ((flags & MSG_PEEK) == 0) { 1428 KASSERT(m->m_nextpkt == nextrecord, 1429 ("soreceive: post-control, nextrecord !sync")); 1430 if (nextrecord == NULL) { 1431 KASSERT(so->so_rcv.sb_mb == m, 1432 ("soreceive: post-control, sb_mb!=m")); 1433 KASSERT(so->so_rcv.sb_lastrecord == m, 1434 ("soreceive: post-control, lastrecord!=m")); 1435 } 1436 } 1437 type = m->m_type; 1438 if (type == MT_OOBDATA) 1439 flags |= MSG_OOB; 1440 } else { 1441 if ((flags & MSG_PEEK) == 0) { 1442 KASSERT(so->so_rcv.sb_mb == nextrecord, 1443 ("soreceive: sb_mb != nextrecord")); 1444 if (so->so_rcv.sb_mb == NULL) { 1445 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1446 ("soreceive: sb_lastercord != NULL")); 1447 } 1448 } 1449 } 1450 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1451 SBLASTRECORDCHK(&so->so_rcv); 1452 SBLASTMBUFCHK(&so->so_rcv); 1453 1454 /* 1455 * Now continue to read any data mbufs off of the head of the socket 1456 * buffer until the read request is satisfied. Note that 'type' is 1457 * used to store the type of any mbuf reads that have happened so far 1458 * such that soreceive() can stop reading if the type changes, which 1459 * causes soreceive() to return only one of regular data and inline 1460 * out-of-band data in a single socket receive operation. 1461 */ 1462 moff = 0; 1463 offset = 0; 1464 while (m != NULL && uio->uio_resid > 0 && error == 0) { 1465 /* 1466 * If the type of mbuf has changed since the last mbuf 1467 * examined ('type'), end the receive operation. 1468 */ 1469 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1470 if (m->m_type == MT_OOBDATA) { 1471 if (type != MT_OOBDATA) 1472 break; 1473 } else if (type == MT_OOBDATA) 1474 break; 1475 else 1476 KASSERT(m->m_type == MT_DATA, 1477 ("m->m_type == %d", m->m_type)); 1478 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1479 len = uio->uio_resid; 1480 if (so->so_oobmark && len > so->so_oobmark - offset) 1481 len = so->so_oobmark - offset; 1482 if (len > m->m_len - moff) 1483 len = m->m_len - moff; 1484 /* 1485 * If mp is set, just pass back the mbufs. 1486 * Otherwise copy them out via the uio, then free. 1487 * Sockbuf must be consistent here (points to current mbuf, 1488 * it points to next record) when we drop priority; 1489 * we must note any additions to the sockbuf when we 1490 * block interrupts again. 1491 */ 1492 if (mp == NULL) { 1493 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1494 SBLASTRECORDCHK(&so->so_rcv); 1495 SBLASTMBUFCHK(&so->so_rcv); 1496 SOCKBUF_UNLOCK(&so->so_rcv); 1497#ifdef ZERO_COPY_SOCKETS 1498 if (so_zero_copy_receive) { 1499 int disposable; 1500 1501 if ((m->m_flags & M_EXT) 1502 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1503 disposable = 1; 1504 else 1505 disposable = 0; 1506 1507 error = uiomoveco(mtod(m, char *) + moff, 1508 (int)len, uio, 1509 disposable); 1510 } else 1511#endif /* ZERO_COPY_SOCKETS */ 1512 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1513 SOCKBUF_LOCK(&so->so_rcv); 1514 if (error) 1515 goto release; 1516 } else 1517 uio->uio_resid -= len; 1518 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1519 if (len == m->m_len - moff) { 1520 if (m->m_flags & M_EOR) 1521 flags |= MSG_EOR; 1522 if (flags & MSG_PEEK) { 1523 m = m->m_next; 1524 moff = 0; 1525 } else { 1526 nextrecord = m->m_nextpkt; 1527 sbfree(&so->so_rcv, m); 1528 if (mp != NULL) { 1529 *mp = m; 1530 mp = &m->m_next; 1531 so->so_rcv.sb_mb = m = m->m_next; 1532 *mp = NULL; 1533 } else { 1534 so->so_rcv.sb_mb = m_free(m); 1535 m = so->so_rcv.sb_mb; 1536 } 1537 sockbuf_pushsync(&so->so_rcv, nextrecord); 1538 SBLASTRECORDCHK(&so->so_rcv); 1539 SBLASTMBUFCHK(&so->so_rcv); 1540 } 1541 } else { 1542 if (flags & MSG_PEEK) 1543 moff += len; 1544 else { 1545 if (mp != NULL) { 1546 int copy_flag; 1547 1548 if (flags & MSG_DONTWAIT) 1549 copy_flag = M_DONTWAIT; 1550 else 1551 copy_flag = M_TRYWAIT; 1552 if (copy_flag == M_TRYWAIT) 1553 SOCKBUF_UNLOCK(&so->so_rcv); 1554 *mp = m_copym(m, 0, len, copy_flag); 1555 if (copy_flag == M_TRYWAIT) 1556 SOCKBUF_LOCK(&so->so_rcv); 1557 if (*mp == NULL) { 1558 /* 1559 * m_copym() couldn't allocate an mbuf. 1560 * Adjust uio_resid back (it was adjusted 1561 * down by len bytes, which we didn't end 1562 * up "copying" over). 1563 */ 1564 uio->uio_resid += len; 1565 break; 1566 } 1567 } 1568 m->m_data += len; 1569 m->m_len -= len; 1570 so->so_rcv.sb_cc -= len; 1571 } 1572 } 1573 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1574 if (so->so_oobmark) { 1575 if ((flags & MSG_PEEK) == 0) { 1576 so->so_oobmark -= len; 1577 if (so->so_oobmark == 0) { 1578 so->so_rcv.sb_state |= SBS_RCVATMARK; 1579 break; 1580 } 1581 } else { 1582 offset += len; 1583 if (offset == so->so_oobmark) 1584 break; 1585 } 1586 } 1587 if (flags & MSG_EOR) 1588 break; 1589 /* 1590 * If the MSG_WAITALL flag is set (for non-atomic socket), 1591 * we must not quit until "uio->uio_resid == 0" or an error 1592 * termination. If a signal/timeout occurs, return 1593 * with a short count but without error. 1594 * Keep sockbuf locked against other readers. 1595 */ 1596 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 1597 !sosendallatonce(so) && nextrecord == NULL) { 1598 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1599 if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE) 1600 break; 1601 /* 1602 * Notify the protocol that some data has been 1603 * drained before blocking. 1604 */ 1605 if (pr->pr_flags & PR_WANTRCVD && so->so_pcb != NULL) { 1606 SOCKBUF_UNLOCK(&so->so_rcv); 1607 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1608 SOCKBUF_LOCK(&so->so_rcv); 1609 } 1610 SBLASTRECORDCHK(&so->so_rcv); 1611 SBLASTMBUFCHK(&so->so_rcv); 1612 error = sbwait(&so->so_rcv); 1613 if (error) 1614 goto release; 1615 m = so->so_rcv.sb_mb; 1616 if (m != NULL) 1617 nextrecord = m->m_nextpkt; 1618 } 1619 } 1620 1621 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1622 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 1623 flags |= MSG_TRUNC; 1624 if ((flags & MSG_PEEK) == 0) 1625 (void) sbdroprecord_locked(&so->so_rcv); 1626 } 1627 if ((flags & MSG_PEEK) == 0) { 1628 if (m == NULL) { 1629 /* 1630 * First part is an inline SB_EMPTY_FIXUP(). Second 1631 * part makes sure sb_lastrecord is up-to-date if 1632 * there is still data in the socket buffer. 1633 */ 1634 so->so_rcv.sb_mb = nextrecord; 1635 if (so->so_rcv.sb_mb == NULL) { 1636 so->so_rcv.sb_mbtail = NULL; 1637 so->so_rcv.sb_lastrecord = NULL; 1638 } else if (nextrecord->m_nextpkt == NULL) 1639 so->so_rcv.sb_lastrecord = nextrecord; 1640 } 1641 SBLASTRECORDCHK(&so->so_rcv); 1642 SBLASTMBUFCHK(&so->so_rcv); 1643 /* 1644 * If soreceive() is being done from the socket callback, then 1645 * don't need to generate ACK to peer to update window, since 1646 * ACK will be generated on return to TCP. 1647 */ 1648 if (!(flags & MSG_SOCALLBCK) && 1649 (pr->pr_flags & PR_WANTRCVD) && so->so_pcb) { 1650 SOCKBUF_UNLOCK(&so->so_rcv); 1651 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1652 SOCKBUF_LOCK(&so->so_rcv); 1653 } 1654 } 1655 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1656 if (orig_resid == uio->uio_resid && orig_resid && 1657 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 1658 sbunlock(&so->so_rcv); 1659 goto restart; 1660 } 1661 1662 if (flagsp != NULL) 1663 *flagsp |= flags; 1664release: 1665 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1666 sbunlock(&so->so_rcv); 1667out: 1668 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1669 SOCKBUF_UNLOCK(&so->so_rcv); 1670 return (error); 1671} 1672 1673int 1674soshutdown(so, how) 1675 struct socket *so; 1676 int how; 1677{ 1678 struct protosw *pr = so->so_proto; 1679 1680 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 1681 return (EINVAL); 1682 1683 if (how != SHUT_WR) 1684 sorflush(so); 1685 if (how != SHUT_RD) 1686 return ((*pr->pr_usrreqs->pru_shutdown)(so)); 1687 return (0); 1688} 1689 1690void 1691sorflush(so) 1692 struct socket *so; 1693{ 1694 struct sockbuf *sb = &so->so_rcv; 1695 struct protosw *pr = so->so_proto; 1696 struct sockbuf asb; 1697 1698 /* 1699 * XXXRW: This is quite ugly. Previously, this code made a copy of 1700 * the socket buffer, then zero'd the original to clear the buffer 1701 * fields. However, with mutexes in the socket buffer, this causes 1702 * problems. We only clear the zeroable bits of the original; 1703 * however, we have to initialize and destroy the mutex in the copy 1704 * so that dom_dispose() and sbrelease() can lock t as needed. 1705 */ 1706 SOCKBUF_LOCK(sb); 1707 sb->sb_flags |= SB_NOINTR; 1708 (void) sblock(sb, M_WAITOK); 1709 /* 1710 * socantrcvmore_locked() drops the socket buffer mutex so that it 1711 * can safely perform wakeups. Re-acquire the mutex before 1712 * continuing. 1713 */ 1714 socantrcvmore_locked(so); 1715 SOCKBUF_LOCK(sb); 1716 sbunlock(sb); 1717 /* 1718 * Invalidate/clear most of the sockbuf structure, but leave 1719 * selinfo and mutex data unchanged. 1720 */ 1721 bzero(&asb, offsetof(struct sockbuf, sb_startzero)); 1722 bcopy(&sb->sb_startzero, &asb.sb_startzero, 1723 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1724 bzero(&sb->sb_startzero, 1725 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1726 SOCKBUF_UNLOCK(sb); 1727 1728 SOCKBUF_LOCK_INIT(&asb, "so_rcv"); 1729 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1730 (*pr->pr_domain->dom_dispose)(asb.sb_mb); 1731 sbrelease(&asb, so); 1732 SOCKBUF_LOCK_DESTROY(&asb); 1733} 1734 1735/* 1736 * Perhaps this routine, and sooptcopyout(), below, ought to come in 1737 * an additional variant to handle the case where the option value needs 1738 * to be some kind of integer, but not a specific size. 1739 * In addition to their use here, these functions are also called by the 1740 * protocol-level pr_ctloutput() routines. 1741 */ 1742int 1743sooptcopyin(sopt, buf, len, minlen) 1744 struct sockopt *sopt; 1745 void *buf; 1746 size_t len; 1747 size_t minlen; 1748{ 1749 size_t valsize; 1750 1751 /* 1752 * If the user gives us more than we wanted, we ignore it, 1753 * but if we don't get the minimum length the caller 1754 * wants, we return EINVAL. On success, sopt->sopt_valsize 1755 * is set to however much we actually retrieved. 1756 */ 1757 if ((valsize = sopt->sopt_valsize) < minlen) 1758 return EINVAL; 1759 if (valsize > len) 1760 sopt->sopt_valsize = valsize = len; 1761 1762 if (sopt->sopt_td != NULL) 1763 return (copyin(sopt->sopt_val, buf, valsize)); 1764 1765 bcopy(sopt->sopt_val, buf, valsize); 1766 return (0); 1767} 1768 1769/* 1770 * Kernel version of setsockopt(2)/ 1771 * XXX: optlen is size_t, not socklen_t 1772 */ 1773int 1774so_setsockopt(struct socket *so, int level, int optname, void *optval, 1775 size_t optlen) 1776{ 1777 struct sockopt sopt; 1778 1779 sopt.sopt_level = level; 1780 sopt.sopt_name = optname; 1781 sopt.sopt_dir = SOPT_SET; 1782 sopt.sopt_val = optval; 1783 sopt.sopt_valsize = optlen; 1784 sopt.sopt_td = NULL; 1785 return (sosetopt(so, &sopt)); 1786} 1787 1788int 1789sosetopt(so, sopt) 1790 struct socket *so; 1791 struct sockopt *sopt; 1792{ 1793 int error, optval; 1794 struct linger l; 1795 struct timeval tv; 1796 u_long val; 1797#ifdef MAC 1798 struct mac extmac; 1799#endif 1800 1801 error = 0; 1802 if (sopt->sopt_level != SOL_SOCKET) { 1803 if (so->so_proto && so->so_proto->pr_ctloutput) 1804 return ((*so->so_proto->pr_ctloutput) 1805 (so, sopt)); 1806 error = ENOPROTOOPT; 1807 } else { 1808 switch (sopt->sopt_name) { 1809#ifdef INET 1810 case SO_ACCEPTFILTER: 1811 error = do_setopt_accept_filter(so, sopt); 1812 if (error) 1813 goto bad; 1814 break; 1815#endif 1816 case SO_LINGER: 1817 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 1818 if (error) 1819 goto bad; 1820 1821 SOCK_LOCK(so); 1822 so->so_linger = l.l_linger; 1823 if (l.l_onoff) 1824 so->so_options |= SO_LINGER; 1825 else 1826 so->so_options &= ~SO_LINGER; 1827 SOCK_UNLOCK(so); 1828 break; 1829 1830 case SO_DEBUG: 1831 case SO_KEEPALIVE: 1832 case SO_DONTROUTE: 1833 case SO_USELOOPBACK: 1834 case SO_BROADCAST: 1835 case SO_REUSEADDR: 1836 case SO_REUSEPORT: 1837 case SO_OOBINLINE: 1838 case SO_TIMESTAMP: 1839 case SO_BINTIME: 1840 case SO_NOSIGPIPE: 1841 error = sooptcopyin(sopt, &optval, sizeof optval, 1842 sizeof optval); 1843 if (error) 1844 goto bad; 1845 SOCK_LOCK(so); 1846 if (optval) 1847 so->so_options |= sopt->sopt_name; 1848 else 1849 so->so_options &= ~sopt->sopt_name; 1850 SOCK_UNLOCK(so); 1851 break; 1852 1853 case SO_SNDBUF: 1854 case SO_RCVBUF: 1855 case SO_SNDLOWAT: 1856 case SO_RCVLOWAT: 1857 error = sooptcopyin(sopt, &optval, sizeof optval, 1858 sizeof optval); 1859 if (error) 1860 goto bad; 1861 1862 /* 1863 * Values < 1 make no sense for any of these 1864 * options, so disallow them. 1865 */ 1866 if (optval < 1) { 1867 error = EINVAL; 1868 goto bad; 1869 } 1870 1871 switch (sopt->sopt_name) { 1872 case SO_SNDBUF: 1873 case SO_RCVBUF: 1874 if (sbreserve(sopt->sopt_name == SO_SNDBUF ? 1875 &so->so_snd : &so->so_rcv, (u_long)optval, 1876 so, curthread) == 0) { 1877 error = ENOBUFS; 1878 goto bad; 1879 } 1880 break; 1881 1882 /* 1883 * Make sure the low-water is never greater than 1884 * the high-water. 1885 */ 1886 case SO_SNDLOWAT: 1887 SOCKBUF_LOCK(&so->so_snd); 1888 so->so_snd.sb_lowat = 1889 (optval > so->so_snd.sb_hiwat) ? 1890 so->so_snd.sb_hiwat : optval; 1891 SOCKBUF_UNLOCK(&so->so_snd); 1892 break; 1893 case SO_RCVLOWAT: 1894 SOCKBUF_LOCK(&so->so_rcv); 1895 so->so_rcv.sb_lowat = 1896 (optval > so->so_rcv.sb_hiwat) ? 1897 so->so_rcv.sb_hiwat : optval; 1898 SOCKBUF_UNLOCK(&so->so_rcv); 1899 break; 1900 } 1901 break; 1902 1903 case SO_SNDTIMEO: 1904 case SO_RCVTIMEO: 1905#ifdef COMPAT_IA32 1906 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 1907 struct timeval32 tv32; 1908 1909 error = sooptcopyin(sopt, &tv32, sizeof tv32, 1910 sizeof tv32); 1911 CP(tv32, tv, tv_sec); 1912 CP(tv32, tv, tv_usec); 1913 } else 1914#endif 1915 error = sooptcopyin(sopt, &tv, sizeof tv, 1916 sizeof tv); 1917 if (error) 1918 goto bad; 1919 1920 /* assert(hz > 0); */ 1921 if (tv.tv_sec < 0 || tv.tv_sec > INT_MAX / hz || 1922 tv.tv_usec < 0 || tv.tv_usec >= 1000000) { 1923 error = EDOM; 1924 goto bad; 1925 } 1926 /* assert(tick > 0); */ 1927 /* assert(ULONG_MAX - INT_MAX >= 1000000); */ 1928 val = (u_long)(tv.tv_sec * hz) + tv.tv_usec / tick; 1929 if (val > INT_MAX) { 1930 error = EDOM; 1931 goto bad; 1932 } 1933 if (val == 0 && tv.tv_usec != 0) 1934 val = 1; 1935 1936 switch (sopt->sopt_name) { 1937 case SO_SNDTIMEO: 1938 so->so_snd.sb_timeo = val; 1939 break; 1940 case SO_RCVTIMEO: 1941 so->so_rcv.sb_timeo = val; 1942 break; 1943 } 1944 break; 1945 1946 case SO_LABEL: 1947#ifdef MAC 1948 error = sooptcopyin(sopt, &extmac, sizeof extmac, 1949 sizeof extmac); 1950 if (error) 1951 goto bad; 1952 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 1953 so, &extmac); 1954#else 1955 error = EOPNOTSUPP; 1956#endif 1957 break; 1958 1959 default: 1960 error = ENOPROTOOPT; 1961 break; 1962 } 1963 if (error == 0 && so->so_proto != NULL && 1964 so->so_proto->pr_ctloutput != NULL) { 1965 (void) ((*so->so_proto->pr_ctloutput) 1966 (so, sopt)); 1967 } 1968 } 1969bad: 1970 return (error); 1971} 1972 1973/* Helper routine for getsockopt */ 1974int 1975sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 1976{ 1977 int error; 1978 size_t valsize; 1979 1980 error = 0; 1981 1982 /* 1983 * Documented get behavior is that we always return a value, 1984 * possibly truncated to fit in the user's buffer. 1985 * Traditional behavior is that we always tell the user 1986 * precisely how much we copied, rather than something useful 1987 * like the total amount we had available for her. 1988 * Note that this interface is not idempotent; the entire answer must 1989 * generated ahead of time. 1990 */ 1991 valsize = min(len, sopt->sopt_valsize); 1992 sopt->sopt_valsize = valsize; 1993 if (sopt->sopt_val != NULL) { 1994 if (sopt->sopt_td != NULL) 1995 error = copyout(buf, sopt->sopt_val, valsize); 1996 else 1997 bcopy(buf, sopt->sopt_val, valsize); 1998 } 1999 return (error); 2000} 2001 2002int 2003sogetopt(so, sopt) 2004 struct socket *so; 2005 struct sockopt *sopt; 2006{ 2007 int error, optval; 2008 struct linger l; 2009 struct timeval tv; 2010#ifdef MAC 2011 struct mac extmac; 2012#endif 2013 2014 error = 0; 2015 if (sopt->sopt_level != SOL_SOCKET) { 2016 if (so->so_proto && so->so_proto->pr_ctloutput) { 2017 return ((*so->so_proto->pr_ctloutput) 2018 (so, sopt)); 2019 } else 2020 return (ENOPROTOOPT); 2021 } else { 2022 switch (sopt->sopt_name) { 2023#ifdef INET 2024 case SO_ACCEPTFILTER: 2025 error = do_getopt_accept_filter(so, sopt); 2026 break; 2027#endif 2028 case SO_LINGER: 2029 SOCK_LOCK(so); 2030 l.l_onoff = so->so_options & SO_LINGER; 2031 l.l_linger = so->so_linger; 2032 SOCK_UNLOCK(so); 2033 error = sooptcopyout(sopt, &l, sizeof l); 2034 break; 2035 2036 case SO_USELOOPBACK: 2037 case SO_DONTROUTE: 2038 case SO_DEBUG: 2039 case SO_KEEPALIVE: 2040 case SO_REUSEADDR: 2041 case SO_REUSEPORT: 2042 case SO_BROADCAST: 2043 case SO_OOBINLINE: 2044 case SO_ACCEPTCONN: 2045 case SO_TIMESTAMP: 2046 case SO_BINTIME: 2047 case SO_NOSIGPIPE: 2048 optval = so->so_options & sopt->sopt_name; 2049integer: 2050 error = sooptcopyout(sopt, &optval, sizeof optval); 2051 break; 2052 2053 case SO_TYPE: 2054 optval = so->so_type; 2055 goto integer; 2056 2057 case SO_ERROR: 2058 optval = so->so_error; 2059 so->so_error = 0; 2060 goto integer; 2061 2062 case SO_SNDBUF: 2063 optval = so->so_snd.sb_hiwat; 2064 goto integer; 2065 2066 case SO_RCVBUF: 2067 optval = so->so_rcv.sb_hiwat; 2068 goto integer; 2069 2070 case SO_SNDLOWAT: 2071 optval = so->so_snd.sb_lowat; 2072 goto integer; 2073 2074 case SO_RCVLOWAT: 2075 optval = so->so_rcv.sb_lowat; 2076 goto integer; 2077 2078 case SO_SNDTIMEO: 2079 case SO_RCVTIMEO: 2080 optval = (sopt->sopt_name == SO_SNDTIMEO ? 2081 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 2082 2083 tv.tv_sec = optval / hz; 2084 tv.tv_usec = (optval % hz) * tick; 2085#ifdef COMPAT_IA32 2086 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 2087 struct timeval32 tv32; 2088 2089 CP(tv, tv32, tv_sec); 2090 CP(tv, tv32, tv_usec); 2091 error = sooptcopyout(sopt, &tv32, sizeof tv32); 2092 } else 2093#endif 2094 error = sooptcopyout(sopt, &tv, sizeof tv); 2095 break; 2096 2097 case SO_LABEL: 2098#ifdef MAC 2099 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2100 sizeof(extmac)); 2101 if (error) 2102 return (error); 2103 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 2104 so, &extmac); 2105 if (error) 2106 return (error); 2107 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2108#else 2109 error = EOPNOTSUPP; 2110#endif 2111 break; 2112 2113 case SO_PEERLABEL: 2114#ifdef MAC 2115 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2116 sizeof(extmac)); 2117 if (error) 2118 return (error); 2119 error = mac_getsockopt_peerlabel( 2120 sopt->sopt_td->td_ucred, so, &extmac); 2121 if (error) 2122 return (error); 2123 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2124#else 2125 error = EOPNOTSUPP; 2126#endif 2127 break; 2128 2129 case SO_LISTENQLIMIT: 2130 optval = so->so_qlimit; 2131 goto integer; 2132 2133 case SO_LISTENQLEN: 2134 optval = so->so_qlen; 2135 goto integer; 2136 2137 case SO_LISTENINCQLEN: 2138 optval = so->so_incqlen; 2139 goto integer; 2140 2141 default: 2142 error = ENOPROTOOPT; 2143 break; 2144 } 2145 return (error); 2146 } 2147} 2148 2149/* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */ 2150int 2151soopt_getm(struct sockopt *sopt, struct mbuf **mp) 2152{ 2153 struct mbuf *m, *m_prev; 2154 int sopt_size = sopt->sopt_valsize; 2155 2156 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2157 if (m == NULL) 2158 return ENOBUFS; 2159 if (sopt_size > MLEN) { 2160 MCLGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT); 2161 if ((m->m_flags & M_EXT) == 0) { 2162 m_free(m); 2163 return ENOBUFS; 2164 } 2165 m->m_len = min(MCLBYTES, sopt_size); 2166 } else { 2167 m->m_len = min(MLEN, sopt_size); 2168 } 2169 sopt_size -= m->m_len; 2170 *mp = m; 2171 m_prev = m; 2172 2173 while (sopt_size) { 2174 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2175 if (m == NULL) { 2176 m_freem(*mp); 2177 return ENOBUFS; 2178 } 2179 if (sopt_size > MLEN) { 2180 MCLGET(m, sopt->sopt_td != NULL ? M_TRYWAIT : 2181 M_DONTWAIT); 2182 if ((m->m_flags & M_EXT) == 0) { 2183 m_freem(m); 2184 m_freem(*mp); 2185 return ENOBUFS; 2186 } 2187 m->m_len = min(MCLBYTES, sopt_size); 2188 } else { 2189 m->m_len = min(MLEN, sopt_size); 2190 } 2191 sopt_size -= m->m_len; 2192 m_prev->m_next = m; 2193 m_prev = m; 2194 } 2195 return (0); 2196} 2197 2198/* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */ 2199int 2200soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 2201{ 2202 struct mbuf *m0 = m; 2203 2204 if (sopt->sopt_val == NULL) 2205 return (0); 2206 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2207 if (sopt->sopt_td != NULL) { 2208 int error; 2209 2210 error = copyin(sopt->sopt_val, mtod(m, char *), 2211 m->m_len); 2212 if (error != 0) { 2213 m_freem(m0); 2214 return(error); 2215 } 2216 } else 2217 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 2218 sopt->sopt_valsize -= m->m_len; 2219 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2220 m = m->m_next; 2221 } 2222 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 2223 panic("ip6_sooptmcopyin"); 2224 return (0); 2225} 2226 2227/* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */ 2228int 2229soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 2230{ 2231 struct mbuf *m0 = m; 2232 size_t valsize = 0; 2233 2234 if (sopt->sopt_val == NULL) 2235 return (0); 2236 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2237 if (sopt->sopt_td != NULL) { 2238 int error; 2239 2240 error = copyout(mtod(m, char *), sopt->sopt_val, 2241 m->m_len); 2242 if (error != 0) { 2243 m_freem(m0); 2244 return(error); 2245 } 2246 } else 2247 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 2248 sopt->sopt_valsize -= m->m_len; 2249 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2250 valsize += m->m_len; 2251 m = m->m_next; 2252 } 2253 if (m != NULL) { 2254 /* enough soopt buffer should be given from user-land */ 2255 m_freem(m0); 2256 return(EINVAL); 2257 } 2258 sopt->sopt_valsize = valsize; 2259 return (0); 2260} 2261 2262void 2263sohasoutofband(so) 2264 struct socket *so; 2265{ 2266 if (so->so_sigio != NULL) 2267 pgsigio(&so->so_sigio, SIGURG, 0); 2268 selwakeuppri(&so->so_rcv.sb_sel, PSOCK); 2269} 2270 2271int 2272sopoll(struct socket *so, int events, struct ucred *active_cred, 2273 struct thread *td) 2274{ 2275 int revents = 0; 2276 2277 SOCKBUF_LOCK(&so->so_snd); 2278 SOCKBUF_LOCK(&so->so_rcv); 2279 if (events & (POLLIN | POLLRDNORM)) 2280 if (soreadable(so)) 2281 revents |= events & (POLLIN | POLLRDNORM); 2282 2283 if (events & POLLINIGNEOF) 2284 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat || 2285 !TAILQ_EMPTY(&so->so_comp) || so->so_error) 2286 revents |= POLLINIGNEOF; 2287 2288 if (events & (POLLOUT | POLLWRNORM)) 2289 if (sowriteable(so)) 2290 revents |= events & (POLLOUT | POLLWRNORM); 2291 2292 if (events & (POLLPRI | POLLRDBAND)) 2293 if (so->so_oobmark || (so->so_rcv.sb_state & SBS_RCVATMARK)) 2294 revents |= events & (POLLPRI | POLLRDBAND); 2295 2296 if (revents == 0) { 2297 if (events & 2298 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | 2299 POLLRDBAND)) { 2300 selrecord(td, &so->so_rcv.sb_sel); 2301 so->so_rcv.sb_flags |= SB_SEL; 2302 } 2303 2304 if (events & (POLLOUT | POLLWRNORM)) { 2305 selrecord(td, &so->so_snd.sb_sel); 2306 so->so_snd.sb_flags |= SB_SEL; 2307 } 2308 } 2309 2310 SOCKBUF_UNLOCK(&so->so_rcv); 2311 SOCKBUF_UNLOCK(&so->so_snd); 2312 return (revents); 2313} 2314 2315int 2316soo_kqfilter(struct file *fp, struct knote *kn) 2317{ 2318 struct socket *so = kn->kn_fp->f_data; 2319 struct sockbuf *sb; 2320 2321 switch (kn->kn_filter) { 2322 case EVFILT_READ: 2323 if (so->so_options & SO_ACCEPTCONN) 2324 kn->kn_fop = &solisten_filtops; 2325 else 2326 kn->kn_fop = &soread_filtops; 2327 sb = &so->so_rcv; 2328 break; 2329 case EVFILT_WRITE: 2330 kn->kn_fop = &sowrite_filtops; 2331 sb = &so->so_snd; 2332 break; 2333 default: 2334 return (EINVAL); 2335 } 2336 2337 SOCKBUF_LOCK(sb); 2338 knlist_add(&sb->sb_sel.si_note, kn, 1); 2339 sb->sb_flags |= SB_KNOTE; 2340 SOCKBUF_UNLOCK(sb); 2341 return (0); 2342} 2343 2344static void 2345filt_sordetach(struct knote *kn) 2346{ 2347 struct socket *so = kn->kn_fp->f_data; 2348 2349 SOCKBUF_LOCK(&so->so_rcv); 2350 knlist_remove(&so->so_rcv.sb_sel.si_note, kn, 1); 2351 if (knlist_empty(&so->so_rcv.sb_sel.si_note)) 2352 so->so_rcv.sb_flags &= ~SB_KNOTE; 2353 SOCKBUF_UNLOCK(&so->so_rcv); 2354} 2355 2356/*ARGSUSED*/ 2357static int 2358filt_soread(struct knote *kn, long hint) 2359{ 2360 struct socket *so; 2361 2362 so = kn->kn_fp->f_data; 2363 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2364 2365 kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl; 2366 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2367 kn->kn_flags |= EV_EOF; 2368 kn->kn_fflags = so->so_error; 2369 return (1); 2370 } else if (so->so_error) /* temporary udp error */ 2371 return (1); 2372 else if (kn->kn_sfflags & NOTE_LOWAT) 2373 return (kn->kn_data >= kn->kn_sdata); 2374 else 2375 return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat); 2376} 2377 2378static void 2379filt_sowdetach(struct knote *kn) 2380{ 2381 struct socket *so = kn->kn_fp->f_data; 2382 2383 SOCKBUF_LOCK(&so->so_snd); 2384 knlist_remove(&so->so_snd.sb_sel.si_note, kn, 1); 2385 if (knlist_empty(&so->so_snd.sb_sel.si_note)) 2386 so->so_snd.sb_flags &= ~SB_KNOTE; 2387 SOCKBUF_UNLOCK(&so->so_snd); 2388} 2389 2390/*ARGSUSED*/ 2391static int 2392filt_sowrite(struct knote *kn, long hint) 2393{ 2394 struct socket *so; 2395 2396 so = kn->kn_fp->f_data; 2397 SOCKBUF_LOCK_ASSERT(&so->so_snd); 2398 kn->kn_data = sbspace(&so->so_snd); 2399 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2400 kn->kn_flags |= EV_EOF; 2401 kn->kn_fflags = so->so_error; 2402 return (1); 2403 } else if (so->so_error) /* temporary udp error */ 2404 return (1); 2405 else if (((so->so_state & SS_ISCONNECTED) == 0) && 2406 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 2407 return (0); 2408 else if (kn->kn_sfflags & NOTE_LOWAT) 2409 return (kn->kn_data >= kn->kn_sdata); 2410 else 2411 return (kn->kn_data >= so->so_snd.sb_lowat); 2412} 2413 2414/*ARGSUSED*/ 2415static int 2416filt_solisten(struct knote *kn, long hint) 2417{ 2418 struct socket *so = kn->kn_fp->f_data; 2419 2420 kn->kn_data = so->so_qlen; 2421 return (! TAILQ_EMPTY(&so->so_comp)); 2422} 2423 2424int 2425socheckuid(struct socket *so, uid_t uid) 2426{ 2427 2428 if (so == NULL) 2429 return (EPERM); 2430 if (so->so_cred->cr_uid != uid) 2431 return (EPERM); 2432 return (0); 2433} 2434 2435static int 2436somaxconn_sysctl(SYSCTL_HANDLER_ARGS) 2437{ 2438 int error; 2439 int val; 2440 2441 val = somaxconn; 2442 error = sysctl_handle_int(oidp, &val, sizeof(int), req); 2443 if (error || !req->newptr ) 2444 return (error); 2445 2446 if (val < 1 || val > USHRT_MAX) 2447 return (EINVAL); 2448 2449 somaxconn = val; 2450 return (0); 2451}
|