1/*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 16 unchanged lines hidden (view full) --- 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93 30 */ 31 32#include <sys/cdefs.h> |
33__FBSDID("$FreeBSD: head/sys/kern/uipc_sockbuf.c 160915 2006-08-02 13:01:58Z rwatson $"); |
34 35#include "opt_param.h" 36 37#include <sys/param.h> 38#include <sys/aio.h> /* for aio_swake proto */ 39#include <sys/kernel.h> 40#include <sys/lock.h> 41#include <sys/mbuf.h> --- 17 unchanged lines hidden (view full) --- 59 */ 60 61u_long sb_max = SB_MAX; 62static u_long sb_max_adj = 63 SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 64 65static u_long sb_efficiency = 8; /* parameter for sbreserve() */ 66 |
67static void sbdrop_internal(struct sockbuf *sb, int len); 68static void sbflush_internal(struct sockbuf *sb); |
69static void sbrelease_internal(struct sockbuf *sb, struct socket *so); 70 71/* |
72 * Socantsendmore indicates that no more data will be sent on the socket; it 73 * would normally be applied to a socket when the user informs the system 74 * that no more data is to be sent, by the protocol code (in case 75 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 76 * received, and will normally be applied to the socket by a protocol when it 77 * detects that the peer will send no more data. Data queued for reading in 78 * the socket may yet be read. |
79 */ 80void |
81socantsendmore_locked(struct socket *so) |
82{ 83 84 SOCKBUF_LOCK_ASSERT(&so->so_snd); 85 86 so->so_snd.sb_state |= SBS_CANTSENDMORE; 87 sowwakeup_locked(so); 88 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 89} 90 91void |
92socantsendmore(struct socket *so) |
93{ 94 95 SOCKBUF_LOCK(&so->so_snd); 96 socantsendmore_locked(so); 97 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 98} 99 100void |
101socantrcvmore_locked(struct socket *so) |
102{ 103 104 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 105 106 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 107 sorwakeup_locked(so); 108 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 109} 110 111void |
112socantrcvmore(struct socket *so) |
113{ 114 115 SOCKBUF_LOCK(&so->so_rcv); 116 socantrcvmore_locked(so); 117 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 118} 119 120/* 121 * Wait for data to arrive at/drain from a socket buffer. 122 */ 123int |
124sbwait(struct sockbuf *sb) |
125{ 126 127 SOCKBUF_LOCK_ASSERT(sb); 128 129 sb->sb_flags |= SB_WAIT; 130 return (msleep(&sb->sb_cc, &sb->sb_mtx, 131 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 132 sb->sb_timeo)); 133} 134 135/* |
136 * Lock a sockbuf already known to be locked; return any error returned from 137 * sleep (EINTR). |
138 */ 139int |
140sb_lock(struct sockbuf *sb) |
141{ 142 int error; 143 144 SOCKBUF_LOCK_ASSERT(sb); 145 146 while (sb->sb_flags & SB_LOCK) { 147 sb->sb_flags |= SB_WANT; 148 error = msleep(&sb->sb_flags, &sb->sb_mtx, 149 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH, 150 "sblock", 0); 151 if (error) 152 return (error); 153 } 154 sb->sb_flags |= SB_LOCK; 155 return (0); 156} 157 158/* |
159 * Wakeup processes waiting on a socket buffer. Do asynchronous notification 160 * via SIGIO if the socket has the SS_ASYNC flag set. |
161 * 162 * Called with the socket buffer lock held; will release the lock by the end 163 * of the function. This allows the caller to acquire the socket buffer lock 164 * while testing for the need for various sorts of wakeup and hold it through 165 * to the point where it's no longer required. We currently hold the lock 166 * through calls out to other subsystems (with the exception of kqueue), and 167 * then release it to avoid lock order issues. It's not clear that's 168 * correct. 169 */ 170void |
171sowakeup(struct socket *so, struct sockbuf *sb) |
172{ 173 174 SOCKBUF_LOCK_ASSERT(sb); 175 176 selwakeuppri(&sb->sb_sel, PSOCK); 177 sb->sb_flags &= ~SB_SEL; 178 if (sb->sb_flags & SB_WAIT) { 179 sb->sb_flags &= ~SB_WAIT; --- 8 unchanged lines hidden (view full) --- 188 if (sb->sb_flags & SB_AIO) 189 aio_swake(so, sb); 190 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 191} 192 193/* 194 * Socket buffer (struct sockbuf) utility routines. 195 * |
196 * Each socket contains two socket buffers: one for sending data and one for 197 * receiving data. Each buffer contains a queue of mbufs, information about 198 * the number of mbufs and amount of data in the queue, and other fields 199 * allowing select() statements and notification on data availability to be 200 * implemented. |
201 * |
202 * Data stored in a socket buffer is maintained as a list of records. Each 203 * record is a list of mbufs chained together with the m_next field. Records 204 * are chained together with the m_nextpkt field. The upper level routine 205 * soreceive() expects the following conventions to be observed when placing 206 * information in the receive buffer: |
207 * |
208 * 1. If the protocol requires each message be preceded by the sender's name, 209 * then a record containing that name must be present before any 210 * associated data (mbuf's must be of type MT_SONAME). 211 * 2. If the protocol supports the exchange of ``access rights'' (really just 212 * additional data associated with the message), and there are ``rights'' 213 * to be received, then a record containing this data should be present 214 * (mbuf's must be of type MT_RIGHTS). 215 * 3. If a name or rights record exists, then it must be followed by a data 216 * record, perhaps of zero length. |
217 * 218 * Before using a new socket structure it is first necessary to reserve 219 * buffer space to the socket, by calling sbreserve(). This should commit 220 * some of the available buffer space in the system buffer pool for the |
221 * socket (currently, it does nothing but enforce limits). The space should 222 * be released by calling sbrelease() when the socket is destroyed. |
223 */ |
224int |
225soreserve(struct socket *so, u_long sndcc, u_long rcvcc) |
226{ 227 struct thread *td = curthread; 228 229 SOCKBUF_LOCK(&so->so_snd); 230 SOCKBUF_LOCK(&so->so_rcv); 231 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 232 goto bad; 233 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) --- 31 unchanged lines hidden (view full) --- 265 sb_max = old_sb_max; 266 return (EINVAL); 267 } 268 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 269 return (0); 270} 271 272/* |
273 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 274 * become limiting if buffering efficiency is near the normal case. |
275 */ 276int |
277sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 278 struct thread *td) |
279{ 280 rlim_t sbsize_limit; 281 282 SOCKBUF_LOCK_ASSERT(sb); 283 284 /* |
285 * td will only be NULL when we're in an interrupt (e.g. in 286 * tcp_input()). 287 * 288 * XXXRW: This comment needs updating, as might the code. |
289 */ 290 if (cc > sb_max_adj) 291 return (0); 292 if (td != NULL) { 293 PROC_LOCK(td->td_proc); 294 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 295 PROC_UNLOCK(td->td_proc); 296 } else 297 sbsize_limit = RLIM_INFINITY; 298 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 299 sbsize_limit)) 300 return (0); 301 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 302 if (sb->sb_lowat > sb->sb_hiwat) 303 sb->sb_lowat = sb->sb_hiwat; 304 return (1); 305} 306 307int |
308sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 309 struct thread *td) |
310{ 311 int error; 312 313 SOCKBUF_LOCK(sb); 314 error = sbreserve_locked(sb, cc, so, td); 315 SOCKBUF_UNLOCK(sb); 316 return (error); 317} 318 319/* 320 * Free mbufs held by a socket, and reserved mbuf space. 321 */ 322static void |
323sbrelease_internal(struct sockbuf *sb, struct socket *so) |
324{ 325 326 sbflush_internal(sb); 327 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 328 RLIM_INFINITY); 329 sb->sb_mbmax = 0; 330} 331 332void |
333sbrelease_locked(struct sockbuf *sb, struct socket *so) |
334{ 335 336 SOCKBUF_LOCK_ASSERT(sb); 337 338 sbrelease_internal(sb, so); 339} 340 341void |
342sbrelease(struct sockbuf *sb, struct socket *so) |
343{ 344 345 SOCKBUF_LOCK(sb); 346 sbrelease_locked(sb, so); 347 SOCKBUF_UNLOCK(sb); 348} 349 350void |
351sbdestroy(struct sockbuf *sb, struct socket *so) |
352{ 353 354 sbrelease_internal(sb, so); 355} 356 357 358/* |
359 * Routines to add and remove data from an mbuf queue. |
360 * |
361 * The routines sbappend() or sbappendrecord() are normally called to append 362 * new mbufs to a socket buffer, after checking that adequate space is 363 * available, comparing the function sbspace() with the amount of data to be 364 * added. sbappendrecord() differs from sbappend() in that data supplied is 365 * treated as the beginning of a new record. To place a sender's address, 366 * optional access rights, and data in a socket receive buffer, 367 * sbappendaddr() should be used. To place access rights and data in a 368 * socket receive buffer, sbappendrights() should be used. In either case, 369 * the new data begins a new record. Note that unlike sbappend() and 370 * sbappendrecord(), these routines check for the caller that there will be 371 * enough space to store the data. Each fails if there is not enough space, 372 * or if it cannot find mbufs to store additional information in. |
373 * |
374 * Reliable protocols may use the socket send buffer to hold data awaiting 375 * acknowledgement. Data is normally copied from a socket send buffer in a 376 * protocol with m_copy for output to a peer, and then removing the data from 377 * the socket buffer with sbdrop() or sbdroprecord() when the data is 378 * acknowledged by the peer. |
379 */ |
380#ifdef SOCKBUF_DEBUG 381void 382sblastrecordchk(struct sockbuf *sb, const char *file, int line) 383{ 384 struct mbuf *m = sb->sb_mb; 385 386 SOCKBUF_LOCK_ASSERT(sb); 387 --- 44 unchanged lines hidden (view full) --- 432 if ((sb)->sb_lastrecord != NULL) \ 433 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 434 else \ 435 (sb)->sb_mb = (m0); \ 436 (sb)->sb_lastrecord = (m0); \ 437} while (/*CONSTCOND*/0) 438 439/* |
440 * Append mbuf chain m to the last record in the socket buffer sb. The 441 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 442 * are discarded and mbufs are compacted where possible. |
443 */ 444void |
445sbappend_locked(struct sockbuf *sb, struct mbuf *m) |
446{ |
447 struct mbuf *n; |
448 449 SOCKBUF_LOCK_ASSERT(sb); 450 451 if (m == 0) 452 return; 453 454 SBLASTRECORDCHK(sb); 455 n = sb->sb_mb; --- 27 unchanged lines hidden (view full) --- 483 sb->sb_lastrecord = m; 484 } 485 } 486 sbcompress(sb, m, n); 487 SBLASTRECORDCHK(sb); 488} 489 490/* |
491 * Append mbuf chain m to the last record in the socket buffer sb. The 492 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 493 * are discarded and mbufs are compacted where possible. |
494 */ 495void |
496sbappend(struct sockbuf *sb, struct mbuf *m) |
497{ 498 499 SOCKBUF_LOCK(sb); 500 sbappend_locked(sb, m); 501 SOCKBUF_UNLOCK(sb); 502} 503 504/* |
505 * This version of sbappend() should only be used when the caller absolutely 506 * knows that there will never be more than one record in the socket buffer, 507 * that is, a stream protocol (such as TCP). |
508 */ 509void 510sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 511{ 512 SOCKBUF_LOCK_ASSERT(sb); 513 514 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 515 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 516 517 SBLASTMBUFCHK(sb); 518 519 sbcompress(sb, m, sb->sb_mbtail); 520 521 sb->sb_lastrecord = sb->sb_mb; 522 SBLASTRECORDCHK(sb); 523} 524 525/* |
526 * This version of sbappend() should only be used when the caller absolutely 527 * knows that there will never be more than one record in the socket buffer, 528 * that is, a stream protocol (such as TCP). |
529 */ 530void 531sbappendstream(struct sockbuf *sb, struct mbuf *m) 532{ 533 534 SOCKBUF_LOCK(sb); 535 sbappendstream_locked(sb, m); 536 SOCKBUF_UNLOCK(sb); 537} 538 539#ifdef SOCKBUF_DEBUG 540void |
541sbcheck(struct sockbuf *sb) |
542{ 543 struct mbuf *m; 544 struct mbuf *n = 0; 545 u_long len = 0, mbcnt = 0; 546 547 SOCKBUF_LOCK_ASSERT(sb); 548 549 for (m = sb->sb_mb; m; m = n) { --- 9 unchanged lines hidden (view full) --- 559 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 560 mbcnt, sb->sb_mbcnt); 561 panic("sbcheck"); 562 } 563} 564#endif 565 566/* |
567 * As above, except the mbuf chain begins a new record. |
568 */ 569void |
570sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) |
571{ |
572 struct mbuf *m; |
573 574 SOCKBUF_LOCK_ASSERT(sb); 575 576 if (m0 == 0) 577 return; 578 m = sb->sb_mb; 579 if (m) 580 while (m->m_nextpkt) 581 m = m->m_nextpkt; 582 /* |
583 * Put the first mbuf on the queue. Note this permits zero length 584 * records. |
585 */ 586 sballoc(sb, m0); 587 SBLASTRECORDCHK(sb); 588 SBLINKRECORD(sb, m0); 589 if (m) 590 m->m_nextpkt = m0; 591 else 592 sb->sb_mb = m0; 593 m = m0->m_next; 594 m0->m_next = 0; 595 if (m && (m0->m_flags & M_EOR)) { 596 m0->m_flags &= ~M_EOR; 597 m->m_flags |= M_EOR; 598 } 599 sbcompress(sb, m, m0); 600} 601 602/* |
603 * As above, except the mbuf chain begins a new record. |
604 */ 605void |
606sbappendrecord(struct sockbuf *sb, struct mbuf *m0) |
607{ 608 609 SOCKBUF_LOCK(sb); 610 sbappendrecord_locked(sb, m0); 611 SOCKBUF_UNLOCK(sb); 612} 613 614/* |
615 * Append address and data, and optionally, control (ancillary) data to the 616 * receive queue of a socket. If present, m0 must include a packet header 617 * with total length. Returns 0 if no space in sockbuf or insufficient 618 * mbufs. |
619 */ 620int |
621sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 622 struct mbuf *m0, struct mbuf *control) |
623{ 624 struct mbuf *m, *n, *nlast; 625 int space = asa->sa_len; 626 627 SOCKBUF_LOCK_ASSERT(sb); 628 629 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 630 panic("sbappendaddr_locked"); --- 26 unchanged lines hidden (view full) --- 657 sb->sb_mbtail = nlast; 658 SBLASTMBUFCHK(sb); 659 660 SBLASTRECORDCHK(sb); 661 return (1); 662} 663 664/* |
665 * Append address and data, and optionally, control (ancillary) data to the 666 * receive queue of a socket. If present, m0 must include a packet header 667 * with total length. Returns 0 if no space in sockbuf or insufficient 668 * mbufs. |
669 */ 670int |
671sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 672 struct mbuf *m0, struct mbuf *control) |
673{ 674 int retval; 675 676 SOCKBUF_LOCK(sb); 677 retval = sbappendaddr_locked(sb, asa, m0, control); 678 SOCKBUF_UNLOCK(sb); 679 return (retval); 680} 681 682int |
683sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 684 struct mbuf *control) |
685{ 686 struct mbuf *m, *n, *mlast; 687 int space; 688 689 SOCKBUF_LOCK_ASSERT(sb); 690 691 if (control == 0) 692 panic("sbappendcontrol_locked"); --- 14 unchanged lines hidden (view full) --- 707 sb->sb_mbtail = mlast; 708 SBLASTMBUFCHK(sb); 709 710 SBLASTRECORDCHK(sb); 711 return (1); 712} 713 714int |
715sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) |
716{ 717 int retval; 718 719 SOCKBUF_LOCK(sb); 720 retval = sbappendcontrol_locked(sb, m0, control); 721 SOCKBUF_UNLOCK(sb); 722 return (retval); 723} --- 14 unchanged lines hidden (view full) --- 738 * will occur. 739 * 740 * (3) The mbuf may be appended to the end of the existing mbuf chain. 741 * 742 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 743 * end-of-record. 744 */ 745void |
746sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) |
747{ |
748 int eor = 0; 749 struct mbuf *o; |
750 751 SOCKBUF_LOCK_ASSERT(sb); 752 753 while (m) { 754 eor |= m->m_flags & M_EOR; 755 if (m->m_len == 0 && 756 (eor == 0 || 757 (((o = m->m_next) || (o = n)) && --- 32 unchanged lines hidden (view full) --- 790 if (eor) { 791 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 792 n->m_flags |= eor; 793 } 794 SBLASTMBUFCHK(sb); 795} 796 797/* |
798 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. |
799 */ 800static void |
801sbflush_internal(struct sockbuf *sb) |
802{ 803 804 if (sb->sb_flags & SB_LOCK) |
805 panic("sbflush_internal: locked"); |
806 while (sb->sb_mbcnt) { 807 /* 808 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 809 * we would loop forever. Panic instead. 810 */ 811 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 812 break; 813 sbdrop_internal(sb, (int)sb->sb_cc); 814 } 815 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) |
816 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 817 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); |
818} 819 820void |
821sbflush_locked(struct sockbuf *sb) |
822{ 823 824 SOCKBUF_LOCK_ASSERT(sb); 825 sbflush_internal(sb); 826} 827 828void |
829sbflush(struct sockbuf *sb) |
830{ 831 832 SOCKBUF_LOCK(sb); 833 sbflush_locked(sb); 834 SOCKBUF_UNLOCK(sb); 835} 836 837/* 838 * Drop data from (the front of) a sockbuf. 839 */ 840static void |
841sbdrop_internal(struct sockbuf *sb, int len) |
842{ |
843 struct mbuf *m; |
844 struct mbuf *next; 845 846 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 847 while (len > 0) { 848 if (m == 0) { 849 if (next == 0) 850 panic("sbdrop"); 851 m = next; --- 17 unchanged lines hidden (view full) --- 869 m = m_free(m); 870 } 871 if (m) { 872 sb->sb_mb = m; 873 m->m_nextpkt = next; 874 } else 875 sb->sb_mb = next; 876 /* |
877 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 878 * sb_lastrecord is up-to-date if we dropped part of the last record. |
879 */ 880 m = sb->sb_mb; 881 if (m == NULL) { 882 sb->sb_mbtail = NULL; 883 sb->sb_lastrecord = NULL; 884 } else if (m->m_nextpkt == NULL) { 885 sb->sb_lastrecord = m; 886 } 887} 888 889/* 890 * Drop data from (the front of) a sockbuf. 891 */ 892void |
893sbdrop_locked(struct sockbuf *sb, int len) |
894{ 895 896 SOCKBUF_LOCK_ASSERT(sb); 897 898 sbdrop_internal(sb, len); 899} 900 901void |
902sbdrop(struct sockbuf *sb, int len) |
903{ 904 905 SOCKBUF_LOCK(sb); 906 sbdrop_locked(sb, len); 907 SOCKBUF_UNLOCK(sb); 908} 909 910/* |
911 * Drop a record off the front of a sockbuf and move the next record to the 912 * front. |
913 */ 914void |
915sbdroprecord_locked(struct sockbuf *sb) |
916{ |
917 struct mbuf *m; |
918 919 SOCKBUF_LOCK_ASSERT(sb); 920 921 m = sb->sb_mb; 922 if (m) { 923 sb->sb_mb = m->m_nextpkt; 924 do { 925 sbfree(sb, m); 926 m = m_free(m); 927 } while (m); 928 } 929 SB_EMPTY_FIXUP(sb); 930} 931 932/* |
933 * Drop a record off the front of a sockbuf and move the next record to the 934 * front. |
935 */ 936void |
937sbdroprecord(struct sockbuf *sb) |
938{ 939 940 SOCKBUF_LOCK(sb); 941 sbdroprecord_locked(sb); 942 SOCKBUF_UNLOCK(sb); 943} 944 945/* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 946static int dummy; 947SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, ""); 948SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW, 949 &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size"); 950SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 951 &sb_efficiency, 0, ""); |