34 35#include "opt_param.h" 36 37#include <sys/param.h> 38#include <sys/aio.h> /* for aio_swake proto */ 39#include <sys/kernel.h> 40#include <sys/lock.h> 41#include <sys/mbuf.h> 42#include <sys/mutex.h> 43#include <sys/proc.h> 44#include <sys/protosw.h> 45#include <sys/resourcevar.h> 46#include <sys/signalvar.h> 47#include <sys/socket.h> 48#include <sys/socketvar.h> 49#include <sys/sysctl.h> 50 51/* 52 * Function pointer set by the AIO routines so that the socket buffer code 53 * can call back into the AIO module if it is loaded. 54 */ 55void (*aio_swake)(struct socket *, struct sockbuf *); 56 57/* 58 * Primitive routines for operating on socket buffers 59 */ 60 61u_long sb_max = SB_MAX; 62static u_long sb_max_adj = 63 SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 64 65static u_long sb_efficiency = 8; /* parameter for sbreserve() */ 66 67static void sbdrop_internal(struct sockbuf *sb, int len); 68static void sbflush_internal(struct sockbuf *sb); 69static void sbrelease_internal(struct sockbuf *sb, struct socket *so); 70 71/* 72 * Socantsendmore indicates that no more data will be sent on the socket; it 73 * would normally be applied to a socket when the user informs the system 74 * that no more data is to be sent, by the protocol code (in case 75 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 76 * received, and will normally be applied to the socket by a protocol when it 77 * detects that the peer will send no more data. Data queued for reading in 78 * the socket may yet be read. 79 */ 80void 81socantsendmore_locked(struct socket *so) 82{ 83 84 SOCKBUF_LOCK_ASSERT(&so->so_snd); 85 86 so->so_snd.sb_state |= SBS_CANTSENDMORE; 87 sowwakeup_locked(so); 88 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 89} 90 91void 92socantsendmore(struct socket *so) 93{ 94 95 SOCKBUF_LOCK(&so->so_snd); 96 socantsendmore_locked(so); 97 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 98} 99 100void 101socantrcvmore_locked(struct socket *so) 102{ 103 104 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 105 106 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 107 sorwakeup_locked(so); 108 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 109} 110 111void 112socantrcvmore(struct socket *so) 113{ 114 115 SOCKBUF_LOCK(&so->so_rcv); 116 socantrcvmore_locked(so); 117 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 118} 119 120/* 121 * Wait for data to arrive at/drain from a socket buffer. 122 */ 123int 124sbwait(struct sockbuf *sb) 125{ 126 127 SOCKBUF_LOCK_ASSERT(sb); 128 129 sb->sb_flags |= SB_WAIT; 130 return (msleep(&sb->sb_cc, &sb->sb_mtx, 131 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 132 sb->sb_timeo)); 133} 134 135/* 136 * Lock a sockbuf already known to be locked; return any error returned from 137 * sleep (EINTR). 138 */ 139int 140sb_lock(struct sockbuf *sb) 141{ 142 int error; 143 144 SOCKBUF_LOCK_ASSERT(sb); 145 146 while (sb->sb_flags & SB_LOCK) { 147 sb->sb_flags |= SB_WANT; 148 error = msleep(&sb->sb_flags, &sb->sb_mtx, 149 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH, 150 "sblock", 0); 151 if (error) 152 return (error); 153 } 154 sb->sb_flags |= SB_LOCK; 155 return (0); 156} 157 158/* 159 * Wakeup processes waiting on a socket buffer. Do asynchronous notification 160 * via SIGIO if the socket has the SS_ASYNC flag set. 161 * 162 * Called with the socket buffer lock held; will release the lock by the end 163 * of the function. This allows the caller to acquire the socket buffer lock 164 * while testing for the need for various sorts of wakeup and hold it through 165 * to the point where it's no longer required. We currently hold the lock 166 * through calls out to other subsystems (with the exception of kqueue), and 167 * then release it to avoid lock order issues. It's not clear that's 168 * correct. 169 */ 170void 171sowakeup(struct socket *so, struct sockbuf *sb) 172{ 173 174 SOCKBUF_LOCK_ASSERT(sb); 175 176 selwakeuppri(&sb->sb_sel, PSOCK); 177 sb->sb_flags &= ~SB_SEL; 178 if (sb->sb_flags & SB_WAIT) { 179 sb->sb_flags &= ~SB_WAIT; 180 wakeup(&sb->sb_cc); 181 } 182 KNOTE_LOCKED(&sb->sb_sel.si_note, 0); 183 SOCKBUF_UNLOCK(sb); 184 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) 185 pgsigio(&so->so_sigio, SIGIO, 0); 186 if (sb->sb_flags & SB_UPCALL) 187 (*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT); 188 if (sb->sb_flags & SB_AIO) 189 aio_swake(so, sb); 190 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 191} 192 193/* 194 * Socket buffer (struct sockbuf) utility routines. 195 * 196 * Each socket contains two socket buffers: one for sending data and one for 197 * receiving data. Each buffer contains a queue of mbufs, information about 198 * the number of mbufs and amount of data in the queue, and other fields 199 * allowing select() statements and notification on data availability to be 200 * implemented. 201 * 202 * Data stored in a socket buffer is maintained as a list of records. Each 203 * record is a list of mbufs chained together with the m_next field. Records 204 * are chained together with the m_nextpkt field. The upper level routine 205 * soreceive() expects the following conventions to be observed when placing 206 * information in the receive buffer: 207 * 208 * 1. If the protocol requires each message be preceded by the sender's name, 209 * then a record containing that name must be present before any 210 * associated data (mbuf's must be of type MT_SONAME). 211 * 2. If the protocol supports the exchange of ``access rights'' (really just 212 * additional data associated with the message), and there are ``rights'' 213 * to be received, then a record containing this data should be present 214 * (mbuf's must be of type MT_RIGHTS). 215 * 3. If a name or rights record exists, then it must be followed by a data 216 * record, perhaps of zero length. 217 * 218 * Before using a new socket structure it is first necessary to reserve 219 * buffer space to the socket, by calling sbreserve(). This should commit 220 * some of the available buffer space in the system buffer pool for the 221 * socket (currently, it does nothing but enforce limits). The space should 222 * be released by calling sbrelease() when the socket is destroyed. 223 */ 224int 225soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 226{ 227 struct thread *td = curthread; 228 229 SOCKBUF_LOCK(&so->so_snd); 230 SOCKBUF_LOCK(&so->so_rcv); 231 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 232 goto bad; 233 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 234 goto bad2; 235 if (so->so_rcv.sb_lowat == 0) 236 so->so_rcv.sb_lowat = 1; 237 if (so->so_snd.sb_lowat == 0) 238 so->so_snd.sb_lowat = MCLBYTES; 239 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 240 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 241 SOCKBUF_UNLOCK(&so->so_rcv); 242 SOCKBUF_UNLOCK(&so->so_snd); 243 return (0); 244bad2: 245 sbrelease_locked(&so->so_snd, so); 246bad: 247 SOCKBUF_UNLOCK(&so->so_rcv); 248 SOCKBUF_UNLOCK(&so->so_snd); 249 return (ENOBUFS); 250} 251 252static int 253sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 254{ 255 int error = 0; 256 u_long tmp_sb_max = sb_max; 257 258 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 259 if (error || !req->newptr) 260 return (error); 261 if (tmp_sb_max < MSIZE + MCLBYTES) 262 return (EINVAL); 263 sb_max = tmp_sb_max; 264 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 265 return (0); 266} 267 268/* 269 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 270 * become limiting if buffering efficiency is near the normal case. 271 */ 272int 273sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 274 struct thread *td) 275{ 276 rlim_t sbsize_limit; 277 278 SOCKBUF_LOCK_ASSERT(sb); 279 280 /* 281 * td will only be NULL when we're in an interrupt (e.g. in 282 * tcp_input()). 283 * 284 * XXXRW: This comment needs updating, as might the code. 285 */ 286 if (cc > sb_max_adj) 287 return (0); 288 if (td != NULL) { 289 PROC_LOCK(td->td_proc); 290 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 291 PROC_UNLOCK(td->td_proc); 292 } else 293 sbsize_limit = RLIM_INFINITY; 294 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 295 sbsize_limit)) 296 return (0); 297 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 298 if (sb->sb_lowat > sb->sb_hiwat) 299 sb->sb_lowat = sb->sb_hiwat; 300 return (1); 301} 302 303int 304sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 305 struct thread *td) 306{ 307 int error; 308 309 SOCKBUF_LOCK(sb); 310 error = sbreserve_locked(sb, cc, so, td); 311 SOCKBUF_UNLOCK(sb); 312 return (error); 313} 314 315/* 316 * Free mbufs held by a socket, and reserved mbuf space. 317 */ 318static void 319sbrelease_internal(struct sockbuf *sb, struct socket *so) 320{ 321 322 sbflush_internal(sb); 323 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 324 RLIM_INFINITY); 325 sb->sb_mbmax = 0; 326} 327 328void 329sbrelease_locked(struct sockbuf *sb, struct socket *so) 330{ 331 332 SOCKBUF_LOCK_ASSERT(sb); 333 334 sbrelease_internal(sb, so); 335} 336 337void 338sbrelease(struct sockbuf *sb, struct socket *so) 339{ 340 341 SOCKBUF_LOCK(sb); 342 sbrelease_locked(sb, so); 343 SOCKBUF_UNLOCK(sb); 344} 345 346void 347sbdestroy(struct sockbuf *sb, struct socket *so) 348{ 349 350 sbrelease_internal(sb, so); 351} 352 353 354/* 355 * Routines to add and remove data from an mbuf queue. 356 * 357 * The routines sbappend() or sbappendrecord() are normally called to append 358 * new mbufs to a socket buffer, after checking that adequate space is 359 * available, comparing the function sbspace() with the amount of data to be 360 * added. sbappendrecord() differs from sbappend() in that data supplied is 361 * treated as the beginning of a new record. To place a sender's address, 362 * optional access rights, and data in a socket receive buffer, 363 * sbappendaddr() should be used. To place access rights and data in a 364 * socket receive buffer, sbappendrights() should be used. In either case, 365 * the new data begins a new record. Note that unlike sbappend() and 366 * sbappendrecord(), these routines check for the caller that there will be 367 * enough space to store the data. Each fails if there is not enough space, 368 * or if it cannot find mbufs to store additional information in. 369 * 370 * Reliable protocols may use the socket send buffer to hold data awaiting 371 * acknowledgement. Data is normally copied from a socket send buffer in a 372 * protocol with m_copy for output to a peer, and then removing the data from 373 * the socket buffer with sbdrop() or sbdroprecord() when the data is 374 * acknowledged by the peer. 375 */ 376#ifdef SOCKBUF_DEBUG 377void 378sblastrecordchk(struct sockbuf *sb, const char *file, int line) 379{ 380 struct mbuf *m = sb->sb_mb; 381 382 SOCKBUF_LOCK_ASSERT(sb); 383 384 while (m && m->m_nextpkt) 385 m = m->m_nextpkt; 386 387 if (m != sb->sb_lastrecord) { 388 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 389 __func__, sb->sb_mb, sb->sb_lastrecord, m); 390 printf("packet chain:\n"); 391 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 392 printf("\t%p\n", m); 393 panic("%s from %s:%u", __func__, file, line); 394 } 395} 396 397void 398sblastmbufchk(struct sockbuf *sb, const char *file, int line) 399{ 400 struct mbuf *m = sb->sb_mb; 401 struct mbuf *n; 402 403 SOCKBUF_LOCK_ASSERT(sb); 404 405 while (m && m->m_nextpkt) 406 m = m->m_nextpkt; 407 408 while (m && m->m_next) 409 m = m->m_next; 410 411 if (m != sb->sb_mbtail) { 412 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 413 __func__, sb->sb_mb, sb->sb_mbtail, m); 414 printf("packet tree:\n"); 415 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 416 printf("\t"); 417 for (n = m; n != NULL; n = n->m_next) 418 printf("%p ", n); 419 printf("\n"); 420 } 421 panic("%s from %s:%u", __func__, file, line); 422 } 423} 424#endif /* SOCKBUF_DEBUG */ 425 426#define SBLINKRECORD(sb, m0) do { \ 427 SOCKBUF_LOCK_ASSERT(sb); \ 428 if ((sb)->sb_lastrecord != NULL) \ 429 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 430 else \ 431 (sb)->sb_mb = (m0); \ 432 (sb)->sb_lastrecord = (m0); \ 433} while (/*CONSTCOND*/0) 434 435/* 436 * Append mbuf chain m to the last record in the socket buffer sb. The 437 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 438 * are discarded and mbufs are compacted where possible. 439 */ 440void 441sbappend_locked(struct sockbuf *sb, struct mbuf *m) 442{ 443 struct mbuf *n; 444 445 SOCKBUF_LOCK_ASSERT(sb); 446 447 if (m == 0) 448 return; 449 450 SBLASTRECORDCHK(sb); 451 n = sb->sb_mb; 452 if (n) { 453 while (n->m_nextpkt) 454 n = n->m_nextpkt; 455 do { 456 if (n->m_flags & M_EOR) { 457 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 458 return; 459 } 460 } while (n->m_next && (n = n->m_next)); 461 } else { 462 /* 463 * XXX Would like to simply use sb_mbtail here, but 464 * XXX I need to verify that I won't miss an EOR that 465 * XXX way. 466 */ 467 if ((n = sb->sb_lastrecord) != NULL) { 468 do { 469 if (n->m_flags & M_EOR) { 470 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 471 return; 472 } 473 } while (n->m_next && (n = n->m_next)); 474 } else { 475 /* 476 * If this is the first record in the socket buffer, 477 * it's also the last record. 478 */ 479 sb->sb_lastrecord = m; 480 } 481 } 482 sbcompress(sb, m, n); 483 SBLASTRECORDCHK(sb); 484} 485 486/* 487 * Append mbuf chain m to the last record in the socket buffer sb. The 488 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 489 * are discarded and mbufs are compacted where possible. 490 */ 491void 492sbappend(struct sockbuf *sb, struct mbuf *m) 493{ 494 495 SOCKBUF_LOCK(sb); 496 sbappend_locked(sb, m); 497 SOCKBUF_UNLOCK(sb); 498} 499 500/* 501 * This version of sbappend() should only be used when the caller absolutely 502 * knows that there will never be more than one record in the socket buffer, 503 * that is, a stream protocol (such as TCP). 504 */ 505void 506sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 507{ 508 SOCKBUF_LOCK_ASSERT(sb); 509 510 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 511 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 512 513 SBLASTMBUFCHK(sb); 514 515 sbcompress(sb, m, sb->sb_mbtail); 516 517 sb->sb_lastrecord = sb->sb_mb; 518 SBLASTRECORDCHK(sb); 519} 520 521/* 522 * This version of sbappend() should only be used when the caller absolutely 523 * knows that there will never be more than one record in the socket buffer, 524 * that is, a stream protocol (such as TCP). 525 */ 526void 527sbappendstream(struct sockbuf *sb, struct mbuf *m) 528{ 529 530 SOCKBUF_LOCK(sb); 531 sbappendstream_locked(sb, m); 532 SOCKBUF_UNLOCK(sb); 533} 534 535#ifdef SOCKBUF_DEBUG 536void 537sbcheck(struct sockbuf *sb) 538{ 539 struct mbuf *m; 540 struct mbuf *n = 0; 541 u_long len = 0, mbcnt = 0; 542 543 SOCKBUF_LOCK_ASSERT(sb); 544 545 for (m = sb->sb_mb; m; m = n) { 546 n = m->m_nextpkt; 547 for (; m; m = m->m_next) { 548 len += m->m_len; 549 mbcnt += MSIZE; 550 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 551 mbcnt += m->m_ext.ext_size; 552 } 553 } 554 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 555 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 556 mbcnt, sb->sb_mbcnt); 557 panic("sbcheck"); 558 } 559} 560#endif 561 562/* 563 * As above, except the mbuf chain begins a new record. 564 */ 565void 566sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 567{ 568 struct mbuf *m; 569 570 SOCKBUF_LOCK_ASSERT(sb); 571 572 if (m0 == 0) 573 return; 574 m = sb->sb_mb; 575 if (m) 576 while (m->m_nextpkt) 577 m = m->m_nextpkt; 578 /* 579 * Put the first mbuf on the queue. Note this permits zero length 580 * records. 581 */ 582 sballoc(sb, m0); 583 SBLASTRECORDCHK(sb); 584 SBLINKRECORD(sb, m0); 585 if (m) 586 m->m_nextpkt = m0; 587 else 588 sb->sb_mb = m0; 589 m = m0->m_next; 590 m0->m_next = 0; 591 if (m && (m0->m_flags & M_EOR)) { 592 m0->m_flags &= ~M_EOR; 593 m->m_flags |= M_EOR; 594 } 595 sbcompress(sb, m, m0); 596} 597 598/* 599 * As above, except the mbuf chain begins a new record. 600 */ 601void 602sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 603{ 604 605 SOCKBUF_LOCK(sb); 606 sbappendrecord_locked(sb, m0); 607 SOCKBUF_UNLOCK(sb); 608} 609 610/* 611 * Append address and data, and optionally, control (ancillary) data to the 612 * receive queue of a socket. If present, m0 must include a packet header 613 * with total length. Returns 0 if no space in sockbuf or insufficient 614 * mbufs. 615 */ 616int 617sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 618 struct mbuf *m0, struct mbuf *control) 619{ 620 struct mbuf *m, *n, *nlast; 621 int space = asa->sa_len; 622 623 SOCKBUF_LOCK_ASSERT(sb); 624 625 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 626 panic("sbappendaddr_locked"); 627 if (m0) 628 space += m0->m_pkthdr.len; 629 space += m_length(control, &n); 630 631 if (space > sbspace(sb)) 632 return (0); 633#if MSIZE <= 256 634 if (asa->sa_len > MLEN) 635 return (0); 636#endif 637 MGET(m, M_DONTWAIT, MT_SONAME); 638 if (m == 0) 639 return (0); 640 m->m_len = asa->sa_len; 641 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 642 if (n) 643 n->m_next = m0; /* concatenate data to control */ 644 else 645 control = m0; 646 m->m_next = control; 647 for (n = m; n->m_next != NULL; n = n->m_next) 648 sballoc(sb, n); 649 sballoc(sb, n); 650 nlast = n; 651 SBLINKRECORD(sb, m); 652 653 sb->sb_mbtail = nlast; 654 SBLASTMBUFCHK(sb); 655 656 SBLASTRECORDCHK(sb); 657 return (1); 658} 659 660/* 661 * Append address and data, and optionally, control (ancillary) data to the 662 * receive queue of a socket. If present, m0 must include a packet header 663 * with total length. Returns 0 if no space in sockbuf or insufficient 664 * mbufs. 665 */ 666int 667sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 668 struct mbuf *m0, struct mbuf *control) 669{ 670 int retval; 671 672 SOCKBUF_LOCK(sb); 673 retval = sbappendaddr_locked(sb, asa, m0, control); 674 SOCKBUF_UNLOCK(sb); 675 return (retval); 676} 677 678int 679sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 680 struct mbuf *control) 681{ 682 struct mbuf *m, *n, *mlast; 683 int space; 684 685 SOCKBUF_LOCK_ASSERT(sb); 686 687 if (control == 0) 688 panic("sbappendcontrol_locked"); 689 space = m_length(control, &n) + m_length(m0, NULL); 690 691 if (space > sbspace(sb)) 692 return (0); 693 n->m_next = m0; /* concatenate data to control */ 694 695 SBLASTRECORDCHK(sb); 696 697 for (m = control; m->m_next; m = m->m_next) 698 sballoc(sb, m); 699 sballoc(sb, m); 700 mlast = m; 701 SBLINKRECORD(sb, control); 702 703 sb->sb_mbtail = mlast; 704 SBLASTMBUFCHK(sb); 705 706 SBLASTRECORDCHK(sb); 707 return (1); 708} 709 710int 711sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 712{ 713 int retval; 714 715 SOCKBUF_LOCK(sb); 716 retval = sbappendcontrol_locked(sb, m0, control); 717 SOCKBUF_UNLOCK(sb); 718 return (retval); 719} 720 721/* 722 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 723 * (n). If (n) is NULL, the buffer is presumed empty. 724 * 725 * When the data is compressed, mbufs in the chain may be handled in one of 726 * three ways: 727 * 728 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 729 * record boundary, and no change in data type). 730 * 731 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 732 * an mbuf already in the socket buffer. This can occur if an 733 * appropriate mbuf exists, there is room, and no merging of data types 734 * will occur. 735 * 736 * (3) The mbuf may be appended to the end of the existing mbuf chain. 737 * 738 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 739 * end-of-record. 740 */ 741void 742sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 743{ 744 int eor = 0; 745 struct mbuf *o; 746 747 SOCKBUF_LOCK_ASSERT(sb); 748 749 while (m) { 750 eor |= m->m_flags & M_EOR; 751 if (m->m_len == 0 && 752 (eor == 0 || 753 (((o = m->m_next) || (o = n)) && 754 o->m_type == m->m_type))) { 755 if (sb->sb_lastrecord == m) 756 sb->sb_lastrecord = m->m_next; 757 m = m_free(m); 758 continue; 759 } 760 if (n && (n->m_flags & M_EOR) == 0 && 761 M_WRITABLE(n) && 762 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 763 m->m_len <= M_TRAILINGSPACE(n) && 764 n->m_type == m->m_type) { 765 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 766 (unsigned)m->m_len); 767 n->m_len += m->m_len; 768 sb->sb_cc += m->m_len; 769 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 770 /* XXX: Probably don't need.*/ 771 sb->sb_ctl += m->m_len; 772 m = m_free(m); 773 continue; 774 } 775 if (n) 776 n->m_next = m; 777 else 778 sb->sb_mb = m; 779 sb->sb_mbtail = m; 780 sballoc(sb, m); 781 n = m; 782 m->m_flags &= ~M_EOR; 783 m = m->m_next; 784 n->m_next = 0; 785 } 786 if (eor) { 787 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 788 n->m_flags |= eor; 789 } 790 SBLASTMBUFCHK(sb); 791} 792 793/* 794 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 795 */ 796static void 797sbflush_internal(struct sockbuf *sb) 798{ 799 800 if (sb->sb_flags & SB_LOCK) 801 panic("sbflush_internal: locked"); 802 while (sb->sb_mbcnt) { 803 /* 804 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 805 * we would loop forever. Panic instead. 806 */ 807 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 808 break; 809 sbdrop_internal(sb, (int)sb->sb_cc); 810 } 811 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) 812 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 813 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); 814} 815 816void 817sbflush_locked(struct sockbuf *sb) 818{ 819 820 SOCKBUF_LOCK_ASSERT(sb); 821 sbflush_internal(sb); 822} 823 824void 825sbflush(struct sockbuf *sb) 826{ 827 828 SOCKBUF_LOCK(sb); 829 sbflush_locked(sb); 830 SOCKBUF_UNLOCK(sb); 831} 832 833/* 834 * Drop data from (the front of) a sockbuf. 835 */ 836static void 837sbdrop_internal(struct sockbuf *sb, int len) 838{ 839 struct mbuf *m; 840 struct mbuf *next; 841 842 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 843 while (len > 0) { 844 if (m == 0) { 845 if (next == 0) 846 panic("sbdrop"); 847 m = next; 848 next = m->m_nextpkt; 849 continue; 850 } 851 if (m->m_len > len) { 852 m->m_len -= len; 853 m->m_data += len; 854 sb->sb_cc -= len; 855 if (sb->sb_sndptroff != 0) 856 sb->sb_sndptroff -= len; 857 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 858 sb->sb_ctl -= len; 859 break; 860 } 861 len -= m->m_len; 862 sbfree(sb, m); 863 m = m_free(m); 864 } 865 while (m && m->m_len == 0) { 866 sbfree(sb, m); 867 m = m_free(m); 868 } 869 if (m) { 870 sb->sb_mb = m; 871 m->m_nextpkt = next; 872 } else 873 sb->sb_mb = next; 874 /* 875 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 876 * sb_lastrecord is up-to-date if we dropped part of the last record. 877 */ 878 m = sb->sb_mb; 879 if (m == NULL) { 880 sb->sb_mbtail = NULL; 881 sb->sb_lastrecord = NULL; 882 } else if (m->m_nextpkt == NULL) { 883 sb->sb_lastrecord = m; 884 } 885} 886 887/* 888 * Drop data from (the front of) a sockbuf. 889 */ 890void 891sbdrop_locked(struct sockbuf *sb, int len) 892{ 893 894 SOCKBUF_LOCK_ASSERT(sb); 895 896 sbdrop_internal(sb, len); 897} 898 899void 900sbdrop(struct sockbuf *sb, int len) 901{ 902 903 SOCKBUF_LOCK(sb); 904 sbdrop_locked(sb, len); 905 SOCKBUF_UNLOCK(sb); 906} 907 908 909/* 910 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 911 * avoid traversal of the entire socket buffer for larger offsets. 912 */ 913struct mbuf * 914sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 915{ 916 struct mbuf *m, *ret; 917 918 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 919 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); 920 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); 921 922 /* 923 * Is off below stored offset? Happens on retransmits. 924 * Just return, we can't help here. 925 */ 926 if (sb->sb_sndptroff > off) { 927 *moff = off; 928 return (sb->sb_mb); 929 } 930 931 /* Return closest mbuf in chain for current offset. */ 932 *moff = off - sb->sb_sndptroff; 933 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 934 935 /* Advance by len to be as close as possible for the next transmit. */ 936 for (off = off - sb->sb_sndptroff + len - 1; 937 off > 0 && off >= m->m_len; 938 m = m->m_next) { 939 sb->sb_sndptroff += m->m_len; 940 off -= m->m_len; 941 } 942 sb->sb_sndptr = m; 943 944 return (ret); 945} 946 947/* 948 * Drop a record off the front of a sockbuf and move the next record to the 949 * front. 950 */ 951void 952sbdroprecord_locked(struct sockbuf *sb) 953{ 954 struct mbuf *m; 955 956 SOCKBUF_LOCK_ASSERT(sb); 957 958 m = sb->sb_mb; 959 if (m) { 960 sb->sb_mb = m->m_nextpkt; 961 do { 962 sbfree(sb, m); 963 m = m_free(m); 964 } while (m); 965 } 966 SB_EMPTY_FIXUP(sb); 967} 968 969/* 970 * Drop a record off the front of a sockbuf and move the next record to the 971 * front. 972 */ 973void 974sbdroprecord(struct sockbuf *sb) 975{ 976 977 SOCKBUF_LOCK(sb); 978 sbdroprecord_locked(sb); 979 SOCKBUF_UNLOCK(sb); 980} 981
| 34 35#include "opt_param.h" 36 37#include <sys/param.h> 38#include <sys/aio.h> /* for aio_swake proto */ 39#include <sys/kernel.h> 40#include <sys/lock.h> 41#include <sys/mbuf.h> 42#include <sys/mutex.h> 43#include <sys/proc.h> 44#include <sys/protosw.h> 45#include <sys/resourcevar.h> 46#include <sys/signalvar.h> 47#include <sys/socket.h> 48#include <sys/socketvar.h> 49#include <sys/sysctl.h> 50 51/* 52 * Function pointer set by the AIO routines so that the socket buffer code 53 * can call back into the AIO module if it is loaded. 54 */ 55void (*aio_swake)(struct socket *, struct sockbuf *); 56 57/* 58 * Primitive routines for operating on socket buffers 59 */ 60 61u_long sb_max = SB_MAX; 62static u_long sb_max_adj = 63 SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 64 65static u_long sb_efficiency = 8; /* parameter for sbreserve() */ 66 67static void sbdrop_internal(struct sockbuf *sb, int len); 68static void sbflush_internal(struct sockbuf *sb); 69static void sbrelease_internal(struct sockbuf *sb, struct socket *so); 70 71/* 72 * Socantsendmore indicates that no more data will be sent on the socket; it 73 * would normally be applied to a socket when the user informs the system 74 * that no more data is to be sent, by the protocol code (in case 75 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 76 * received, and will normally be applied to the socket by a protocol when it 77 * detects that the peer will send no more data. Data queued for reading in 78 * the socket may yet be read. 79 */ 80void 81socantsendmore_locked(struct socket *so) 82{ 83 84 SOCKBUF_LOCK_ASSERT(&so->so_snd); 85 86 so->so_snd.sb_state |= SBS_CANTSENDMORE; 87 sowwakeup_locked(so); 88 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 89} 90 91void 92socantsendmore(struct socket *so) 93{ 94 95 SOCKBUF_LOCK(&so->so_snd); 96 socantsendmore_locked(so); 97 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 98} 99 100void 101socantrcvmore_locked(struct socket *so) 102{ 103 104 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 105 106 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 107 sorwakeup_locked(so); 108 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 109} 110 111void 112socantrcvmore(struct socket *so) 113{ 114 115 SOCKBUF_LOCK(&so->so_rcv); 116 socantrcvmore_locked(so); 117 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 118} 119 120/* 121 * Wait for data to arrive at/drain from a socket buffer. 122 */ 123int 124sbwait(struct sockbuf *sb) 125{ 126 127 SOCKBUF_LOCK_ASSERT(sb); 128 129 sb->sb_flags |= SB_WAIT; 130 return (msleep(&sb->sb_cc, &sb->sb_mtx, 131 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 132 sb->sb_timeo)); 133} 134 135/* 136 * Lock a sockbuf already known to be locked; return any error returned from 137 * sleep (EINTR). 138 */ 139int 140sb_lock(struct sockbuf *sb) 141{ 142 int error; 143 144 SOCKBUF_LOCK_ASSERT(sb); 145 146 while (sb->sb_flags & SB_LOCK) { 147 sb->sb_flags |= SB_WANT; 148 error = msleep(&sb->sb_flags, &sb->sb_mtx, 149 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH, 150 "sblock", 0); 151 if (error) 152 return (error); 153 } 154 sb->sb_flags |= SB_LOCK; 155 return (0); 156} 157 158/* 159 * Wakeup processes waiting on a socket buffer. Do asynchronous notification 160 * via SIGIO if the socket has the SS_ASYNC flag set. 161 * 162 * Called with the socket buffer lock held; will release the lock by the end 163 * of the function. This allows the caller to acquire the socket buffer lock 164 * while testing for the need for various sorts of wakeup and hold it through 165 * to the point where it's no longer required. We currently hold the lock 166 * through calls out to other subsystems (with the exception of kqueue), and 167 * then release it to avoid lock order issues. It's not clear that's 168 * correct. 169 */ 170void 171sowakeup(struct socket *so, struct sockbuf *sb) 172{ 173 174 SOCKBUF_LOCK_ASSERT(sb); 175 176 selwakeuppri(&sb->sb_sel, PSOCK); 177 sb->sb_flags &= ~SB_SEL; 178 if (sb->sb_flags & SB_WAIT) { 179 sb->sb_flags &= ~SB_WAIT; 180 wakeup(&sb->sb_cc); 181 } 182 KNOTE_LOCKED(&sb->sb_sel.si_note, 0); 183 SOCKBUF_UNLOCK(sb); 184 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) 185 pgsigio(&so->so_sigio, SIGIO, 0); 186 if (sb->sb_flags & SB_UPCALL) 187 (*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT); 188 if (sb->sb_flags & SB_AIO) 189 aio_swake(so, sb); 190 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 191} 192 193/* 194 * Socket buffer (struct sockbuf) utility routines. 195 * 196 * Each socket contains two socket buffers: one for sending data and one for 197 * receiving data. Each buffer contains a queue of mbufs, information about 198 * the number of mbufs and amount of data in the queue, and other fields 199 * allowing select() statements and notification on data availability to be 200 * implemented. 201 * 202 * Data stored in a socket buffer is maintained as a list of records. Each 203 * record is a list of mbufs chained together with the m_next field. Records 204 * are chained together with the m_nextpkt field. The upper level routine 205 * soreceive() expects the following conventions to be observed when placing 206 * information in the receive buffer: 207 * 208 * 1. If the protocol requires each message be preceded by the sender's name, 209 * then a record containing that name must be present before any 210 * associated data (mbuf's must be of type MT_SONAME). 211 * 2. If the protocol supports the exchange of ``access rights'' (really just 212 * additional data associated with the message), and there are ``rights'' 213 * to be received, then a record containing this data should be present 214 * (mbuf's must be of type MT_RIGHTS). 215 * 3. If a name or rights record exists, then it must be followed by a data 216 * record, perhaps of zero length. 217 * 218 * Before using a new socket structure it is first necessary to reserve 219 * buffer space to the socket, by calling sbreserve(). This should commit 220 * some of the available buffer space in the system buffer pool for the 221 * socket (currently, it does nothing but enforce limits). The space should 222 * be released by calling sbrelease() when the socket is destroyed. 223 */ 224int 225soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 226{ 227 struct thread *td = curthread; 228 229 SOCKBUF_LOCK(&so->so_snd); 230 SOCKBUF_LOCK(&so->so_rcv); 231 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 232 goto bad; 233 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 234 goto bad2; 235 if (so->so_rcv.sb_lowat == 0) 236 so->so_rcv.sb_lowat = 1; 237 if (so->so_snd.sb_lowat == 0) 238 so->so_snd.sb_lowat = MCLBYTES; 239 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 240 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 241 SOCKBUF_UNLOCK(&so->so_rcv); 242 SOCKBUF_UNLOCK(&so->so_snd); 243 return (0); 244bad2: 245 sbrelease_locked(&so->so_snd, so); 246bad: 247 SOCKBUF_UNLOCK(&so->so_rcv); 248 SOCKBUF_UNLOCK(&so->so_snd); 249 return (ENOBUFS); 250} 251 252static int 253sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 254{ 255 int error = 0; 256 u_long tmp_sb_max = sb_max; 257 258 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 259 if (error || !req->newptr) 260 return (error); 261 if (tmp_sb_max < MSIZE + MCLBYTES) 262 return (EINVAL); 263 sb_max = tmp_sb_max; 264 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 265 return (0); 266} 267 268/* 269 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 270 * become limiting if buffering efficiency is near the normal case. 271 */ 272int 273sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 274 struct thread *td) 275{ 276 rlim_t sbsize_limit; 277 278 SOCKBUF_LOCK_ASSERT(sb); 279 280 /* 281 * td will only be NULL when we're in an interrupt (e.g. in 282 * tcp_input()). 283 * 284 * XXXRW: This comment needs updating, as might the code. 285 */ 286 if (cc > sb_max_adj) 287 return (0); 288 if (td != NULL) { 289 PROC_LOCK(td->td_proc); 290 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 291 PROC_UNLOCK(td->td_proc); 292 } else 293 sbsize_limit = RLIM_INFINITY; 294 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 295 sbsize_limit)) 296 return (0); 297 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 298 if (sb->sb_lowat > sb->sb_hiwat) 299 sb->sb_lowat = sb->sb_hiwat; 300 return (1); 301} 302 303int 304sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 305 struct thread *td) 306{ 307 int error; 308 309 SOCKBUF_LOCK(sb); 310 error = sbreserve_locked(sb, cc, so, td); 311 SOCKBUF_UNLOCK(sb); 312 return (error); 313} 314 315/* 316 * Free mbufs held by a socket, and reserved mbuf space. 317 */ 318static void 319sbrelease_internal(struct sockbuf *sb, struct socket *so) 320{ 321 322 sbflush_internal(sb); 323 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 324 RLIM_INFINITY); 325 sb->sb_mbmax = 0; 326} 327 328void 329sbrelease_locked(struct sockbuf *sb, struct socket *so) 330{ 331 332 SOCKBUF_LOCK_ASSERT(sb); 333 334 sbrelease_internal(sb, so); 335} 336 337void 338sbrelease(struct sockbuf *sb, struct socket *so) 339{ 340 341 SOCKBUF_LOCK(sb); 342 sbrelease_locked(sb, so); 343 SOCKBUF_UNLOCK(sb); 344} 345 346void 347sbdestroy(struct sockbuf *sb, struct socket *so) 348{ 349 350 sbrelease_internal(sb, so); 351} 352 353 354/* 355 * Routines to add and remove data from an mbuf queue. 356 * 357 * The routines sbappend() or sbappendrecord() are normally called to append 358 * new mbufs to a socket buffer, after checking that adequate space is 359 * available, comparing the function sbspace() with the amount of data to be 360 * added. sbappendrecord() differs from sbappend() in that data supplied is 361 * treated as the beginning of a new record. To place a sender's address, 362 * optional access rights, and data in a socket receive buffer, 363 * sbappendaddr() should be used. To place access rights and data in a 364 * socket receive buffer, sbappendrights() should be used. In either case, 365 * the new data begins a new record. Note that unlike sbappend() and 366 * sbappendrecord(), these routines check for the caller that there will be 367 * enough space to store the data. Each fails if there is not enough space, 368 * or if it cannot find mbufs to store additional information in. 369 * 370 * Reliable protocols may use the socket send buffer to hold data awaiting 371 * acknowledgement. Data is normally copied from a socket send buffer in a 372 * protocol with m_copy for output to a peer, and then removing the data from 373 * the socket buffer with sbdrop() or sbdroprecord() when the data is 374 * acknowledged by the peer. 375 */ 376#ifdef SOCKBUF_DEBUG 377void 378sblastrecordchk(struct sockbuf *sb, const char *file, int line) 379{ 380 struct mbuf *m = sb->sb_mb; 381 382 SOCKBUF_LOCK_ASSERT(sb); 383 384 while (m && m->m_nextpkt) 385 m = m->m_nextpkt; 386 387 if (m != sb->sb_lastrecord) { 388 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 389 __func__, sb->sb_mb, sb->sb_lastrecord, m); 390 printf("packet chain:\n"); 391 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 392 printf("\t%p\n", m); 393 panic("%s from %s:%u", __func__, file, line); 394 } 395} 396 397void 398sblastmbufchk(struct sockbuf *sb, const char *file, int line) 399{ 400 struct mbuf *m = sb->sb_mb; 401 struct mbuf *n; 402 403 SOCKBUF_LOCK_ASSERT(sb); 404 405 while (m && m->m_nextpkt) 406 m = m->m_nextpkt; 407 408 while (m && m->m_next) 409 m = m->m_next; 410 411 if (m != sb->sb_mbtail) { 412 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 413 __func__, sb->sb_mb, sb->sb_mbtail, m); 414 printf("packet tree:\n"); 415 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 416 printf("\t"); 417 for (n = m; n != NULL; n = n->m_next) 418 printf("%p ", n); 419 printf("\n"); 420 } 421 panic("%s from %s:%u", __func__, file, line); 422 } 423} 424#endif /* SOCKBUF_DEBUG */ 425 426#define SBLINKRECORD(sb, m0) do { \ 427 SOCKBUF_LOCK_ASSERT(sb); \ 428 if ((sb)->sb_lastrecord != NULL) \ 429 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 430 else \ 431 (sb)->sb_mb = (m0); \ 432 (sb)->sb_lastrecord = (m0); \ 433} while (/*CONSTCOND*/0) 434 435/* 436 * Append mbuf chain m to the last record in the socket buffer sb. The 437 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 438 * are discarded and mbufs are compacted where possible. 439 */ 440void 441sbappend_locked(struct sockbuf *sb, struct mbuf *m) 442{ 443 struct mbuf *n; 444 445 SOCKBUF_LOCK_ASSERT(sb); 446 447 if (m == 0) 448 return; 449 450 SBLASTRECORDCHK(sb); 451 n = sb->sb_mb; 452 if (n) { 453 while (n->m_nextpkt) 454 n = n->m_nextpkt; 455 do { 456 if (n->m_flags & M_EOR) { 457 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 458 return; 459 } 460 } while (n->m_next && (n = n->m_next)); 461 } else { 462 /* 463 * XXX Would like to simply use sb_mbtail here, but 464 * XXX I need to verify that I won't miss an EOR that 465 * XXX way. 466 */ 467 if ((n = sb->sb_lastrecord) != NULL) { 468 do { 469 if (n->m_flags & M_EOR) { 470 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 471 return; 472 } 473 } while (n->m_next && (n = n->m_next)); 474 } else { 475 /* 476 * If this is the first record in the socket buffer, 477 * it's also the last record. 478 */ 479 sb->sb_lastrecord = m; 480 } 481 } 482 sbcompress(sb, m, n); 483 SBLASTRECORDCHK(sb); 484} 485 486/* 487 * Append mbuf chain m to the last record in the socket buffer sb. The 488 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 489 * are discarded and mbufs are compacted where possible. 490 */ 491void 492sbappend(struct sockbuf *sb, struct mbuf *m) 493{ 494 495 SOCKBUF_LOCK(sb); 496 sbappend_locked(sb, m); 497 SOCKBUF_UNLOCK(sb); 498} 499 500/* 501 * This version of sbappend() should only be used when the caller absolutely 502 * knows that there will never be more than one record in the socket buffer, 503 * that is, a stream protocol (such as TCP). 504 */ 505void 506sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 507{ 508 SOCKBUF_LOCK_ASSERT(sb); 509 510 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 511 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 512 513 SBLASTMBUFCHK(sb); 514 515 sbcompress(sb, m, sb->sb_mbtail); 516 517 sb->sb_lastrecord = sb->sb_mb; 518 SBLASTRECORDCHK(sb); 519} 520 521/* 522 * This version of sbappend() should only be used when the caller absolutely 523 * knows that there will never be more than one record in the socket buffer, 524 * that is, a stream protocol (such as TCP). 525 */ 526void 527sbappendstream(struct sockbuf *sb, struct mbuf *m) 528{ 529 530 SOCKBUF_LOCK(sb); 531 sbappendstream_locked(sb, m); 532 SOCKBUF_UNLOCK(sb); 533} 534 535#ifdef SOCKBUF_DEBUG 536void 537sbcheck(struct sockbuf *sb) 538{ 539 struct mbuf *m; 540 struct mbuf *n = 0; 541 u_long len = 0, mbcnt = 0; 542 543 SOCKBUF_LOCK_ASSERT(sb); 544 545 for (m = sb->sb_mb; m; m = n) { 546 n = m->m_nextpkt; 547 for (; m; m = m->m_next) { 548 len += m->m_len; 549 mbcnt += MSIZE; 550 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 551 mbcnt += m->m_ext.ext_size; 552 } 553 } 554 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 555 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 556 mbcnt, sb->sb_mbcnt); 557 panic("sbcheck"); 558 } 559} 560#endif 561 562/* 563 * As above, except the mbuf chain begins a new record. 564 */ 565void 566sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 567{ 568 struct mbuf *m; 569 570 SOCKBUF_LOCK_ASSERT(sb); 571 572 if (m0 == 0) 573 return; 574 m = sb->sb_mb; 575 if (m) 576 while (m->m_nextpkt) 577 m = m->m_nextpkt; 578 /* 579 * Put the first mbuf on the queue. Note this permits zero length 580 * records. 581 */ 582 sballoc(sb, m0); 583 SBLASTRECORDCHK(sb); 584 SBLINKRECORD(sb, m0); 585 if (m) 586 m->m_nextpkt = m0; 587 else 588 sb->sb_mb = m0; 589 m = m0->m_next; 590 m0->m_next = 0; 591 if (m && (m0->m_flags & M_EOR)) { 592 m0->m_flags &= ~M_EOR; 593 m->m_flags |= M_EOR; 594 } 595 sbcompress(sb, m, m0); 596} 597 598/* 599 * As above, except the mbuf chain begins a new record. 600 */ 601void 602sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 603{ 604 605 SOCKBUF_LOCK(sb); 606 sbappendrecord_locked(sb, m0); 607 SOCKBUF_UNLOCK(sb); 608} 609 610/* 611 * Append address and data, and optionally, control (ancillary) data to the 612 * receive queue of a socket. If present, m0 must include a packet header 613 * with total length. Returns 0 if no space in sockbuf or insufficient 614 * mbufs. 615 */ 616int 617sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 618 struct mbuf *m0, struct mbuf *control) 619{ 620 struct mbuf *m, *n, *nlast; 621 int space = asa->sa_len; 622 623 SOCKBUF_LOCK_ASSERT(sb); 624 625 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 626 panic("sbappendaddr_locked"); 627 if (m0) 628 space += m0->m_pkthdr.len; 629 space += m_length(control, &n); 630 631 if (space > sbspace(sb)) 632 return (0); 633#if MSIZE <= 256 634 if (asa->sa_len > MLEN) 635 return (0); 636#endif 637 MGET(m, M_DONTWAIT, MT_SONAME); 638 if (m == 0) 639 return (0); 640 m->m_len = asa->sa_len; 641 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 642 if (n) 643 n->m_next = m0; /* concatenate data to control */ 644 else 645 control = m0; 646 m->m_next = control; 647 for (n = m; n->m_next != NULL; n = n->m_next) 648 sballoc(sb, n); 649 sballoc(sb, n); 650 nlast = n; 651 SBLINKRECORD(sb, m); 652 653 sb->sb_mbtail = nlast; 654 SBLASTMBUFCHK(sb); 655 656 SBLASTRECORDCHK(sb); 657 return (1); 658} 659 660/* 661 * Append address and data, and optionally, control (ancillary) data to the 662 * receive queue of a socket. If present, m0 must include a packet header 663 * with total length. Returns 0 if no space in sockbuf or insufficient 664 * mbufs. 665 */ 666int 667sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 668 struct mbuf *m0, struct mbuf *control) 669{ 670 int retval; 671 672 SOCKBUF_LOCK(sb); 673 retval = sbappendaddr_locked(sb, asa, m0, control); 674 SOCKBUF_UNLOCK(sb); 675 return (retval); 676} 677 678int 679sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 680 struct mbuf *control) 681{ 682 struct mbuf *m, *n, *mlast; 683 int space; 684 685 SOCKBUF_LOCK_ASSERT(sb); 686 687 if (control == 0) 688 panic("sbappendcontrol_locked"); 689 space = m_length(control, &n) + m_length(m0, NULL); 690 691 if (space > sbspace(sb)) 692 return (0); 693 n->m_next = m0; /* concatenate data to control */ 694 695 SBLASTRECORDCHK(sb); 696 697 for (m = control; m->m_next; m = m->m_next) 698 sballoc(sb, m); 699 sballoc(sb, m); 700 mlast = m; 701 SBLINKRECORD(sb, control); 702 703 sb->sb_mbtail = mlast; 704 SBLASTMBUFCHK(sb); 705 706 SBLASTRECORDCHK(sb); 707 return (1); 708} 709 710int 711sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 712{ 713 int retval; 714 715 SOCKBUF_LOCK(sb); 716 retval = sbappendcontrol_locked(sb, m0, control); 717 SOCKBUF_UNLOCK(sb); 718 return (retval); 719} 720 721/* 722 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 723 * (n). If (n) is NULL, the buffer is presumed empty. 724 * 725 * When the data is compressed, mbufs in the chain may be handled in one of 726 * three ways: 727 * 728 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 729 * record boundary, and no change in data type). 730 * 731 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 732 * an mbuf already in the socket buffer. This can occur if an 733 * appropriate mbuf exists, there is room, and no merging of data types 734 * will occur. 735 * 736 * (3) The mbuf may be appended to the end of the existing mbuf chain. 737 * 738 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 739 * end-of-record. 740 */ 741void 742sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 743{ 744 int eor = 0; 745 struct mbuf *o; 746 747 SOCKBUF_LOCK_ASSERT(sb); 748 749 while (m) { 750 eor |= m->m_flags & M_EOR; 751 if (m->m_len == 0 && 752 (eor == 0 || 753 (((o = m->m_next) || (o = n)) && 754 o->m_type == m->m_type))) { 755 if (sb->sb_lastrecord == m) 756 sb->sb_lastrecord = m->m_next; 757 m = m_free(m); 758 continue; 759 } 760 if (n && (n->m_flags & M_EOR) == 0 && 761 M_WRITABLE(n) && 762 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 763 m->m_len <= M_TRAILINGSPACE(n) && 764 n->m_type == m->m_type) { 765 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 766 (unsigned)m->m_len); 767 n->m_len += m->m_len; 768 sb->sb_cc += m->m_len; 769 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 770 /* XXX: Probably don't need.*/ 771 sb->sb_ctl += m->m_len; 772 m = m_free(m); 773 continue; 774 } 775 if (n) 776 n->m_next = m; 777 else 778 sb->sb_mb = m; 779 sb->sb_mbtail = m; 780 sballoc(sb, m); 781 n = m; 782 m->m_flags &= ~M_EOR; 783 m = m->m_next; 784 n->m_next = 0; 785 } 786 if (eor) { 787 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 788 n->m_flags |= eor; 789 } 790 SBLASTMBUFCHK(sb); 791} 792 793/* 794 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 795 */ 796static void 797sbflush_internal(struct sockbuf *sb) 798{ 799 800 if (sb->sb_flags & SB_LOCK) 801 panic("sbflush_internal: locked"); 802 while (sb->sb_mbcnt) { 803 /* 804 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 805 * we would loop forever. Panic instead. 806 */ 807 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 808 break; 809 sbdrop_internal(sb, (int)sb->sb_cc); 810 } 811 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) 812 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 813 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); 814} 815 816void 817sbflush_locked(struct sockbuf *sb) 818{ 819 820 SOCKBUF_LOCK_ASSERT(sb); 821 sbflush_internal(sb); 822} 823 824void 825sbflush(struct sockbuf *sb) 826{ 827 828 SOCKBUF_LOCK(sb); 829 sbflush_locked(sb); 830 SOCKBUF_UNLOCK(sb); 831} 832 833/* 834 * Drop data from (the front of) a sockbuf. 835 */ 836static void 837sbdrop_internal(struct sockbuf *sb, int len) 838{ 839 struct mbuf *m; 840 struct mbuf *next; 841 842 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 843 while (len > 0) { 844 if (m == 0) { 845 if (next == 0) 846 panic("sbdrop"); 847 m = next; 848 next = m->m_nextpkt; 849 continue; 850 } 851 if (m->m_len > len) { 852 m->m_len -= len; 853 m->m_data += len; 854 sb->sb_cc -= len; 855 if (sb->sb_sndptroff != 0) 856 sb->sb_sndptroff -= len; 857 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 858 sb->sb_ctl -= len; 859 break; 860 } 861 len -= m->m_len; 862 sbfree(sb, m); 863 m = m_free(m); 864 } 865 while (m && m->m_len == 0) { 866 sbfree(sb, m); 867 m = m_free(m); 868 } 869 if (m) { 870 sb->sb_mb = m; 871 m->m_nextpkt = next; 872 } else 873 sb->sb_mb = next; 874 /* 875 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 876 * sb_lastrecord is up-to-date if we dropped part of the last record. 877 */ 878 m = sb->sb_mb; 879 if (m == NULL) { 880 sb->sb_mbtail = NULL; 881 sb->sb_lastrecord = NULL; 882 } else if (m->m_nextpkt == NULL) { 883 sb->sb_lastrecord = m; 884 } 885} 886 887/* 888 * Drop data from (the front of) a sockbuf. 889 */ 890void 891sbdrop_locked(struct sockbuf *sb, int len) 892{ 893 894 SOCKBUF_LOCK_ASSERT(sb); 895 896 sbdrop_internal(sb, len); 897} 898 899void 900sbdrop(struct sockbuf *sb, int len) 901{ 902 903 SOCKBUF_LOCK(sb); 904 sbdrop_locked(sb, len); 905 SOCKBUF_UNLOCK(sb); 906} 907 908 909/* 910 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 911 * avoid traversal of the entire socket buffer for larger offsets. 912 */ 913struct mbuf * 914sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 915{ 916 struct mbuf *m, *ret; 917 918 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 919 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); 920 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); 921 922 /* 923 * Is off below stored offset? Happens on retransmits. 924 * Just return, we can't help here. 925 */ 926 if (sb->sb_sndptroff > off) { 927 *moff = off; 928 return (sb->sb_mb); 929 } 930 931 /* Return closest mbuf in chain for current offset. */ 932 *moff = off - sb->sb_sndptroff; 933 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 934 935 /* Advance by len to be as close as possible for the next transmit. */ 936 for (off = off - sb->sb_sndptroff + len - 1; 937 off > 0 && off >= m->m_len; 938 m = m->m_next) { 939 sb->sb_sndptroff += m->m_len; 940 off -= m->m_len; 941 } 942 sb->sb_sndptr = m; 943 944 return (ret); 945} 946 947/* 948 * Drop a record off the front of a sockbuf and move the next record to the 949 * front. 950 */ 951void 952sbdroprecord_locked(struct sockbuf *sb) 953{ 954 struct mbuf *m; 955 956 SOCKBUF_LOCK_ASSERT(sb); 957 958 m = sb->sb_mb; 959 if (m) { 960 sb->sb_mb = m->m_nextpkt; 961 do { 962 sbfree(sb, m); 963 m = m_free(m); 964 } while (m); 965 } 966 SB_EMPTY_FIXUP(sb); 967} 968 969/* 970 * Drop a record off the front of a sockbuf and move the next record to the 971 * front. 972 */ 973void 974sbdroprecord(struct sockbuf *sb) 975{ 976 977 SOCKBUF_LOCK(sb); 978 sbdroprecord_locked(sb); 979 SOCKBUF_UNLOCK(sb); 980} 981
|