196 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 197} 198 199/* 200 * Socket buffer (struct sockbuf) utility routines. 201 * 202 * Each socket contains two socket buffers: one for sending data and one for 203 * receiving data. Each buffer contains a queue of mbufs, information about 204 * the number of mbufs and amount of data in the queue, and other fields 205 * allowing select() statements and notification on data availability to be 206 * implemented. 207 * 208 * Data stored in a socket buffer is maintained as a list of records. Each 209 * record is a list of mbufs chained together with the m_next field. Records 210 * are chained together with the m_nextpkt field. The upper level routine 211 * soreceive() expects the following conventions to be observed when placing 212 * information in the receive buffer: 213 * 214 * 1. If the protocol requires each message be preceded by the sender's name, 215 * then a record containing that name must be present before any 216 * associated data (mbuf's must be of type MT_SONAME). 217 * 2. If the protocol supports the exchange of ``access rights'' (really just 218 * additional data associated with the message), and there are ``rights'' 219 * to be received, then a record containing this data should be present 220 * (mbuf's must be of type MT_RIGHTS). 221 * 3. If a name or rights record exists, then it must be followed by a data 222 * record, perhaps of zero length. 223 * 224 * Before using a new socket structure it is first necessary to reserve 225 * buffer space to the socket, by calling sbreserve(). This should commit 226 * some of the available buffer space in the system buffer pool for the 227 * socket (currently, it does nothing but enforce limits). The space should 228 * be released by calling sbrelease() when the socket is destroyed. 229 */ 230int 231soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 232{ 233 struct thread *td = curthread; 234 235 SOCKBUF_LOCK(&so->so_snd); 236 SOCKBUF_LOCK(&so->so_rcv); 237 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 238 goto bad; 239 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 240 goto bad2; 241 if (so->so_rcv.sb_lowat == 0) 242 so->so_rcv.sb_lowat = 1; 243 if (so->so_snd.sb_lowat == 0) 244 so->so_snd.sb_lowat = MCLBYTES; 245 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 246 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 247 SOCKBUF_UNLOCK(&so->so_rcv); 248 SOCKBUF_UNLOCK(&so->so_snd); 249 return (0); 250bad2: 251 sbrelease_locked(&so->so_snd, so); 252bad: 253 SOCKBUF_UNLOCK(&so->so_rcv); 254 SOCKBUF_UNLOCK(&so->so_snd); 255 return (ENOBUFS); 256} 257 258static int 259sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 260{ 261 int error = 0; 262 u_long tmp_sb_max = sb_max; 263 264 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 265 if (error || !req->newptr) 266 return (error); 267 if (tmp_sb_max < MSIZE + MCLBYTES) 268 return (EINVAL); 269 sb_max = tmp_sb_max; 270 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 271 return (0); 272} 273 274/* 275 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 276 * become limiting if buffering efficiency is near the normal case. 277 */ 278int 279sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 280 struct thread *td) 281{ 282 rlim_t sbsize_limit; 283 284 SOCKBUF_LOCK_ASSERT(sb); 285 286 /* 287 * When a thread is passed, we take into account the thread's socket 288 * buffer size limit. The caller will generally pass curthread, but 289 * in the TCP input path, NULL will be passed to indicate that no 290 * appropriate thread resource limits are available. In that case, 291 * we don't apply a process limit. 292 */ 293 if (cc > sb_max_adj) 294 return (0); 295 if (td != NULL) { 296 PROC_LOCK(td->td_proc); 297 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 298 PROC_UNLOCK(td->td_proc); 299 } else 300 sbsize_limit = RLIM_INFINITY; 301 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 302 sbsize_limit)) 303 return (0); 304 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 305 if (sb->sb_lowat > sb->sb_hiwat) 306 sb->sb_lowat = sb->sb_hiwat; 307 return (1); 308} 309 310int 311sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 312 struct thread *td) 313{ 314 int error; 315 316 SOCKBUF_LOCK(sb); 317 error = sbreserve_locked(sb, cc, so, td); 318 SOCKBUF_UNLOCK(sb); 319 return (error); 320} 321 322/* 323 * Free mbufs held by a socket, and reserved mbuf space. 324 */ 325void 326sbrelease_internal(struct sockbuf *sb, struct socket *so) 327{ 328 329 sbflush_internal(sb); 330 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 331 RLIM_INFINITY); 332 sb->sb_mbmax = 0; 333} 334 335void 336sbrelease_locked(struct sockbuf *sb, struct socket *so) 337{ 338 339 SOCKBUF_LOCK_ASSERT(sb); 340 341 sbrelease_internal(sb, so); 342} 343 344void 345sbrelease(struct sockbuf *sb, struct socket *so) 346{ 347 348 SOCKBUF_LOCK(sb); 349 sbrelease_locked(sb, so); 350 SOCKBUF_UNLOCK(sb); 351} 352 353void 354sbdestroy(struct sockbuf *sb, struct socket *so) 355{ 356 357 sbrelease_internal(sb, so); 358} 359 360/* 361 * Routines to add and remove data from an mbuf queue. 362 * 363 * The routines sbappend() or sbappendrecord() are normally called to append 364 * new mbufs to a socket buffer, after checking that adequate space is 365 * available, comparing the function sbspace() with the amount of data to be 366 * added. sbappendrecord() differs from sbappend() in that data supplied is 367 * treated as the beginning of a new record. To place a sender's address, 368 * optional access rights, and data in a socket receive buffer, 369 * sbappendaddr() should be used. To place access rights and data in a 370 * socket receive buffer, sbappendrights() should be used. In either case, 371 * the new data begins a new record. Note that unlike sbappend() and 372 * sbappendrecord(), these routines check for the caller that there will be 373 * enough space to store the data. Each fails if there is not enough space, 374 * or if it cannot find mbufs to store additional information in. 375 * 376 * Reliable protocols may use the socket send buffer to hold data awaiting 377 * acknowledgement. Data is normally copied from a socket send buffer in a 378 * protocol with m_copy for output to a peer, and then removing the data from 379 * the socket buffer with sbdrop() or sbdroprecord() when the data is 380 * acknowledged by the peer. 381 */ 382#ifdef SOCKBUF_DEBUG 383void 384sblastrecordchk(struct sockbuf *sb, const char *file, int line) 385{ 386 struct mbuf *m = sb->sb_mb; 387 388 SOCKBUF_LOCK_ASSERT(sb); 389 390 while (m && m->m_nextpkt) 391 m = m->m_nextpkt; 392 393 if (m != sb->sb_lastrecord) { 394 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 395 __func__, sb->sb_mb, sb->sb_lastrecord, m); 396 printf("packet chain:\n"); 397 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 398 printf("\t%p\n", m); 399 panic("%s from %s:%u", __func__, file, line); 400 } 401} 402 403void 404sblastmbufchk(struct sockbuf *sb, const char *file, int line) 405{ 406 struct mbuf *m = sb->sb_mb; 407 struct mbuf *n; 408 409 SOCKBUF_LOCK_ASSERT(sb); 410 411 while (m && m->m_nextpkt) 412 m = m->m_nextpkt; 413 414 while (m && m->m_next) 415 m = m->m_next; 416 417 if (m != sb->sb_mbtail) { 418 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 419 __func__, sb->sb_mb, sb->sb_mbtail, m); 420 printf("packet tree:\n"); 421 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 422 printf("\t"); 423 for (n = m; n != NULL; n = n->m_next) 424 printf("%p ", n); 425 printf("\n"); 426 } 427 panic("%s from %s:%u", __func__, file, line); 428 } 429} 430#endif /* SOCKBUF_DEBUG */ 431 432#define SBLINKRECORD(sb, m0) do { \ 433 SOCKBUF_LOCK_ASSERT(sb); \ 434 if ((sb)->sb_lastrecord != NULL) \ 435 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 436 else \ 437 (sb)->sb_mb = (m0); \ 438 (sb)->sb_lastrecord = (m0); \ 439} while (/*CONSTCOND*/0) 440 441/* 442 * Append mbuf chain m to the last record in the socket buffer sb. The 443 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 444 * are discarded and mbufs are compacted where possible. 445 */ 446void 447sbappend_locked(struct sockbuf *sb, struct mbuf *m) 448{ 449 struct mbuf *n; 450 451 SOCKBUF_LOCK_ASSERT(sb); 452 453 if (m == 0) 454 return; 455 456 SBLASTRECORDCHK(sb); 457 n = sb->sb_mb; 458 if (n) { 459 while (n->m_nextpkt) 460 n = n->m_nextpkt; 461 do { 462 if (n->m_flags & M_EOR) { 463 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 464 return; 465 } 466 } while (n->m_next && (n = n->m_next)); 467 } else { 468 /* 469 * XXX Would like to simply use sb_mbtail here, but 470 * XXX I need to verify that I won't miss an EOR that 471 * XXX way. 472 */ 473 if ((n = sb->sb_lastrecord) != NULL) { 474 do { 475 if (n->m_flags & M_EOR) { 476 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 477 return; 478 } 479 } while (n->m_next && (n = n->m_next)); 480 } else { 481 /* 482 * If this is the first record in the socket buffer, 483 * it's also the last record. 484 */ 485 sb->sb_lastrecord = m; 486 } 487 } 488 sbcompress(sb, m, n); 489 SBLASTRECORDCHK(sb); 490} 491 492/* 493 * Append mbuf chain m to the last record in the socket buffer sb. The 494 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 495 * are discarded and mbufs are compacted where possible. 496 */ 497void 498sbappend(struct sockbuf *sb, struct mbuf *m) 499{ 500 501 SOCKBUF_LOCK(sb); 502 sbappend_locked(sb, m); 503 SOCKBUF_UNLOCK(sb); 504} 505 506/* 507 * This version of sbappend() should only be used when the caller absolutely 508 * knows that there will never be more than one record in the socket buffer, 509 * that is, a stream protocol (such as TCP). 510 */ 511void 512sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 513{ 514 SOCKBUF_LOCK_ASSERT(sb); 515 516 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 517 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 518 519 SBLASTMBUFCHK(sb); 520 521 sbcompress(sb, m, sb->sb_mbtail); 522 523 sb->sb_lastrecord = sb->sb_mb; 524 SBLASTRECORDCHK(sb); 525} 526 527/* 528 * This version of sbappend() should only be used when the caller absolutely 529 * knows that there will never be more than one record in the socket buffer, 530 * that is, a stream protocol (such as TCP). 531 */ 532void 533sbappendstream(struct sockbuf *sb, struct mbuf *m) 534{ 535 536 SOCKBUF_LOCK(sb); 537 sbappendstream_locked(sb, m); 538 SOCKBUF_UNLOCK(sb); 539} 540 541#ifdef SOCKBUF_DEBUG 542void 543sbcheck(struct sockbuf *sb) 544{ 545 struct mbuf *m; 546 struct mbuf *n = 0; 547 u_long len = 0, mbcnt = 0; 548 549 SOCKBUF_LOCK_ASSERT(sb); 550 551 for (m = sb->sb_mb; m; m = n) { 552 n = m->m_nextpkt; 553 for (; m; m = m->m_next) { 554 len += m->m_len; 555 mbcnt += MSIZE; 556 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 557 mbcnt += m->m_ext.ext_size; 558 } 559 } 560 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 561 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 562 mbcnt, sb->sb_mbcnt); 563 panic("sbcheck"); 564 } 565} 566#endif 567 568/* 569 * As above, except the mbuf chain begins a new record. 570 */ 571void 572sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 573{ 574 struct mbuf *m; 575 576 SOCKBUF_LOCK_ASSERT(sb); 577 578 if (m0 == 0) 579 return; 580 /* 581 * Put the first mbuf on the queue. Note this permits zero length 582 * records. 583 */ 584 sballoc(sb, m0); 585 SBLASTRECORDCHK(sb); 586 SBLINKRECORD(sb, m0); 587 sb->sb_mbtail = m0; 588 m = m0->m_next; 589 m0->m_next = 0; 590 if (m && (m0->m_flags & M_EOR)) { 591 m0->m_flags &= ~M_EOR; 592 m->m_flags |= M_EOR; 593 } 594 /* always call sbcompress() so it can do SBLASTMBUFCHK() */ 595 sbcompress(sb, m, m0); 596} 597 598/* 599 * As above, except the mbuf chain begins a new record. 600 */ 601void 602sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 603{ 604 605 SOCKBUF_LOCK(sb); 606 sbappendrecord_locked(sb, m0); 607 SOCKBUF_UNLOCK(sb); 608} 609 610/* 611 * Append address and data, and optionally, control (ancillary) data to the 612 * receive queue of a socket. If present, m0 must include a packet header 613 * with total length. Returns 0 if no space in sockbuf or insufficient 614 * mbufs. 615 */ 616int 617sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 618 struct mbuf *m0, struct mbuf *control) 619{ 620 struct mbuf *m, *n, *nlast; 621 int space = asa->sa_len; 622 623 SOCKBUF_LOCK_ASSERT(sb); 624 625 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 626 panic("sbappendaddr_locked"); 627 if (m0) 628 space += m0->m_pkthdr.len; 629 space += m_length(control, &n); 630 631 if (space > sbspace(sb)) 632 return (0); 633#if MSIZE <= 256 634 if (asa->sa_len > MLEN) 635 return (0); 636#endif 637 MGET(m, M_DONTWAIT, MT_SONAME); 638 if (m == 0) 639 return (0); 640 m->m_len = asa->sa_len; 641 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 642 if (n) 643 n->m_next = m0; /* concatenate data to control */ 644 else 645 control = m0; 646 m->m_next = control; 647 for (n = m; n->m_next != NULL; n = n->m_next) 648 sballoc(sb, n); 649 sballoc(sb, n); 650 nlast = n; 651 SBLINKRECORD(sb, m); 652 653 sb->sb_mbtail = nlast; 654 SBLASTMBUFCHK(sb); 655 656 SBLASTRECORDCHK(sb); 657 return (1); 658} 659 660/* 661 * Append address and data, and optionally, control (ancillary) data to the 662 * receive queue of a socket. If present, m0 must include a packet header 663 * with total length. Returns 0 if no space in sockbuf or insufficient 664 * mbufs. 665 */ 666int 667sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 668 struct mbuf *m0, struct mbuf *control) 669{ 670 int retval; 671 672 SOCKBUF_LOCK(sb); 673 retval = sbappendaddr_locked(sb, asa, m0, control); 674 SOCKBUF_UNLOCK(sb); 675 return (retval); 676} 677 678int 679sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 680 struct mbuf *control) 681{ 682 struct mbuf *m, *n, *mlast; 683 int space; 684 685 SOCKBUF_LOCK_ASSERT(sb); 686 687 if (control == 0) 688 panic("sbappendcontrol_locked"); 689 space = m_length(control, &n) + m_length(m0, NULL); 690 691 if (space > sbspace(sb)) 692 return (0); 693 n->m_next = m0; /* concatenate data to control */ 694 695 SBLASTRECORDCHK(sb); 696 697 for (m = control; m->m_next; m = m->m_next) 698 sballoc(sb, m); 699 sballoc(sb, m); 700 mlast = m; 701 SBLINKRECORD(sb, control); 702 703 sb->sb_mbtail = mlast; 704 SBLASTMBUFCHK(sb); 705 706 SBLASTRECORDCHK(sb); 707 return (1); 708} 709 710int 711sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 712{ 713 int retval; 714 715 SOCKBUF_LOCK(sb); 716 retval = sbappendcontrol_locked(sb, m0, control); 717 SOCKBUF_UNLOCK(sb); 718 return (retval); 719} 720 721/* 722 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 723 * (n). If (n) is NULL, the buffer is presumed empty. 724 * 725 * When the data is compressed, mbufs in the chain may be handled in one of 726 * three ways: 727 * 728 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 729 * record boundary, and no change in data type). 730 * 731 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 732 * an mbuf already in the socket buffer. This can occur if an 733 * appropriate mbuf exists, there is room, and no merging of data types 734 * will occur. 735 * 736 * (3) The mbuf may be appended to the end of the existing mbuf chain. 737 * 738 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 739 * end-of-record. 740 */ 741void 742sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 743{ 744 int eor = 0; 745 struct mbuf *o; 746 747 SOCKBUF_LOCK_ASSERT(sb); 748 749 while (m) { 750 eor |= m->m_flags & M_EOR; 751 if (m->m_len == 0 && 752 (eor == 0 || 753 (((o = m->m_next) || (o = n)) && 754 o->m_type == m->m_type))) { 755 if (sb->sb_lastrecord == m) 756 sb->sb_lastrecord = m->m_next; 757 m = m_free(m); 758 continue; 759 } 760 if (n && (n->m_flags & M_EOR) == 0 && 761 M_WRITABLE(n) && 762 ((sb->sb_flags & SB_NOCOALESCE) == 0) && 763 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 764 m->m_len <= M_TRAILINGSPACE(n) && 765 n->m_type == m->m_type) { 766 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 767 (unsigned)m->m_len); 768 n->m_len += m->m_len; 769 sb->sb_cc += m->m_len; 770 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 771 /* XXX: Probably don't need.*/ 772 sb->sb_ctl += m->m_len; 773 m = m_free(m); 774 continue; 775 } 776 if (n) 777 n->m_next = m; 778 else 779 sb->sb_mb = m; 780 sb->sb_mbtail = m; 781 sballoc(sb, m); 782 n = m; 783 m->m_flags &= ~M_EOR; 784 m = m->m_next; 785 n->m_next = 0; 786 } 787 if (eor) { 788 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 789 n->m_flags |= eor; 790 } 791 SBLASTMBUFCHK(sb); 792} 793 794/* 795 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 796 */ 797static void 798sbflush_internal(struct sockbuf *sb) 799{ 800 801 while (sb->sb_mbcnt) { 802 /* 803 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 804 * we would loop forever. Panic instead. 805 */ 806 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 807 break; 808 sbdrop_internal(sb, (int)sb->sb_cc); 809 } 810 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) 811 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 812 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); 813} 814 815void 816sbflush_locked(struct sockbuf *sb) 817{ 818 819 SOCKBUF_LOCK_ASSERT(sb); 820 sbflush_internal(sb); 821} 822 823void 824sbflush(struct sockbuf *sb) 825{ 826 827 SOCKBUF_LOCK(sb); 828 sbflush_locked(sb); 829 SOCKBUF_UNLOCK(sb); 830} 831 832/* 833 * Drop data from (the front of) a sockbuf. 834 */ 835static void 836sbdrop_internal(struct sockbuf *sb, int len) 837{ 838 struct mbuf *m; 839 struct mbuf *next; 840 841 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 842 while (len > 0) { 843 if (m == 0) { 844 if (next == 0) 845 panic("sbdrop"); 846 m = next; 847 next = m->m_nextpkt; 848 continue; 849 } 850 if (m->m_len > len) { 851 m->m_len -= len; 852 m->m_data += len; 853 sb->sb_cc -= len; 854 if (sb->sb_sndptroff != 0) 855 sb->sb_sndptroff -= len; 856 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 857 sb->sb_ctl -= len; 858 break; 859 } 860 len -= m->m_len; 861 sbfree(sb, m); 862 m = m_free(m); 863 } 864 while (m && m->m_len == 0) { 865 sbfree(sb, m); 866 m = m_free(m); 867 } 868 if (m) { 869 sb->sb_mb = m; 870 m->m_nextpkt = next; 871 } else 872 sb->sb_mb = next; 873 /* 874 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 875 * sb_lastrecord is up-to-date if we dropped part of the last record. 876 */ 877 m = sb->sb_mb; 878 if (m == NULL) { 879 sb->sb_mbtail = NULL; 880 sb->sb_lastrecord = NULL; 881 } else if (m->m_nextpkt == NULL) { 882 sb->sb_lastrecord = m; 883 } 884} 885 886/* 887 * Drop data from (the front of) a sockbuf. 888 */ 889void 890sbdrop_locked(struct sockbuf *sb, int len) 891{ 892 893 SOCKBUF_LOCK_ASSERT(sb); 894 895 sbdrop_internal(sb, len); 896} 897 898void 899sbdrop(struct sockbuf *sb, int len) 900{ 901 902 SOCKBUF_LOCK(sb); 903 sbdrop_locked(sb, len); 904 SOCKBUF_UNLOCK(sb); 905} 906 907/* 908 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 909 * avoid traversal of the entire socket buffer for larger offsets. 910 */ 911struct mbuf * 912sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 913{ 914 struct mbuf *m, *ret; 915 916 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 917 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); 918 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); 919 920 /* 921 * Is off below stored offset? Happens on retransmits. 922 * Just return, we can't help here. 923 */ 924 if (sb->sb_sndptroff > off) { 925 *moff = off; 926 return (sb->sb_mb); 927 } 928 929 /* Return closest mbuf in chain for current offset. */ 930 *moff = off - sb->sb_sndptroff; 931 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 932 933 /* Advance by len to be as close as possible for the next transmit. */ 934 for (off = off - sb->sb_sndptroff + len - 1; 935 off > 0 && m != NULL && off >= m->m_len; 936 m = m->m_next) { 937 sb->sb_sndptroff += m->m_len; 938 off -= m->m_len; 939 } 940 if (off > 0 && m == NULL) 941 panic("%s: sockbuf %p and mbuf %p clashing", __func__, sb, ret); 942 sb->sb_sndptr = m; 943 944 return (ret); 945} 946 947/* 948 * Drop a record off the front of a sockbuf and move the next record to the 949 * front. 950 */ 951void 952sbdroprecord_locked(struct sockbuf *sb) 953{ 954 struct mbuf *m; 955 956 SOCKBUF_LOCK_ASSERT(sb); 957 958 m = sb->sb_mb; 959 if (m) { 960 sb->sb_mb = m->m_nextpkt; 961 do { 962 sbfree(sb, m); 963 m = m_free(m); 964 } while (m); 965 } 966 SB_EMPTY_FIXUP(sb); 967} 968 969/* 970 * Drop a record off the front of a sockbuf and move the next record to the 971 * front. 972 */ 973void 974sbdroprecord(struct sockbuf *sb) 975{ 976 977 SOCKBUF_LOCK(sb); 978 sbdroprecord_locked(sb); 979 SOCKBUF_UNLOCK(sb); 980} 981 982/* 983 * Create a "control" mbuf containing the specified data with the specified 984 * type for presentation on a socket buffer. 985 */ 986struct mbuf * 987sbcreatecontrol(caddr_t p, int size, int type, int level) 988{ 989 struct cmsghdr *cp; 990 struct mbuf *m; 991 992 if (CMSG_SPACE((u_int)size) > MCLBYTES) 993 return ((struct mbuf *) NULL); 994 if (CMSG_SPACE((u_int)size) > MLEN) 995 m = m_getcl(M_DONTWAIT, MT_CONTROL, 0); 996 else 997 m = m_get(M_DONTWAIT, MT_CONTROL); 998 if (m == NULL) 999 return ((struct mbuf *) NULL); 1000 cp = mtod(m, struct cmsghdr *); 1001 m->m_len = 0; 1002 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m), 1003 ("sbcreatecontrol: short mbuf")); 1004 if (p != NULL) 1005 (void)memcpy(CMSG_DATA(cp), p, size); 1006 m->m_len = CMSG_SPACE(size); 1007 cp->cmsg_len = CMSG_LEN(size); 1008 cp->cmsg_level = level; 1009 cp->cmsg_type = type; 1010 return (m); 1011} 1012 1013/* 1014 * This does the same for socket buffers that sotoxsocket does for sockets: 1015 * generate an user-format data structure describing the socket buffer. Note 1016 * that the xsockbuf structure, since it is always embedded in a socket, does 1017 * not include a self pointer nor a length. We make this entry point public 1018 * in case some other mechanism needs it. 1019 */ 1020void 1021sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) 1022{ 1023 1024 xsb->sb_cc = sb->sb_cc; 1025 xsb->sb_hiwat = sb->sb_hiwat; 1026 xsb->sb_mbcnt = sb->sb_mbcnt; 1027 xsb->sb_mcnt = sb->sb_mcnt; 1028 xsb->sb_ccnt = sb->sb_ccnt; 1029 xsb->sb_mbmax = sb->sb_mbmax; 1030 xsb->sb_lowat = sb->sb_lowat; 1031 xsb->sb_flags = sb->sb_flags; 1032 xsb->sb_timeo = sb->sb_timeo; 1033} 1034 1035/* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 1036static int dummy; 1037SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, ""); 1038SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW, 1039 &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size"); 1040SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 1041 &sb_efficiency, 0, "");
| 206 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 207} 208 209/* 210 * Socket buffer (struct sockbuf) utility routines. 211 * 212 * Each socket contains two socket buffers: one for sending data and one for 213 * receiving data. Each buffer contains a queue of mbufs, information about 214 * the number of mbufs and amount of data in the queue, and other fields 215 * allowing select() statements and notification on data availability to be 216 * implemented. 217 * 218 * Data stored in a socket buffer is maintained as a list of records. Each 219 * record is a list of mbufs chained together with the m_next field. Records 220 * are chained together with the m_nextpkt field. The upper level routine 221 * soreceive() expects the following conventions to be observed when placing 222 * information in the receive buffer: 223 * 224 * 1. If the protocol requires each message be preceded by the sender's name, 225 * then a record containing that name must be present before any 226 * associated data (mbuf's must be of type MT_SONAME). 227 * 2. If the protocol supports the exchange of ``access rights'' (really just 228 * additional data associated with the message), and there are ``rights'' 229 * to be received, then a record containing this data should be present 230 * (mbuf's must be of type MT_RIGHTS). 231 * 3. If a name or rights record exists, then it must be followed by a data 232 * record, perhaps of zero length. 233 * 234 * Before using a new socket structure it is first necessary to reserve 235 * buffer space to the socket, by calling sbreserve(). This should commit 236 * some of the available buffer space in the system buffer pool for the 237 * socket (currently, it does nothing but enforce limits). The space should 238 * be released by calling sbrelease() when the socket is destroyed. 239 */ 240int 241soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 242{ 243 struct thread *td = curthread; 244 245 SOCKBUF_LOCK(&so->so_snd); 246 SOCKBUF_LOCK(&so->so_rcv); 247 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 248 goto bad; 249 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 250 goto bad2; 251 if (so->so_rcv.sb_lowat == 0) 252 so->so_rcv.sb_lowat = 1; 253 if (so->so_snd.sb_lowat == 0) 254 so->so_snd.sb_lowat = MCLBYTES; 255 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 256 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 257 SOCKBUF_UNLOCK(&so->so_rcv); 258 SOCKBUF_UNLOCK(&so->so_snd); 259 return (0); 260bad2: 261 sbrelease_locked(&so->so_snd, so); 262bad: 263 SOCKBUF_UNLOCK(&so->so_rcv); 264 SOCKBUF_UNLOCK(&so->so_snd); 265 return (ENOBUFS); 266} 267 268static int 269sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 270{ 271 int error = 0; 272 u_long tmp_sb_max = sb_max; 273 274 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 275 if (error || !req->newptr) 276 return (error); 277 if (tmp_sb_max < MSIZE + MCLBYTES) 278 return (EINVAL); 279 sb_max = tmp_sb_max; 280 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 281 return (0); 282} 283 284/* 285 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 286 * become limiting if buffering efficiency is near the normal case. 287 */ 288int 289sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 290 struct thread *td) 291{ 292 rlim_t sbsize_limit; 293 294 SOCKBUF_LOCK_ASSERT(sb); 295 296 /* 297 * When a thread is passed, we take into account the thread's socket 298 * buffer size limit. The caller will generally pass curthread, but 299 * in the TCP input path, NULL will be passed to indicate that no 300 * appropriate thread resource limits are available. In that case, 301 * we don't apply a process limit. 302 */ 303 if (cc > sb_max_adj) 304 return (0); 305 if (td != NULL) { 306 PROC_LOCK(td->td_proc); 307 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 308 PROC_UNLOCK(td->td_proc); 309 } else 310 sbsize_limit = RLIM_INFINITY; 311 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 312 sbsize_limit)) 313 return (0); 314 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 315 if (sb->sb_lowat > sb->sb_hiwat) 316 sb->sb_lowat = sb->sb_hiwat; 317 return (1); 318} 319 320int 321sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 322 struct thread *td) 323{ 324 int error; 325 326 SOCKBUF_LOCK(sb); 327 error = sbreserve_locked(sb, cc, so, td); 328 SOCKBUF_UNLOCK(sb); 329 return (error); 330} 331 332/* 333 * Free mbufs held by a socket, and reserved mbuf space. 334 */ 335void 336sbrelease_internal(struct sockbuf *sb, struct socket *so) 337{ 338 339 sbflush_internal(sb); 340 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 341 RLIM_INFINITY); 342 sb->sb_mbmax = 0; 343} 344 345void 346sbrelease_locked(struct sockbuf *sb, struct socket *so) 347{ 348 349 SOCKBUF_LOCK_ASSERT(sb); 350 351 sbrelease_internal(sb, so); 352} 353 354void 355sbrelease(struct sockbuf *sb, struct socket *so) 356{ 357 358 SOCKBUF_LOCK(sb); 359 sbrelease_locked(sb, so); 360 SOCKBUF_UNLOCK(sb); 361} 362 363void 364sbdestroy(struct sockbuf *sb, struct socket *so) 365{ 366 367 sbrelease_internal(sb, so); 368} 369 370/* 371 * Routines to add and remove data from an mbuf queue. 372 * 373 * The routines sbappend() or sbappendrecord() are normally called to append 374 * new mbufs to a socket buffer, after checking that adequate space is 375 * available, comparing the function sbspace() with the amount of data to be 376 * added. sbappendrecord() differs from sbappend() in that data supplied is 377 * treated as the beginning of a new record. To place a sender's address, 378 * optional access rights, and data in a socket receive buffer, 379 * sbappendaddr() should be used. To place access rights and data in a 380 * socket receive buffer, sbappendrights() should be used. In either case, 381 * the new data begins a new record. Note that unlike sbappend() and 382 * sbappendrecord(), these routines check for the caller that there will be 383 * enough space to store the data. Each fails if there is not enough space, 384 * or if it cannot find mbufs to store additional information in. 385 * 386 * Reliable protocols may use the socket send buffer to hold data awaiting 387 * acknowledgement. Data is normally copied from a socket send buffer in a 388 * protocol with m_copy for output to a peer, and then removing the data from 389 * the socket buffer with sbdrop() or sbdroprecord() when the data is 390 * acknowledged by the peer. 391 */ 392#ifdef SOCKBUF_DEBUG 393void 394sblastrecordchk(struct sockbuf *sb, const char *file, int line) 395{ 396 struct mbuf *m = sb->sb_mb; 397 398 SOCKBUF_LOCK_ASSERT(sb); 399 400 while (m && m->m_nextpkt) 401 m = m->m_nextpkt; 402 403 if (m != sb->sb_lastrecord) { 404 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 405 __func__, sb->sb_mb, sb->sb_lastrecord, m); 406 printf("packet chain:\n"); 407 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 408 printf("\t%p\n", m); 409 panic("%s from %s:%u", __func__, file, line); 410 } 411} 412 413void 414sblastmbufchk(struct sockbuf *sb, const char *file, int line) 415{ 416 struct mbuf *m = sb->sb_mb; 417 struct mbuf *n; 418 419 SOCKBUF_LOCK_ASSERT(sb); 420 421 while (m && m->m_nextpkt) 422 m = m->m_nextpkt; 423 424 while (m && m->m_next) 425 m = m->m_next; 426 427 if (m != sb->sb_mbtail) { 428 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 429 __func__, sb->sb_mb, sb->sb_mbtail, m); 430 printf("packet tree:\n"); 431 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 432 printf("\t"); 433 for (n = m; n != NULL; n = n->m_next) 434 printf("%p ", n); 435 printf("\n"); 436 } 437 panic("%s from %s:%u", __func__, file, line); 438 } 439} 440#endif /* SOCKBUF_DEBUG */ 441 442#define SBLINKRECORD(sb, m0) do { \ 443 SOCKBUF_LOCK_ASSERT(sb); \ 444 if ((sb)->sb_lastrecord != NULL) \ 445 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 446 else \ 447 (sb)->sb_mb = (m0); \ 448 (sb)->sb_lastrecord = (m0); \ 449} while (/*CONSTCOND*/0) 450 451/* 452 * Append mbuf chain m to the last record in the socket buffer sb. The 453 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 454 * are discarded and mbufs are compacted where possible. 455 */ 456void 457sbappend_locked(struct sockbuf *sb, struct mbuf *m) 458{ 459 struct mbuf *n; 460 461 SOCKBUF_LOCK_ASSERT(sb); 462 463 if (m == 0) 464 return; 465 466 SBLASTRECORDCHK(sb); 467 n = sb->sb_mb; 468 if (n) { 469 while (n->m_nextpkt) 470 n = n->m_nextpkt; 471 do { 472 if (n->m_flags & M_EOR) { 473 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 474 return; 475 } 476 } while (n->m_next && (n = n->m_next)); 477 } else { 478 /* 479 * XXX Would like to simply use sb_mbtail here, but 480 * XXX I need to verify that I won't miss an EOR that 481 * XXX way. 482 */ 483 if ((n = sb->sb_lastrecord) != NULL) { 484 do { 485 if (n->m_flags & M_EOR) { 486 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 487 return; 488 } 489 } while (n->m_next && (n = n->m_next)); 490 } else { 491 /* 492 * If this is the first record in the socket buffer, 493 * it's also the last record. 494 */ 495 sb->sb_lastrecord = m; 496 } 497 } 498 sbcompress(sb, m, n); 499 SBLASTRECORDCHK(sb); 500} 501 502/* 503 * Append mbuf chain m to the last record in the socket buffer sb. The 504 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 505 * are discarded and mbufs are compacted where possible. 506 */ 507void 508sbappend(struct sockbuf *sb, struct mbuf *m) 509{ 510 511 SOCKBUF_LOCK(sb); 512 sbappend_locked(sb, m); 513 SOCKBUF_UNLOCK(sb); 514} 515 516/* 517 * This version of sbappend() should only be used when the caller absolutely 518 * knows that there will never be more than one record in the socket buffer, 519 * that is, a stream protocol (such as TCP). 520 */ 521void 522sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 523{ 524 SOCKBUF_LOCK_ASSERT(sb); 525 526 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 527 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 528 529 SBLASTMBUFCHK(sb); 530 531 sbcompress(sb, m, sb->sb_mbtail); 532 533 sb->sb_lastrecord = sb->sb_mb; 534 SBLASTRECORDCHK(sb); 535} 536 537/* 538 * This version of sbappend() should only be used when the caller absolutely 539 * knows that there will never be more than one record in the socket buffer, 540 * that is, a stream protocol (such as TCP). 541 */ 542void 543sbappendstream(struct sockbuf *sb, struct mbuf *m) 544{ 545 546 SOCKBUF_LOCK(sb); 547 sbappendstream_locked(sb, m); 548 SOCKBUF_UNLOCK(sb); 549} 550 551#ifdef SOCKBUF_DEBUG 552void 553sbcheck(struct sockbuf *sb) 554{ 555 struct mbuf *m; 556 struct mbuf *n = 0; 557 u_long len = 0, mbcnt = 0; 558 559 SOCKBUF_LOCK_ASSERT(sb); 560 561 for (m = sb->sb_mb; m; m = n) { 562 n = m->m_nextpkt; 563 for (; m; m = m->m_next) { 564 len += m->m_len; 565 mbcnt += MSIZE; 566 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 567 mbcnt += m->m_ext.ext_size; 568 } 569 } 570 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 571 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 572 mbcnt, sb->sb_mbcnt); 573 panic("sbcheck"); 574 } 575} 576#endif 577 578/* 579 * As above, except the mbuf chain begins a new record. 580 */ 581void 582sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 583{ 584 struct mbuf *m; 585 586 SOCKBUF_LOCK_ASSERT(sb); 587 588 if (m0 == 0) 589 return; 590 /* 591 * Put the first mbuf on the queue. Note this permits zero length 592 * records. 593 */ 594 sballoc(sb, m0); 595 SBLASTRECORDCHK(sb); 596 SBLINKRECORD(sb, m0); 597 sb->sb_mbtail = m0; 598 m = m0->m_next; 599 m0->m_next = 0; 600 if (m && (m0->m_flags & M_EOR)) { 601 m0->m_flags &= ~M_EOR; 602 m->m_flags |= M_EOR; 603 } 604 /* always call sbcompress() so it can do SBLASTMBUFCHK() */ 605 sbcompress(sb, m, m0); 606} 607 608/* 609 * As above, except the mbuf chain begins a new record. 610 */ 611void 612sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 613{ 614 615 SOCKBUF_LOCK(sb); 616 sbappendrecord_locked(sb, m0); 617 SOCKBUF_UNLOCK(sb); 618} 619 620/* 621 * Append address and data, and optionally, control (ancillary) data to the 622 * receive queue of a socket. If present, m0 must include a packet header 623 * with total length. Returns 0 if no space in sockbuf or insufficient 624 * mbufs. 625 */ 626int 627sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 628 struct mbuf *m0, struct mbuf *control) 629{ 630 struct mbuf *m, *n, *nlast; 631 int space = asa->sa_len; 632 633 SOCKBUF_LOCK_ASSERT(sb); 634 635 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 636 panic("sbappendaddr_locked"); 637 if (m0) 638 space += m0->m_pkthdr.len; 639 space += m_length(control, &n); 640 641 if (space > sbspace(sb)) 642 return (0); 643#if MSIZE <= 256 644 if (asa->sa_len > MLEN) 645 return (0); 646#endif 647 MGET(m, M_DONTWAIT, MT_SONAME); 648 if (m == 0) 649 return (0); 650 m->m_len = asa->sa_len; 651 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 652 if (n) 653 n->m_next = m0; /* concatenate data to control */ 654 else 655 control = m0; 656 m->m_next = control; 657 for (n = m; n->m_next != NULL; n = n->m_next) 658 sballoc(sb, n); 659 sballoc(sb, n); 660 nlast = n; 661 SBLINKRECORD(sb, m); 662 663 sb->sb_mbtail = nlast; 664 SBLASTMBUFCHK(sb); 665 666 SBLASTRECORDCHK(sb); 667 return (1); 668} 669 670/* 671 * Append address and data, and optionally, control (ancillary) data to the 672 * receive queue of a socket. If present, m0 must include a packet header 673 * with total length. Returns 0 if no space in sockbuf or insufficient 674 * mbufs. 675 */ 676int 677sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 678 struct mbuf *m0, struct mbuf *control) 679{ 680 int retval; 681 682 SOCKBUF_LOCK(sb); 683 retval = sbappendaddr_locked(sb, asa, m0, control); 684 SOCKBUF_UNLOCK(sb); 685 return (retval); 686} 687 688int 689sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 690 struct mbuf *control) 691{ 692 struct mbuf *m, *n, *mlast; 693 int space; 694 695 SOCKBUF_LOCK_ASSERT(sb); 696 697 if (control == 0) 698 panic("sbappendcontrol_locked"); 699 space = m_length(control, &n) + m_length(m0, NULL); 700 701 if (space > sbspace(sb)) 702 return (0); 703 n->m_next = m0; /* concatenate data to control */ 704 705 SBLASTRECORDCHK(sb); 706 707 for (m = control; m->m_next; m = m->m_next) 708 sballoc(sb, m); 709 sballoc(sb, m); 710 mlast = m; 711 SBLINKRECORD(sb, control); 712 713 sb->sb_mbtail = mlast; 714 SBLASTMBUFCHK(sb); 715 716 SBLASTRECORDCHK(sb); 717 return (1); 718} 719 720int 721sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 722{ 723 int retval; 724 725 SOCKBUF_LOCK(sb); 726 retval = sbappendcontrol_locked(sb, m0, control); 727 SOCKBUF_UNLOCK(sb); 728 return (retval); 729} 730 731/* 732 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 733 * (n). If (n) is NULL, the buffer is presumed empty. 734 * 735 * When the data is compressed, mbufs in the chain may be handled in one of 736 * three ways: 737 * 738 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 739 * record boundary, and no change in data type). 740 * 741 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 742 * an mbuf already in the socket buffer. This can occur if an 743 * appropriate mbuf exists, there is room, and no merging of data types 744 * will occur. 745 * 746 * (3) The mbuf may be appended to the end of the existing mbuf chain. 747 * 748 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 749 * end-of-record. 750 */ 751void 752sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 753{ 754 int eor = 0; 755 struct mbuf *o; 756 757 SOCKBUF_LOCK_ASSERT(sb); 758 759 while (m) { 760 eor |= m->m_flags & M_EOR; 761 if (m->m_len == 0 && 762 (eor == 0 || 763 (((o = m->m_next) || (o = n)) && 764 o->m_type == m->m_type))) { 765 if (sb->sb_lastrecord == m) 766 sb->sb_lastrecord = m->m_next; 767 m = m_free(m); 768 continue; 769 } 770 if (n && (n->m_flags & M_EOR) == 0 && 771 M_WRITABLE(n) && 772 ((sb->sb_flags & SB_NOCOALESCE) == 0) && 773 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 774 m->m_len <= M_TRAILINGSPACE(n) && 775 n->m_type == m->m_type) { 776 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 777 (unsigned)m->m_len); 778 n->m_len += m->m_len; 779 sb->sb_cc += m->m_len; 780 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 781 /* XXX: Probably don't need.*/ 782 sb->sb_ctl += m->m_len; 783 m = m_free(m); 784 continue; 785 } 786 if (n) 787 n->m_next = m; 788 else 789 sb->sb_mb = m; 790 sb->sb_mbtail = m; 791 sballoc(sb, m); 792 n = m; 793 m->m_flags &= ~M_EOR; 794 m = m->m_next; 795 n->m_next = 0; 796 } 797 if (eor) { 798 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 799 n->m_flags |= eor; 800 } 801 SBLASTMBUFCHK(sb); 802} 803 804/* 805 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 806 */ 807static void 808sbflush_internal(struct sockbuf *sb) 809{ 810 811 while (sb->sb_mbcnt) { 812 /* 813 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 814 * we would loop forever. Panic instead. 815 */ 816 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 817 break; 818 sbdrop_internal(sb, (int)sb->sb_cc); 819 } 820 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) 821 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 822 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); 823} 824 825void 826sbflush_locked(struct sockbuf *sb) 827{ 828 829 SOCKBUF_LOCK_ASSERT(sb); 830 sbflush_internal(sb); 831} 832 833void 834sbflush(struct sockbuf *sb) 835{ 836 837 SOCKBUF_LOCK(sb); 838 sbflush_locked(sb); 839 SOCKBUF_UNLOCK(sb); 840} 841 842/* 843 * Drop data from (the front of) a sockbuf. 844 */ 845static void 846sbdrop_internal(struct sockbuf *sb, int len) 847{ 848 struct mbuf *m; 849 struct mbuf *next; 850 851 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 852 while (len > 0) { 853 if (m == 0) { 854 if (next == 0) 855 panic("sbdrop"); 856 m = next; 857 next = m->m_nextpkt; 858 continue; 859 } 860 if (m->m_len > len) { 861 m->m_len -= len; 862 m->m_data += len; 863 sb->sb_cc -= len; 864 if (sb->sb_sndptroff != 0) 865 sb->sb_sndptroff -= len; 866 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 867 sb->sb_ctl -= len; 868 break; 869 } 870 len -= m->m_len; 871 sbfree(sb, m); 872 m = m_free(m); 873 } 874 while (m && m->m_len == 0) { 875 sbfree(sb, m); 876 m = m_free(m); 877 } 878 if (m) { 879 sb->sb_mb = m; 880 m->m_nextpkt = next; 881 } else 882 sb->sb_mb = next; 883 /* 884 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 885 * sb_lastrecord is up-to-date if we dropped part of the last record. 886 */ 887 m = sb->sb_mb; 888 if (m == NULL) { 889 sb->sb_mbtail = NULL; 890 sb->sb_lastrecord = NULL; 891 } else if (m->m_nextpkt == NULL) { 892 sb->sb_lastrecord = m; 893 } 894} 895 896/* 897 * Drop data from (the front of) a sockbuf. 898 */ 899void 900sbdrop_locked(struct sockbuf *sb, int len) 901{ 902 903 SOCKBUF_LOCK_ASSERT(sb); 904 905 sbdrop_internal(sb, len); 906} 907 908void 909sbdrop(struct sockbuf *sb, int len) 910{ 911 912 SOCKBUF_LOCK(sb); 913 sbdrop_locked(sb, len); 914 SOCKBUF_UNLOCK(sb); 915} 916 917/* 918 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 919 * avoid traversal of the entire socket buffer for larger offsets. 920 */ 921struct mbuf * 922sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 923{ 924 struct mbuf *m, *ret; 925 926 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 927 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); 928 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); 929 930 /* 931 * Is off below stored offset? Happens on retransmits. 932 * Just return, we can't help here. 933 */ 934 if (sb->sb_sndptroff > off) { 935 *moff = off; 936 return (sb->sb_mb); 937 } 938 939 /* Return closest mbuf in chain for current offset. */ 940 *moff = off - sb->sb_sndptroff; 941 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 942 943 /* Advance by len to be as close as possible for the next transmit. */ 944 for (off = off - sb->sb_sndptroff + len - 1; 945 off > 0 && m != NULL && off >= m->m_len; 946 m = m->m_next) { 947 sb->sb_sndptroff += m->m_len; 948 off -= m->m_len; 949 } 950 if (off > 0 && m == NULL) 951 panic("%s: sockbuf %p and mbuf %p clashing", __func__, sb, ret); 952 sb->sb_sndptr = m; 953 954 return (ret); 955} 956 957/* 958 * Drop a record off the front of a sockbuf and move the next record to the 959 * front. 960 */ 961void 962sbdroprecord_locked(struct sockbuf *sb) 963{ 964 struct mbuf *m; 965 966 SOCKBUF_LOCK_ASSERT(sb); 967 968 m = sb->sb_mb; 969 if (m) { 970 sb->sb_mb = m->m_nextpkt; 971 do { 972 sbfree(sb, m); 973 m = m_free(m); 974 } while (m); 975 } 976 SB_EMPTY_FIXUP(sb); 977} 978 979/* 980 * Drop a record off the front of a sockbuf and move the next record to the 981 * front. 982 */ 983void 984sbdroprecord(struct sockbuf *sb) 985{ 986 987 SOCKBUF_LOCK(sb); 988 sbdroprecord_locked(sb); 989 SOCKBUF_UNLOCK(sb); 990} 991 992/* 993 * Create a "control" mbuf containing the specified data with the specified 994 * type for presentation on a socket buffer. 995 */ 996struct mbuf * 997sbcreatecontrol(caddr_t p, int size, int type, int level) 998{ 999 struct cmsghdr *cp; 1000 struct mbuf *m; 1001 1002 if (CMSG_SPACE((u_int)size) > MCLBYTES) 1003 return ((struct mbuf *) NULL); 1004 if (CMSG_SPACE((u_int)size) > MLEN) 1005 m = m_getcl(M_DONTWAIT, MT_CONTROL, 0); 1006 else 1007 m = m_get(M_DONTWAIT, MT_CONTROL); 1008 if (m == NULL) 1009 return ((struct mbuf *) NULL); 1010 cp = mtod(m, struct cmsghdr *); 1011 m->m_len = 0; 1012 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m), 1013 ("sbcreatecontrol: short mbuf")); 1014 if (p != NULL) 1015 (void)memcpy(CMSG_DATA(cp), p, size); 1016 m->m_len = CMSG_SPACE(size); 1017 cp->cmsg_len = CMSG_LEN(size); 1018 cp->cmsg_level = level; 1019 cp->cmsg_type = type; 1020 return (m); 1021} 1022 1023/* 1024 * This does the same for socket buffers that sotoxsocket does for sockets: 1025 * generate an user-format data structure describing the socket buffer. Note 1026 * that the xsockbuf structure, since it is always embedded in a socket, does 1027 * not include a self pointer nor a length. We make this entry point public 1028 * in case some other mechanism needs it. 1029 */ 1030void 1031sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) 1032{ 1033 1034 xsb->sb_cc = sb->sb_cc; 1035 xsb->sb_hiwat = sb->sb_hiwat; 1036 xsb->sb_mbcnt = sb->sb_mbcnt; 1037 xsb->sb_mcnt = sb->sb_mcnt; 1038 xsb->sb_ccnt = sb->sb_ccnt; 1039 xsb->sb_mbmax = sb->sb_mbmax; 1040 xsb->sb_lowat = sb->sb_lowat; 1041 xsb->sb_flags = sb->sb_flags; 1042 xsb->sb_timeo = sb->sb_timeo; 1043} 1044 1045/* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 1046static int dummy; 1047SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, ""); 1048SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW, 1049 &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size"); 1050SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 1051 &sb_efficiency, 0, "");
|