uipc_socket2.c revision 1.140
1/* $NetBSD: uipc_socket2.c,v 1.140 2021/10/02 02:07:41 thorpej Exp $ */ 2 3/*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * Copyright (c) 1982, 1986, 1988, 1990, 1993 31 * The Regents of the University of California. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. Neither the name of the University nor the names of its contributors 42 * may be used to endorse or promote products derived from this software 43 * without specific prior written permission. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * @(#)uipc_socket2.c 8.2 (Berkeley) 2/14/95 58 */ 59 60#include <sys/cdefs.h> 61__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.140 2021/10/02 02:07:41 thorpej Exp $"); 62 63#ifdef _KERNEL_OPT 64#include "opt_ddb.h" 65#include "opt_inet.h" 66#include "opt_mbuftrace.h" 67#include "opt_sb_max.h" 68#endif 69 70#include <sys/param.h> 71#include <sys/systm.h> 72#include <sys/proc.h> 73#include <sys/file.h> 74#include <sys/buf.h> 75#include <sys/mbuf.h> 76#include <sys/protosw.h> 77#include <sys/domain.h> 78#include <sys/poll.h> 79#include <sys/socket.h> 80#include <sys/socketvar.h> 81#include <sys/signalvar.h> 82#include <sys/kauth.h> 83#include <sys/pool.h> 84#include <sys/uidinfo.h> 85 86#ifdef DDB 87#include <sys/filedesc.h> 88#endif 89 90/* 91 * Primitive routines for operating on sockets and socket buffers. 92 * 93 * Connection life-cycle: 94 * 95 * Normal sequence from the active (originating) side: 96 * 97 * - soisconnecting() is called during processing of connect() call, 98 * - resulting in an eventual call to soisconnected() if/when the 99 * connection is established. 100 * 101 * When the connection is torn down during processing of disconnect(): 102 * 103 * - soisdisconnecting() is called and, 104 * - soisdisconnected() is called when the connection to the peer 105 * is totally severed. 106 * 107 * The semantics of these routines are such that connectionless protocols 108 * can call soisconnected() and soisdisconnected() only, bypassing the 109 * in-progress calls when setting up a ``connection'' takes no time. 110 * 111 * From the passive side, a socket is created with two queues of sockets: 112 * 113 * - so_q0 (0) for partial connections (i.e. connections in progress) 114 * - so_q (1) for connections already made and awaiting user acceptance. 115 * 116 * As a protocol is preparing incoming connections, it creates a socket 117 * structure queued on so_q0 by calling sonewconn(). When the connection 118 * is established, soisconnected() is called, and transfers the 119 * socket structure to so_q, making it available to accept(). 120 * 121 * If a socket is closed with sockets on either so_q0 or so_q, these 122 * sockets are dropped. 123 * 124 * Locking rules and assumptions: 125 * 126 * o socket::so_lock can change on the fly. The low level routines used 127 * to lock sockets are aware of this. When so_lock is acquired, the 128 * routine locking must check to see if so_lock still points to the 129 * lock that was acquired. If so_lock has changed in the meantime, the 130 * now irrelevant lock that was acquired must be dropped and the lock 131 * operation retried. Although not proven here, this is completely safe 132 * on a multiprocessor system, even with relaxed memory ordering, given 133 * the next two rules: 134 * 135 * o In order to mutate so_lock, the lock pointed to by the current value 136 * of so_lock must be held: i.e., the socket must be held locked by the 137 * changing thread. The thread must issue membar_exit() to prevent 138 * memory accesses being reordered, and can set so_lock to the desired 139 * value. If the lock pointed to by the new value of so_lock is not 140 * held by the changing thread, the socket must then be considered 141 * unlocked. 142 * 143 * o If so_lock is mutated, and the previous lock referred to by so_lock 144 * could still be visible to other threads in the system (e.g. via file 145 * descriptor or protocol-internal reference), then the old lock must 146 * remain valid until the socket and/or protocol control block has been 147 * torn down. 148 * 149 * o If a socket has a non-NULL so_head value (i.e. is in the process of 150 * connecting), then locking the socket must also lock the socket pointed 151 * to by so_head: their lock pointers must match. 152 * 153 * o If a socket has connections in progress (so_q, so_q0 not empty) then 154 * locking the socket must also lock the sockets attached to both queues. 155 * Again, their lock pointers must match. 156 * 157 * o Beyond the initial lock assignment in socreate(), assigning locks to 158 * sockets is the responsibility of the individual protocols / protocol 159 * domains. 160 */ 161 162static pool_cache_t socket_cache; 163u_long sb_max = SB_MAX;/* maximum socket buffer size */ 164static u_long sb_max_adj; /* adjusted sb_max */ 165 166void 167soisconnecting(struct socket *so) 168{ 169 170 KASSERT(solocked(so)); 171 172 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 173 so->so_state |= SS_ISCONNECTING; 174} 175 176void 177soisconnected(struct socket *so) 178{ 179 struct socket *head; 180 181 head = so->so_head; 182 183 KASSERT(solocked(so)); 184 KASSERT(head == NULL || solocked2(so, head)); 185 186 so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING); 187 so->so_state |= SS_ISCONNECTED; 188 if (head && so->so_onq == &head->so_q0) { 189 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 190 /* 191 * Re-enqueue and wake up any waiters, e.g. 192 * processes blocking on accept(). 193 */ 194 soqremque(so, 0); 195 soqinsque(head, so, 1); 196 sorwakeup(head); 197 cv_broadcast(&head->so_cv); 198 } else { 199 so->so_upcall = 200 head->so_accf->so_accept_filter->accf_callback; 201 so->so_upcallarg = head->so_accf->so_accept_filter_arg; 202 so->so_rcv.sb_flags |= SB_UPCALL; 203 so->so_options &= ~SO_ACCEPTFILTER; 204 (*so->so_upcall)(so, so->so_upcallarg, 205 POLLIN|POLLRDNORM, M_DONTWAIT); 206 } 207 } else { 208 cv_broadcast(&so->so_cv); 209 sorwakeup(so); 210 sowwakeup(so); 211 } 212} 213 214void 215soisdisconnecting(struct socket *so) 216{ 217 218 KASSERT(solocked(so)); 219 220 so->so_state &= ~SS_ISCONNECTING; 221 so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE); 222 cv_broadcast(&so->so_cv); 223 sowwakeup(so); 224 sorwakeup(so); 225} 226 227void 228soisdisconnected(struct socket *so) 229{ 230 231 KASSERT(solocked(so)); 232 233 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 234 so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED); 235 cv_broadcast(&so->so_cv); 236 sowwakeup(so); 237 sorwakeup(so); 238} 239 240void 241soinit2(void) 242{ 243 244 socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0, 245 "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL); 246} 247 248/* 249 * sonewconn: accept a new connection. 250 * 251 * When an attempt at a new connection is noted on a socket which accepts 252 * connections, sonewconn(9) is called. If the connection is possible 253 * (subject to space constraints, etc) then we allocate a new structure, 254 * properly linked into the data structure of the original socket. 255 * 256 * => If 'soready' is true, then socket will become ready for accept() i.e. 257 * inserted into the so_q queue, SS_ISCONNECTED set and waiters awoken. 258 * => May be called from soft-interrupt context. 259 * => Listening socket should be locked. 260 * => Returns the new socket locked. 261 */ 262struct socket * 263sonewconn(struct socket *head, bool soready) 264{ 265 struct socket *so; 266 int soqueue, error; 267 268 KASSERT(solocked(head)); 269 270 if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) { 271 /* 272 * Listen queue overflow. If there is an accept filter 273 * active, pass through the oldest cxn it's handling. 274 */ 275 if (head->so_accf == NULL) { 276 return NULL; 277 } else { 278 struct socket *so2, *next; 279 280 /* Pass the oldest connection waiting in the 281 accept filter */ 282 for (so2 = TAILQ_FIRST(&head->so_q0); 283 so2 != NULL; so2 = next) { 284 next = TAILQ_NEXT(so2, so_qe); 285 if (so2->so_upcall == NULL) { 286 continue; 287 } 288 so2->so_upcall = NULL; 289 so2->so_upcallarg = NULL; 290 so2->so_options &= ~SO_ACCEPTFILTER; 291 so2->so_rcv.sb_flags &= ~SB_UPCALL; 292 soisconnected(so2); 293 break; 294 } 295 296 /* If nothing was nudged out of the acept filter, bail 297 * out; otherwise proceed allocating the socket. */ 298 if (so2 == NULL) { 299 return NULL; 300 } 301 } 302 } 303 if ((head->so_options & SO_ACCEPTFILTER) != 0) { 304 soready = false; 305 } 306 soqueue = soready ? 1 : 0; 307 308 if ((so = soget(false)) == NULL) { 309 return NULL; 310 } 311 so->so_type = head->so_type; 312 so->so_options = head->so_options & ~SO_ACCEPTCONN; 313 so->so_linger = head->so_linger; 314 so->so_state = head->so_state | SS_NOFDREF; 315 so->so_proto = head->so_proto; 316 so->so_timeo = head->so_timeo; 317 so->so_pgid = head->so_pgid; 318 so->so_send = head->so_send; 319 so->so_receive = head->so_receive; 320 so->so_uidinfo = head->so_uidinfo; 321 so->so_egid = head->so_egid; 322 so->so_cpid = head->so_cpid; 323 324 /* 325 * Share the lock with the listening-socket, it may get unshared 326 * once the connection is complete. 327 */ 328 mutex_obj_hold(head->so_lock); 329 so->so_lock = head->so_lock; 330 331 /* 332 * Reserve the space for socket buffers. 333 */ 334#ifdef MBUFTRACE 335 so->so_mowner = head->so_mowner; 336 so->so_rcv.sb_mowner = head->so_rcv.sb_mowner; 337 so->so_snd.sb_mowner = head->so_snd.sb_mowner; 338#endif 339 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 340 goto out; 341 } 342 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 343 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 344 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 345 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 346 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 347 so->so_snd.sb_flags |= head->so_snd.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 348 349 /* 350 * Finally, perform the protocol attach. Note: a new socket 351 * lock may be assigned at this point (if so, it will be held). 352 */ 353 error = (*so->so_proto->pr_usrreqs->pr_attach)(so, 0); 354 if (error) { 355out: 356 KASSERT(solocked(so)); 357 KASSERT(so->so_accf == NULL); 358 soput(so); 359 360 /* Note: the listening socket shall stay locked. */ 361 KASSERT(solocked(head)); 362 return NULL; 363 } 364 KASSERT(solocked2(head, so)); 365 366 /* 367 * Insert into the queue. If ready, update the connection status 368 * and wake up any waiters, e.g. processes blocking on accept(). 369 */ 370 soqinsque(head, so, soqueue); 371 if (soready) { 372 so->so_state |= SS_ISCONNECTED; 373 sorwakeup(head); 374 cv_broadcast(&head->so_cv); 375 } 376 return so; 377} 378 379struct socket * 380soget(bool waitok) 381{ 382 struct socket *so; 383 384 so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT)); 385 if (__predict_false(so == NULL)) 386 return (NULL); 387 memset(so, 0, sizeof(*so)); 388 TAILQ_INIT(&so->so_q0); 389 TAILQ_INIT(&so->so_q); 390 cv_init(&so->so_cv, "socket"); 391 cv_init(&so->so_rcv.sb_cv, "netio"); 392 cv_init(&so->so_snd.sb_cv, "netio"); 393 selinit(&so->so_rcv.sb_sel); 394 selinit(&so->so_snd.sb_sel); 395 so->so_rcv.sb_so = so; 396 so->so_snd.sb_so = so; 397 return so; 398} 399 400void 401soput(struct socket *so) 402{ 403 404 KASSERT(!cv_has_waiters(&so->so_cv)); 405 KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv)); 406 KASSERT(!cv_has_waiters(&so->so_snd.sb_cv)); 407 seldestroy(&so->so_rcv.sb_sel); 408 seldestroy(&so->so_snd.sb_sel); 409 mutex_obj_free(so->so_lock); 410 cv_destroy(&so->so_cv); 411 cv_destroy(&so->so_rcv.sb_cv); 412 cv_destroy(&so->so_snd.sb_cv); 413 pool_cache_put(socket_cache, so); 414} 415 416/* 417 * soqinsque: insert socket of a new connection into the specified 418 * accept queue of the listening socket (head). 419 * 420 * q = 0: queue of partial connections 421 * q = 1: queue of incoming connections 422 */ 423void 424soqinsque(struct socket *head, struct socket *so, int q) 425{ 426 KASSERT(q == 0 || q == 1); 427 KASSERT(solocked2(head, so)); 428 KASSERT(so->so_onq == NULL); 429 KASSERT(so->so_head == NULL); 430 431 so->so_head = head; 432 if (q == 0) { 433 head->so_q0len++; 434 so->so_onq = &head->so_q0; 435 } else { 436 head->so_qlen++; 437 so->so_onq = &head->so_q; 438 } 439 TAILQ_INSERT_TAIL(so->so_onq, so, so_qe); 440} 441 442/* 443 * soqremque: remove socket from the specified queue. 444 * 445 * => Returns true if socket was removed from the specified queue. 446 * => False if socket was not removed (because it was in other queue). 447 */ 448bool 449soqremque(struct socket *so, int q) 450{ 451 struct socket *head = so->so_head; 452 453 KASSERT(q == 0 || q == 1); 454 KASSERT(solocked(so)); 455 KASSERT(so->so_onq != NULL); 456 KASSERT(head != NULL); 457 458 if (q == 0) { 459 if (so->so_onq != &head->so_q0) 460 return false; 461 head->so_q0len--; 462 } else { 463 if (so->so_onq != &head->so_q) 464 return false; 465 head->so_qlen--; 466 } 467 KASSERT(solocked2(so, head)); 468 TAILQ_REMOVE(so->so_onq, so, so_qe); 469 so->so_onq = NULL; 470 so->so_head = NULL; 471 return true; 472} 473 474/* 475 * socantsendmore: indicates that no more data will be sent on the 476 * socket; it would normally be applied to a socket when the user 477 * informs the system that no more data is to be sent, by the protocol 478 * code (in case pr_shutdown()). 479 */ 480void 481socantsendmore(struct socket *so) 482{ 483 KASSERT(solocked(so)); 484 485 so->so_state |= SS_CANTSENDMORE; 486 sowwakeup(so); 487} 488 489/* 490 * socantrcvmore(): indicates that no more data will be received and 491 * will normally be applied to the socket by a protocol when it detects 492 * that the peer will send no more data. Data queued for reading in 493 * the socket may yet be read. 494 */ 495void 496socantrcvmore(struct socket *so) 497{ 498 KASSERT(solocked(so)); 499 500 so->so_state |= SS_CANTRCVMORE; 501 sorwakeup(so); 502} 503 504/* 505 * soroverflow(): indicates that data was attempted to be sent 506 * but the receiving buffer overflowed. 507 */ 508void 509soroverflow(struct socket *so) 510{ 511 KASSERT(solocked(so)); 512 513 so->so_rcv.sb_overflowed++; 514 if (so->so_options & SO_RERROR) { 515 so->so_rerror = ENOBUFS; 516 sorwakeup(so); 517 } 518} 519 520/* 521 * Wait for data to arrive at/drain from a socket buffer. 522 */ 523int 524sbwait(struct sockbuf *sb) 525{ 526 struct socket *so; 527 kmutex_t *lock; 528 int error; 529 530 so = sb->sb_so; 531 532 KASSERT(solocked(so)); 533 534 sb->sb_flags |= SB_NOTIFY; 535 lock = so->so_lock; 536 if ((sb->sb_flags & SB_NOINTR) != 0) 537 error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo); 538 else 539 error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo); 540 if (__predict_false(lock != so->so_lock)) 541 solockretry(so, lock); 542 return error; 543} 544 545/* 546 * Wakeup processes waiting on a socket buffer. 547 * Do asynchronous notification via SIGIO 548 * if the socket buffer has the SB_ASYNC flag set. 549 */ 550void 551sowakeup(struct socket *so, struct sockbuf *sb, int code) 552{ 553 int band; 554 555 KASSERT(solocked(so)); 556 KASSERT(sb->sb_so == so); 557 558 switch (code) { 559 case POLL_IN: 560 band = POLLIN|POLLRDNORM; 561 break; 562 563 case POLL_OUT: 564 band = POLLOUT|POLLWRNORM; 565 break; 566 567 case POLL_HUP: 568 band = POLLHUP; 569 break; 570 571 default: 572 band = 0; 573#ifdef DIAGNOSTIC 574 printf("bad siginfo code %d in socket notification.\n", code); 575#endif 576 break; 577 } 578 579 sb->sb_flags &= ~SB_NOTIFY; 580 selnotify(&sb->sb_sel, band, NOTE_SUBMIT); 581 cv_broadcast(&sb->sb_cv); 582 if (sb->sb_flags & SB_ASYNC) 583 fownsignal(so->so_pgid, SIGIO, code, band, so); 584 if (sb->sb_flags & SB_UPCALL) 585 (*so->so_upcall)(so, so->so_upcallarg, band, M_DONTWAIT); 586} 587 588/* 589 * Reset a socket's lock pointer. Wake all threads waiting on the 590 * socket's condition variables so that they can restart their waits 591 * using the new lock. The existing lock must be held. 592 */ 593void 594solockreset(struct socket *so, kmutex_t *lock) 595{ 596 597 KASSERT(solocked(so)); 598 599 so->so_lock = lock; 600 cv_broadcast(&so->so_snd.sb_cv); 601 cv_broadcast(&so->so_rcv.sb_cv); 602 cv_broadcast(&so->so_cv); 603} 604 605/* 606 * Socket buffer (struct sockbuf) utility routines. 607 * 608 * Each socket contains two socket buffers: one for sending data and 609 * one for receiving data. Each buffer contains a queue of mbufs, 610 * information about the number of mbufs and amount of data in the 611 * queue, and other fields allowing poll() statements and notification 612 * on data availability to be implemented. 613 * 614 * Data stored in a socket buffer is maintained as a list of records. 615 * Each record is a list of mbufs chained together with the m_next 616 * field. Records are chained together with the m_nextpkt field. The upper 617 * level routine soreceive() expects the following conventions to be 618 * observed when placing information in the receive buffer: 619 * 620 * 1. If the protocol requires each message be preceded by the sender's 621 * name, then a record containing that name must be present before 622 * any associated data (mbuf's must be of type MT_SONAME). 623 * 2. If the protocol supports the exchange of ``access rights'' (really 624 * just additional data associated with the message), and there are 625 * ``rights'' to be received, then a record containing this data 626 * should be present (mbuf's must be of type MT_CONTROL). 627 * 3. If a name or rights record exists, then it must be followed by 628 * a data record, perhaps of zero length. 629 * 630 * Before using a new socket structure it is first necessary to reserve 631 * buffer space to the socket, by calling sbreserve(). This should commit 632 * some of the available buffer space in the system buffer pool for the 633 * socket (currently, it does nothing but enforce limits). The space 634 * should be released by calling sbrelease() when the socket is destroyed. 635 */ 636 637int 638sb_max_set(u_long new_sbmax) 639{ 640 int s; 641 642 if (new_sbmax < (16 * 1024)) 643 return (EINVAL); 644 645 s = splsoftnet(); 646 sb_max = new_sbmax; 647 sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES); 648 splx(s); 649 650 return (0); 651} 652 653int 654soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 655{ 656 KASSERT(so->so_pcb == NULL || solocked(so)); 657 658 /* 659 * there's at least one application (a configure script of screen) 660 * which expects a fifo is writable even if it has "some" bytes 661 * in its buffer. 662 * so we want to make sure (hiwat - lowat) >= (some bytes). 663 * 664 * PIPE_BUF here is an arbitrary value chosen as (some bytes) above. 665 * we expect it's large enough for such applications. 666 */ 667 u_long lowat = MAX(sock_loan_thresh, MCLBYTES); 668 u_long hiwat = lowat + PIPE_BUF; 669 670 if (sndcc < hiwat) 671 sndcc = hiwat; 672 if (sbreserve(&so->so_snd, sndcc, so) == 0) 673 goto bad; 674 if (sbreserve(&so->so_rcv, rcvcc, so) == 0) 675 goto bad2; 676 if (so->so_rcv.sb_lowat == 0) 677 so->so_rcv.sb_lowat = 1; 678 if (so->so_snd.sb_lowat == 0) 679 so->so_snd.sb_lowat = lowat; 680 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 681 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 682 return (0); 683 bad2: 684 sbrelease(&so->so_snd, so); 685 bad: 686 return (ENOBUFS); 687} 688 689/* 690 * Allot mbufs to a sockbuf. 691 * Attempt to scale mbmax so that mbcnt doesn't become limiting 692 * if buffering efficiency is near the normal case. 693 */ 694int 695sbreserve(struct sockbuf *sb, u_long cc, struct socket *so) 696{ 697 struct lwp *l = curlwp; /* XXX */ 698 rlim_t maxcc; 699 struct uidinfo *uidinfo; 700 701 KASSERT(so->so_pcb == NULL || solocked(so)); 702 KASSERT(sb->sb_so == so); 703 KASSERT(sb_max_adj != 0); 704 705 if (cc == 0 || cc > sb_max_adj) 706 return (0); 707 708 maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur; 709 710 uidinfo = so->so_uidinfo; 711 if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc)) 712 return 0; 713 sb->sb_mbmax = uimin(cc * 2, sb_max); 714 if (sb->sb_lowat > sb->sb_hiwat) 715 sb->sb_lowat = sb->sb_hiwat; 716 717 return (1); 718} 719 720/* 721 * Free mbufs held by a socket, and reserved mbuf space. We do not assert 722 * that the socket is held locked here: see sorflush(). 723 */ 724void 725sbrelease(struct sockbuf *sb, struct socket *so) 726{ 727 728 KASSERT(sb->sb_so == so); 729 730 sbflush(sb); 731 (void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY); 732 sb->sb_mbmax = 0; 733} 734 735/* 736 * Routines to add and remove 737 * data from an mbuf queue. 738 * 739 * The routines sbappend() or sbappendrecord() are normally called to 740 * append new mbufs to a socket buffer, after checking that adequate 741 * space is available, comparing the function sbspace() with the amount 742 * of data to be added. sbappendrecord() differs from sbappend() in 743 * that data supplied is treated as the beginning of a new record. 744 * To place a sender's address, optional access rights, and data in a 745 * socket receive buffer, sbappendaddr() should be used. To place 746 * access rights and data in a socket receive buffer, sbappendrights() 747 * should be used. In either case, the new data begins a new record. 748 * Note that unlike sbappend() and sbappendrecord(), these routines check 749 * for the caller that there will be enough space to store the data. 750 * Each fails if there is not enough space, or if it cannot find mbufs 751 * to store additional information in. 752 * 753 * Reliable protocols may use the socket send buffer to hold data 754 * awaiting acknowledgement. Data is normally copied from a socket 755 * send buffer in a protocol with m_copym for output to a peer, 756 * and then removing the data from the socket buffer with sbdrop() 757 * or sbdroprecord() when the data is acknowledged by the peer. 758 */ 759 760#ifdef SOCKBUF_DEBUG 761void 762sblastrecordchk(struct sockbuf *sb, const char *where) 763{ 764 struct mbuf *m = sb->sb_mb; 765 766 KASSERT(solocked(sb->sb_so)); 767 768 while (m && m->m_nextpkt) 769 m = m->m_nextpkt; 770 771 if (m != sb->sb_lastrecord) { 772 printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n", 773 sb->sb_mb, sb->sb_lastrecord, m); 774 printf("packet chain:\n"); 775 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 776 printf("\t%p\n", m); 777 panic("sblastrecordchk from %s", where); 778 } 779} 780 781void 782sblastmbufchk(struct sockbuf *sb, const char *where) 783{ 784 struct mbuf *m = sb->sb_mb; 785 struct mbuf *n; 786 787 KASSERT(solocked(sb->sb_so)); 788 789 while (m && m->m_nextpkt) 790 m = m->m_nextpkt; 791 792 while (m && m->m_next) 793 m = m->m_next; 794 795 if (m != sb->sb_mbtail) { 796 printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n", 797 sb->sb_mb, sb->sb_mbtail, m); 798 printf("packet tree:\n"); 799 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 800 printf("\t"); 801 for (n = m; n != NULL; n = n->m_next) 802 printf("%p ", n); 803 printf("\n"); 804 } 805 panic("sblastmbufchk from %s", where); 806 } 807} 808#endif /* SOCKBUF_DEBUG */ 809 810/* 811 * Link a chain of records onto a socket buffer 812 */ 813#define SBLINKRECORDCHAIN(sb, m0, mlast) \ 814do { \ 815 if ((sb)->sb_lastrecord != NULL) \ 816 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 817 else \ 818 (sb)->sb_mb = (m0); \ 819 (sb)->sb_lastrecord = (mlast); \ 820} while (/*CONSTCOND*/0) 821 822 823#define SBLINKRECORD(sb, m0) \ 824 SBLINKRECORDCHAIN(sb, m0, m0) 825 826/* 827 * Append mbuf chain m to the last record in the 828 * socket buffer sb. The additional space associated 829 * the mbuf chain is recorded in sb. Empty mbufs are 830 * discarded and mbufs are compacted where possible. 831 */ 832void 833sbappend(struct sockbuf *sb, struct mbuf *m) 834{ 835 struct mbuf *n; 836 837 KASSERT(solocked(sb->sb_so)); 838 839 if (m == NULL) 840 return; 841 842#ifdef MBUFTRACE 843 m_claimm(m, sb->sb_mowner); 844#endif 845 846 SBLASTRECORDCHK(sb, "sbappend 1"); 847 848 if ((n = sb->sb_lastrecord) != NULL) { 849 /* 850 * XXX Would like to simply use sb_mbtail here, but 851 * XXX I need to verify that I won't miss an EOR that 852 * XXX way. 853 */ 854 do { 855 if (n->m_flags & M_EOR) { 856 sbappendrecord(sb, m); /* XXXXXX!!!! */ 857 return; 858 } 859 } while (n->m_next && (n = n->m_next)); 860 } else { 861 /* 862 * If this is the first record in the socket buffer, it's 863 * also the last record. 864 */ 865 sb->sb_lastrecord = m; 866 } 867 sbcompress(sb, m, n); 868 SBLASTRECORDCHK(sb, "sbappend 2"); 869} 870 871/* 872 * This version of sbappend() should only be used when the caller 873 * absolutely knows that there will never be more than one record 874 * in the socket buffer, that is, a stream protocol (such as TCP). 875 */ 876void 877sbappendstream(struct sockbuf *sb, struct mbuf *m) 878{ 879 880 KASSERT(solocked(sb->sb_so)); 881 KDASSERT(m->m_nextpkt == NULL); 882 KASSERT(sb->sb_mb == sb->sb_lastrecord); 883 884 SBLASTMBUFCHK(sb, __func__); 885 886#ifdef MBUFTRACE 887 m_claimm(m, sb->sb_mowner); 888#endif 889 890 sbcompress(sb, m, sb->sb_mbtail); 891 892 sb->sb_lastrecord = sb->sb_mb; 893 SBLASTRECORDCHK(sb, __func__); 894} 895 896#ifdef SOCKBUF_DEBUG 897void 898sbcheck(struct sockbuf *sb) 899{ 900 struct mbuf *m, *m2; 901 u_long len, mbcnt; 902 903 KASSERT(solocked(sb->sb_so)); 904 905 len = 0; 906 mbcnt = 0; 907 for (m = sb->sb_mb; m; m = m->m_nextpkt) { 908 for (m2 = m; m2 != NULL; m2 = m2->m_next) { 909 len += m2->m_len; 910 mbcnt += MSIZE; 911 if (m2->m_flags & M_EXT) 912 mbcnt += m2->m_ext.ext_size; 913 if (m2->m_nextpkt != NULL) 914 panic("sbcheck nextpkt"); 915 } 916 } 917 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 918 printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc, 919 mbcnt, sb->sb_mbcnt); 920 panic("sbcheck"); 921 } 922} 923#endif 924 925/* 926 * As above, except the mbuf chain 927 * begins a new record. 928 */ 929void 930sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 931{ 932 struct mbuf *m; 933 934 KASSERT(solocked(sb->sb_so)); 935 936 if (m0 == NULL) 937 return; 938 939#ifdef MBUFTRACE 940 m_claimm(m0, sb->sb_mowner); 941#endif 942 /* 943 * Put the first mbuf on the queue. 944 * Note this permits zero length records. 945 */ 946 sballoc(sb, m0); 947 SBLASTRECORDCHK(sb, "sbappendrecord 1"); 948 SBLINKRECORD(sb, m0); 949 m = m0->m_next; 950 m0->m_next = 0; 951 if (m && (m0->m_flags & M_EOR)) { 952 m0->m_flags &= ~M_EOR; 953 m->m_flags |= M_EOR; 954 } 955 sbcompress(sb, m, m0); 956 SBLASTRECORDCHK(sb, "sbappendrecord 2"); 957} 958 959/* 960 * As above except that OOB data 961 * is inserted at the beginning of the sockbuf, 962 * but after any other OOB data. 963 */ 964void 965sbinsertoob(struct sockbuf *sb, struct mbuf *m0) 966{ 967 struct mbuf *m, **mp; 968 969 KASSERT(solocked(sb->sb_so)); 970 971 if (m0 == NULL) 972 return; 973 974 SBLASTRECORDCHK(sb, "sbinsertoob 1"); 975 976 for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) { 977 again: 978 switch (m->m_type) { 979 980 case MT_OOBDATA: 981 continue; /* WANT next train */ 982 983 case MT_CONTROL: 984 if ((m = m->m_next) != NULL) 985 goto again; /* inspect THIS train further */ 986 } 987 break; 988 } 989 /* 990 * Put the first mbuf on the queue. 991 * Note this permits zero length records. 992 */ 993 sballoc(sb, m0); 994 m0->m_nextpkt = *mp; 995 if (*mp == NULL) { 996 /* m0 is actually the new tail */ 997 sb->sb_lastrecord = m0; 998 } 999 *mp = m0; 1000 m = m0->m_next; 1001 m0->m_next = 0; 1002 if (m && (m0->m_flags & M_EOR)) { 1003 m0->m_flags &= ~M_EOR; 1004 m->m_flags |= M_EOR; 1005 } 1006 sbcompress(sb, m, m0); 1007 SBLASTRECORDCHK(sb, "sbinsertoob 2"); 1008} 1009 1010/* 1011 * Append address and data, and optionally, control (ancillary) data 1012 * to the receive queue of a socket. If present, 1013 * m0 must include a packet header with total length. 1014 * Returns 0 if no space in sockbuf or insufficient mbufs. 1015 */ 1016int 1017sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, 1018 struct mbuf *control) 1019{ 1020 struct mbuf *m, *n, *nlast; 1021 int space, len; 1022 1023 KASSERT(solocked(sb->sb_so)); 1024 1025 space = asa->sa_len; 1026 1027 if (m0 != NULL) { 1028 if ((m0->m_flags & M_PKTHDR) == 0) 1029 panic("sbappendaddr"); 1030 space += m0->m_pkthdr.len; 1031#ifdef MBUFTRACE 1032 m_claimm(m0, sb->sb_mowner); 1033#endif 1034 } 1035 for (n = control; n; n = n->m_next) { 1036 space += n->m_len; 1037 MCLAIM(n, sb->sb_mowner); 1038 if (n->m_next == NULL) /* keep pointer to last control buf */ 1039 break; 1040 } 1041 if (space > sbspace(sb)) 1042 return (0); 1043 m = m_get(M_DONTWAIT, MT_SONAME); 1044 if (m == NULL) 1045 return (0); 1046 MCLAIM(m, sb->sb_mowner); 1047 /* 1048 * XXX avoid 'comparison always true' warning which isn't easily 1049 * avoided. 1050 */ 1051 len = asa->sa_len; 1052 if (len > MLEN) { 1053 MEXTMALLOC(m, asa->sa_len, M_NOWAIT); 1054 if ((m->m_flags & M_EXT) == 0) { 1055 m_free(m); 1056 return (0); 1057 } 1058 } 1059 m->m_len = asa->sa_len; 1060 memcpy(mtod(m, void *), asa, asa->sa_len); 1061 if (n) 1062 n->m_next = m0; /* concatenate data to control */ 1063 else 1064 control = m0; 1065 m->m_next = control; 1066 1067 SBLASTRECORDCHK(sb, "sbappendaddr 1"); 1068 1069 for (n = m; n->m_next != NULL; n = n->m_next) 1070 sballoc(sb, n); 1071 sballoc(sb, n); 1072 nlast = n; 1073 SBLINKRECORD(sb, m); 1074 1075 sb->sb_mbtail = nlast; 1076 SBLASTMBUFCHK(sb, "sbappendaddr"); 1077 SBLASTRECORDCHK(sb, "sbappendaddr 2"); 1078 1079 return (1); 1080} 1081 1082/* 1083 * Helper for sbappendchainaddr: prepend a struct sockaddr* to 1084 * an mbuf chain. 1085 */ 1086static inline struct mbuf * 1087m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0, 1088 const struct sockaddr *asa) 1089{ 1090 struct mbuf *m; 1091 const int salen = asa->sa_len; 1092 1093 KASSERT(solocked(sb->sb_so)); 1094 1095 /* only the first in each chain need be a pkthdr */ 1096 m = m_gethdr(M_DONTWAIT, MT_SONAME); 1097 if (m == NULL) 1098 return NULL; 1099 MCLAIM(m, sb->sb_mowner); 1100#ifdef notyet 1101 if (salen > MHLEN) { 1102 MEXTMALLOC(m, salen, M_NOWAIT); 1103 if ((m->m_flags & M_EXT) == 0) { 1104 m_free(m); 1105 return NULL; 1106 } 1107 } 1108#else 1109 KASSERT(salen <= MHLEN); 1110#endif 1111 m->m_len = salen; 1112 memcpy(mtod(m, void *), asa, salen); 1113 m->m_next = m0; 1114 m->m_pkthdr.len = salen + m0->m_pkthdr.len; 1115 1116 return m; 1117} 1118 1119int 1120sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa, 1121 struct mbuf *m0, int sbprio) 1122{ 1123 struct mbuf *m, *n, *n0, *nlast; 1124 int error; 1125 1126 KASSERT(solocked(sb->sb_so)); 1127 1128 /* 1129 * XXX sbprio reserved for encoding priority of this* request: 1130 * SB_PRIO_NONE --> honour normal sb limits 1131 * SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space, 1132 * take whole chain. Intended for large requests 1133 * that should be delivered atomically (all, or none). 1134 * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow 1135 * over normal socket limits, for messages indicating 1136 * buffer overflow in earlier normal/lower-priority messages 1137 * SB_PRIO_BESTEFFORT --> ignore limits entirely. 1138 * Intended for kernel-generated messages only. 1139 * Up to generator to avoid total mbuf resource exhaustion. 1140 */ 1141 (void)sbprio; 1142 1143 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 1144 panic("sbappendaddrchain"); 1145 1146#ifdef notyet 1147 space = sbspace(sb); 1148 1149 /* 1150 * Enforce SB_PRIO_* limits as described above. 1151 */ 1152#endif 1153 1154 n0 = NULL; 1155 nlast = NULL; 1156 for (m = m0; m; m = m->m_nextpkt) { 1157 struct mbuf *np; 1158 1159#ifdef MBUFTRACE 1160 m_claimm(m, sb->sb_mowner); 1161#endif 1162 1163 /* Prepend sockaddr to this record (m) of input chain m0 */ 1164 n = m_prepend_sockaddr(sb, m, asa); 1165 if (n == NULL) { 1166 error = ENOBUFS; 1167 goto bad; 1168 } 1169 1170 /* Append record (asa+m) to end of new chain n0 */ 1171 if (n0 == NULL) { 1172 n0 = n; 1173 } else { 1174 nlast->m_nextpkt = n; 1175 } 1176 /* Keep track of last record on new chain */ 1177 nlast = n; 1178 1179 for (np = n; np; np = np->m_next) 1180 sballoc(sb, np); 1181 } 1182 1183 SBLASTRECORDCHK(sb, "sbappendaddrchain 1"); 1184 1185 /* Drop the entire chain of (asa+m) records onto the socket */ 1186 SBLINKRECORDCHAIN(sb, n0, nlast); 1187 1188 SBLASTRECORDCHK(sb, "sbappendaddrchain 2"); 1189 1190 for (m = nlast; m->m_next; m = m->m_next) 1191 ; 1192 sb->sb_mbtail = m; 1193 SBLASTMBUFCHK(sb, "sbappendaddrchain"); 1194 1195 return (1); 1196 1197bad: 1198 /* 1199 * On error, free the prepended addreseses. For consistency 1200 * with sbappendaddr(), leave it to our caller to free 1201 * the input record chain passed to us as m0. 1202 */ 1203 while ((n = n0) != NULL) { 1204 struct mbuf *np; 1205 1206 /* Undo the sballoc() of this record */ 1207 for (np = n; np; np = np->m_next) 1208 sbfree(sb, np); 1209 1210 n0 = n->m_nextpkt; /* iterate at next prepended address */ 1211 np = m_free(n); /* free prepended address (not data) */ 1212 } 1213 return error; 1214} 1215 1216 1217int 1218sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 1219{ 1220 struct mbuf *m, *mlast, *n; 1221 int space; 1222 1223 KASSERT(solocked(sb->sb_so)); 1224 1225 space = 0; 1226 if (control == NULL) 1227 panic("sbappendcontrol"); 1228 for (m = control; ; m = m->m_next) { 1229 space += m->m_len; 1230 MCLAIM(m, sb->sb_mowner); 1231 if (m->m_next == NULL) 1232 break; 1233 } 1234 n = m; /* save pointer to last control buffer */ 1235 for (m = m0; m; m = m->m_next) { 1236 MCLAIM(m, sb->sb_mowner); 1237 space += m->m_len; 1238 } 1239 if (space > sbspace(sb)) 1240 return (0); 1241 n->m_next = m0; /* concatenate data to control */ 1242 1243 SBLASTRECORDCHK(sb, "sbappendcontrol 1"); 1244 1245 for (m = control; m->m_next != NULL; m = m->m_next) 1246 sballoc(sb, m); 1247 sballoc(sb, m); 1248 mlast = m; 1249 SBLINKRECORD(sb, control); 1250 1251 sb->sb_mbtail = mlast; 1252 SBLASTMBUFCHK(sb, "sbappendcontrol"); 1253 SBLASTRECORDCHK(sb, "sbappendcontrol 2"); 1254 1255 return (1); 1256} 1257 1258/* 1259 * Compress mbuf chain m into the socket 1260 * buffer sb following mbuf n. If n 1261 * is null, the buffer is presumed empty. 1262 */ 1263void 1264sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 1265{ 1266 int eor; 1267 struct mbuf *o; 1268 1269 KASSERT(solocked(sb->sb_so)); 1270 1271 eor = 0; 1272 while (m) { 1273 eor |= m->m_flags & M_EOR; 1274 if (m->m_len == 0 && 1275 (eor == 0 || 1276 (((o = m->m_next) || (o = n)) && 1277 o->m_type == m->m_type))) { 1278 if (sb->sb_lastrecord == m) 1279 sb->sb_lastrecord = m->m_next; 1280 m = m_free(m); 1281 continue; 1282 } 1283 if (n && (n->m_flags & M_EOR) == 0 && 1284 /* M_TRAILINGSPACE() checks buffer writeability */ 1285 m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */ 1286 m->m_len <= M_TRAILINGSPACE(n) && 1287 n->m_type == m->m_type) { 1288 memcpy(mtod(n, char *) + n->m_len, mtod(m, void *), 1289 (unsigned)m->m_len); 1290 n->m_len += m->m_len; 1291 sb->sb_cc += m->m_len; 1292 m = m_free(m); 1293 continue; 1294 } 1295 if (n) 1296 n->m_next = m; 1297 else 1298 sb->sb_mb = m; 1299 sb->sb_mbtail = m; 1300 sballoc(sb, m); 1301 n = m; 1302 m->m_flags &= ~M_EOR; 1303 m = m->m_next; 1304 n->m_next = 0; 1305 } 1306 if (eor) { 1307 if (n) 1308 n->m_flags |= eor; 1309 else 1310 printf("semi-panic: sbcompress\n"); 1311 } 1312 SBLASTMBUFCHK(sb, __func__); 1313} 1314 1315/* 1316 * Free all mbufs in a sockbuf. 1317 * Check that all resources are reclaimed. 1318 */ 1319void 1320sbflush(struct sockbuf *sb) 1321{ 1322 1323 KASSERT(solocked(sb->sb_so)); 1324 KASSERT((sb->sb_flags & SB_LOCK) == 0); 1325 1326 while (sb->sb_mbcnt) 1327 sbdrop(sb, (int)sb->sb_cc); 1328 1329 KASSERT(sb->sb_cc == 0); 1330 KASSERT(sb->sb_mb == NULL); 1331 KASSERT(sb->sb_mbtail == NULL); 1332 KASSERT(sb->sb_lastrecord == NULL); 1333} 1334 1335/* 1336 * Drop data from (the front of) a sockbuf. 1337 */ 1338void 1339sbdrop(struct sockbuf *sb, int len) 1340{ 1341 struct mbuf *m, *next; 1342 1343 KASSERT(solocked(sb->sb_so)); 1344 1345 next = (m = sb->sb_mb) ? m->m_nextpkt : NULL; 1346 while (len > 0) { 1347 if (m == NULL) { 1348 if (next == NULL) 1349 panic("sbdrop(%p,%d): cc=%lu", 1350 sb, len, sb->sb_cc); 1351 m = next; 1352 next = m->m_nextpkt; 1353 continue; 1354 } 1355 if (m->m_len > len) { 1356 m->m_len -= len; 1357 m->m_data += len; 1358 sb->sb_cc -= len; 1359 break; 1360 } 1361 len -= m->m_len; 1362 sbfree(sb, m); 1363 m = m_free(m); 1364 } 1365 while (m && m->m_len == 0) { 1366 sbfree(sb, m); 1367 m = m_free(m); 1368 } 1369 if (m) { 1370 sb->sb_mb = m; 1371 m->m_nextpkt = next; 1372 } else 1373 sb->sb_mb = next; 1374 /* 1375 * First part is an inline SB_EMPTY_FIXUP(). Second part 1376 * makes sure sb_lastrecord is up-to-date if we dropped 1377 * part of the last record. 1378 */ 1379 m = sb->sb_mb; 1380 if (m == NULL) { 1381 sb->sb_mbtail = NULL; 1382 sb->sb_lastrecord = NULL; 1383 } else if (m->m_nextpkt == NULL) 1384 sb->sb_lastrecord = m; 1385} 1386 1387/* 1388 * Drop a record off the front of a sockbuf 1389 * and move the next record to the front. 1390 */ 1391void 1392sbdroprecord(struct sockbuf *sb) 1393{ 1394 struct mbuf *m, *mn; 1395 1396 KASSERT(solocked(sb->sb_so)); 1397 1398 m = sb->sb_mb; 1399 if (m) { 1400 sb->sb_mb = m->m_nextpkt; 1401 do { 1402 sbfree(sb, m); 1403 mn = m_free(m); 1404 } while ((m = mn) != NULL); 1405 } 1406 SB_EMPTY_FIXUP(sb); 1407} 1408 1409/* 1410 * Create a "control" mbuf containing the specified data 1411 * with the specified type for presentation on a socket buffer. 1412 */ 1413struct mbuf * 1414sbcreatecontrol1(void **p, int size, int type, int level, int flags) 1415{ 1416 struct cmsghdr *cp; 1417 struct mbuf *m; 1418 int space = CMSG_SPACE(size); 1419 1420 if ((flags & M_DONTWAIT) && space > MCLBYTES) { 1421 printf("%s: message too large %d\n", __func__, space); 1422 return NULL; 1423 } 1424 1425 if ((m = m_get(flags, MT_CONTROL)) == NULL) 1426 return NULL; 1427 if (space > MLEN) { 1428 if (space > MCLBYTES) 1429 MEXTMALLOC(m, space, M_WAITOK); 1430 else 1431 MCLGET(m, flags); 1432 if ((m->m_flags & M_EXT) == 0) { 1433 m_free(m); 1434 return NULL; 1435 } 1436 } 1437 cp = mtod(m, struct cmsghdr *); 1438 *p = CMSG_DATA(cp); 1439 m->m_len = space; 1440 cp->cmsg_len = CMSG_LEN(size); 1441 cp->cmsg_level = level; 1442 cp->cmsg_type = type; 1443 1444 memset(cp + 1, 0, CMSG_LEN(0) - sizeof(*cp)); 1445 memset((uint8_t *)*p + size, 0, CMSG_ALIGN(size) - size); 1446 1447 return m; 1448} 1449 1450struct mbuf * 1451sbcreatecontrol(void *p, int size, int type, int level) 1452{ 1453 struct mbuf *m; 1454 void *v; 1455 1456 m = sbcreatecontrol1(&v, size, type, level, M_DONTWAIT); 1457 if (m == NULL) 1458 return NULL; 1459 memcpy(v, p, size); 1460 return m; 1461} 1462 1463void 1464solockretry(struct socket *so, kmutex_t *lock) 1465{ 1466 1467 while (lock != so->so_lock) { 1468 mutex_exit(lock); 1469 lock = so->so_lock; 1470 mutex_enter(lock); 1471 } 1472} 1473 1474bool 1475solocked(const struct socket *so) 1476{ 1477 1478 return mutex_owned(so->so_lock); 1479} 1480 1481bool 1482solocked2(const struct socket *so1, const struct socket *so2) 1483{ 1484 const kmutex_t *lock; 1485 1486 lock = so1->so_lock; 1487 if (lock != so2->so_lock) 1488 return false; 1489 return mutex_owned(lock); 1490} 1491 1492/* 1493 * sosetlock: assign a default lock to a new socket. 1494 */ 1495void 1496sosetlock(struct socket *so) 1497{ 1498 if (so->so_lock == NULL) { 1499 kmutex_t *lock = softnet_lock; 1500 1501 so->so_lock = lock; 1502 mutex_obj_hold(lock); 1503 mutex_enter(lock); 1504 } 1505 KASSERT(solocked(so)); 1506} 1507 1508/* 1509 * Set lock on sockbuf sb; sleep if lock is already held. 1510 * Unless SB_NOINTR is set on sockbuf, sleep is interruptible. 1511 * Returns error without lock if sleep is interrupted. 1512 */ 1513int 1514sblock(struct sockbuf *sb, int wf) 1515{ 1516 struct socket *so; 1517 kmutex_t *lock; 1518 int error; 1519 1520 KASSERT(solocked(sb->sb_so)); 1521 1522 for (;;) { 1523 if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) { 1524 sb->sb_flags |= SB_LOCK; 1525 return 0; 1526 } 1527 if (wf != M_WAITOK) 1528 return EWOULDBLOCK; 1529 so = sb->sb_so; 1530 lock = so->so_lock; 1531 if ((sb->sb_flags & SB_NOINTR) != 0) { 1532 cv_wait(&so->so_cv, lock); 1533 error = 0; 1534 } else 1535 error = cv_wait_sig(&so->so_cv, lock); 1536 if (__predict_false(lock != so->so_lock)) 1537 solockretry(so, lock); 1538 if (error != 0) 1539 return error; 1540 } 1541} 1542 1543void 1544sbunlock(struct sockbuf *sb) 1545{ 1546 struct socket *so; 1547 1548 so = sb->sb_so; 1549 1550 KASSERT(solocked(so)); 1551 KASSERT((sb->sb_flags & SB_LOCK) != 0); 1552 1553 sb->sb_flags &= ~SB_LOCK; 1554 cv_broadcast(&so->so_cv); 1555} 1556 1557int 1558sowait(struct socket *so, bool catch_p, int timo) 1559{ 1560 kmutex_t *lock; 1561 int error; 1562 1563 KASSERT(solocked(so)); 1564 KASSERT(catch_p || timo != 0); 1565 1566 lock = so->so_lock; 1567 if (catch_p) 1568 error = cv_timedwait_sig(&so->so_cv, lock, timo); 1569 else 1570 error = cv_timedwait(&so->so_cv, lock, timo); 1571 if (__predict_false(lock != so->so_lock)) 1572 solockretry(so, lock); 1573 return error; 1574} 1575 1576#ifdef DDB 1577 1578/* 1579 * Currently, sofindproc() is used only from DDB. It could be used from others 1580 * by using db_mutex_enter() 1581 */ 1582 1583static inline int 1584db_mutex_enter(kmutex_t *mtx) 1585{ 1586 extern int db_active; 1587 int rv; 1588 1589 if (!db_active) { 1590 mutex_enter(mtx); 1591 rv = 1; 1592 } else 1593 rv = mutex_tryenter(mtx); 1594 1595 return rv; 1596} 1597 1598int 1599sofindproc(struct socket *so, int all, void (*pr)(const char *, ...)) 1600{ 1601 proc_t *p; 1602 filedesc_t *fdp; 1603 fdtab_t *dt; 1604 fdfile_t *ff; 1605 file_t *fp = NULL; 1606 int found = 0; 1607 int i, t; 1608 1609 if (so == NULL) 1610 return 0; 1611 1612 t = db_mutex_enter(&proc_lock); 1613 if (!t) { 1614 pr("could not acquire proc_lock mutex\n"); 1615 return 0; 1616 } 1617 PROCLIST_FOREACH(p, &allproc) { 1618 if (p->p_stat == SIDL) 1619 continue; 1620 fdp = p->p_fd; 1621 t = db_mutex_enter(&fdp->fd_lock); 1622 if (!t) { 1623 pr("could not acquire fd_lock mutex\n"); 1624 continue; 1625 } 1626 dt = atomic_load_consume(&fdp->fd_dt); 1627 for (i = 0; i < dt->dt_nfiles; i++) { 1628 ff = dt->dt_ff[i]; 1629 if (ff == NULL) 1630 continue; 1631 1632 fp = atomic_load_consume(&ff->ff_file); 1633 if (fp == NULL) 1634 continue; 1635 1636 t = db_mutex_enter(&fp->f_lock); 1637 if (!t) { 1638 pr("could not acquire f_lock mutex\n"); 1639 continue; 1640 } 1641 if ((struct socket *)fp->f_data != so) { 1642 mutex_exit(&fp->f_lock); 1643 continue; 1644 } 1645 found++; 1646 if (pr) 1647 pr("socket %p: owner %s(pid=%d)\n", 1648 so, p->p_comm, p->p_pid); 1649 mutex_exit(&fp->f_lock); 1650 if (all == 0) 1651 break; 1652 } 1653 mutex_exit(&fdp->fd_lock); 1654 if (all == 0 && found != 0) 1655 break; 1656 } 1657 mutex_exit(&proc_lock); 1658 1659 return found; 1660} 1661 1662void 1663socket_print(const char *modif, void (*pr)(const char *, ...)) 1664{ 1665 file_t *fp; 1666 struct socket *so; 1667 struct sockbuf *sb_snd, *sb_rcv; 1668 struct mbuf *m_rec, *m; 1669 bool opt_v = false; 1670 bool opt_m = false; 1671 bool opt_a = false; 1672 bool opt_p = false; 1673 int nrecs, nmbufs; 1674 char ch; 1675 const char *family; 1676 1677 while ( (ch = *(modif++)) != '\0') { 1678 switch (ch) { 1679 case 'v': 1680 opt_v = true; 1681 break; 1682 case 'm': 1683 opt_m = true; 1684 break; 1685 case 'a': 1686 opt_a = true; 1687 break; 1688 case 'p': 1689 opt_p = true; 1690 break; 1691 } 1692 } 1693 if (opt_v == false && pr) 1694 (pr)("Ignore empty sockets. use /v to print all.\n"); 1695 if (opt_p == true && pr) 1696 (pr)("Don't search owner process.\n"); 1697 1698 LIST_FOREACH(fp, &filehead, f_list) { 1699 if (fp->f_type != DTYPE_SOCKET) 1700 continue; 1701 so = (struct socket *)fp->f_data; 1702 if (so == NULL) 1703 continue; 1704 1705 if (so->so_proto->pr_domain->dom_family == AF_INET) 1706 family = "INET"; 1707#ifdef INET6 1708 else if (so->so_proto->pr_domain->dom_family == AF_INET6) 1709 family = "INET6"; 1710#endif 1711 else if (so->so_proto->pr_domain->dom_family == pseudo_AF_KEY) 1712 family = "KEY"; 1713 else if (so->so_proto->pr_domain->dom_family == AF_ROUTE) 1714 family = "ROUTE"; 1715 else 1716 continue; 1717 1718 sb_snd = &so->so_snd; 1719 sb_rcv = &so->so_rcv; 1720 1721 if (opt_v != true && 1722 sb_snd->sb_cc == 0 && sb_rcv->sb_cc == 0) 1723 continue; 1724 1725 pr("---SOCKET %p: type %s\n", so, family); 1726 if (opt_p != true) 1727 sofindproc(so, opt_a == true ? 1 : 0, pr); 1728 pr("Send Buffer Bytes: %d [bytes]\n", sb_snd->sb_cc); 1729 pr("Send Buffer mbufs:\n"); 1730 m_rec = m = sb_snd->sb_mb; 1731 nrecs = 0; 1732 nmbufs = 0; 1733 while (m_rec) { 1734 nrecs++; 1735 if (opt_m == true) 1736 pr(" mbuf chain %p\n", m_rec); 1737 while (m) { 1738 nmbufs++; 1739 m = m->m_next; 1740 } 1741 m_rec = m = m_rec->m_nextpkt; 1742 } 1743 pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs); 1744 1745 pr("Recv Buffer Usage: %d [bytes]\n", sb_rcv->sb_cc); 1746 pr("Recv Buffer mbufs:\n"); 1747 m_rec = m = sb_rcv->sb_mb; 1748 nrecs = 0; 1749 nmbufs = 0; 1750 while (m_rec) { 1751 nrecs++; 1752 if (opt_m == true) 1753 pr(" mbuf chain %p\n", m_rec); 1754 while (m) { 1755 nmbufs++; 1756 m = m->m_next; 1757 } 1758 m_rec = m = m_rec->m_nextpkt; 1759 } 1760 pr(" Total %d records, %d mbufs.\n", nrecs, nmbufs); 1761 } 1762} 1763#endif /* DDB */ 1764