uipc_socket2.c revision 1.128
1/* $NetBSD: uipc_socket2.c,v 1.128 2018/03/19 16:26:26 roy Exp $ */ 2 3/*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * Copyright (c) 1982, 1986, 1988, 1990, 1993 31 * The Regents of the University of California. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. Neither the name of the University nor the names of its contributors 42 * may be used to endorse or promote products derived from this software 43 * without specific prior written permission. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * @(#)uipc_socket2.c 8.2 (Berkeley) 2/14/95 58 */ 59 60#include <sys/cdefs.h> 61__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.128 2018/03/19 16:26:26 roy Exp $"); 62 63#ifdef _KERNEL_OPT 64#include "opt_mbuftrace.h" 65#include "opt_sb_max.h" 66#endif 67 68#include <sys/param.h> 69#include <sys/systm.h> 70#include <sys/proc.h> 71#include <sys/file.h> 72#include <sys/buf.h> 73#include <sys/mbuf.h> 74#include <sys/protosw.h> 75#include <sys/domain.h> 76#include <sys/poll.h> 77#include <sys/socket.h> 78#include <sys/socketvar.h> 79#include <sys/signalvar.h> 80#include <sys/kauth.h> 81#include <sys/pool.h> 82#include <sys/uidinfo.h> 83 84/* 85 * Primitive routines for operating on sockets and socket buffers. 86 * 87 * Connection life-cycle: 88 * 89 * Normal sequence from the active (originating) side: 90 * 91 * - soisconnecting() is called during processing of connect() call, 92 * - resulting in an eventual call to soisconnected() if/when the 93 * connection is established. 94 * 95 * When the connection is torn down during processing of disconnect(): 96 * 97 * - soisdisconnecting() is called and, 98 * - soisdisconnected() is called when the connection to the peer 99 * is totally severed. 100 * 101 * The semantics of these routines are such that connectionless protocols 102 * can call soisconnected() and soisdisconnected() only, bypassing the 103 * in-progress calls when setting up a ``connection'' takes no time. 104 * 105 * From the passive side, a socket is created with two queues of sockets: 106 * 107 * - so_q0 (0) for partial connections (i.e. connections in progress) 108 * - so_q (1) for connections already made and awaiting user acceptance. 109 * 110 * As a protocol is preparing incoming connections, it creates a socket 111 * structure queued on so_q0 by calling sonewconn(). When the connection 112 * is established, soisconnected() is called, and transfers the 113 * socket structure to so_q, making it available to accept(). 114 * 115 * If a socket is closed with sockets on either so_q0 or so_q, these 116 * sockets are dropped. 117 * 118 * Locking rules and assumptions: 119 * 120 * o socket::so_lock can change on the fly. The low level routines used 121 * to lock sockets are aware of this. When so_lock is acquired, the 122 * routine locking must check to see if so_lock still points to the 123 * lock that was acquired. If so_lock has changed in the meantime, the 124 * now irrelevant lock that was acquired must be dropped and the lock 125 * operation retried. Although not proven here, this is completely safe 126 * on a multiprocessor system, even with relaxed memory ordering, given 127 * the next two rules: 128 * 129 * o In order to mutate so_lock, the lock pointed to by the current value 130 * of so_lock must be held: i.e., the socket must be held locked by the 131 * changing thread. The thread must issue membar_exit() to prevent 132 * memory accesses being reordered, and can set so_lock to the desired 133 * value. If the lock pointed to by the new value of so_lock is not 134 * held by the changing thread, the socket must then be considered 135 * unlocked. 136 * 137 * o If so_lock is mutated, and the previous lock referred to by so_lock 138 * could still be visible to other threads in the system (e.g. via file 139 * descriptor or protocol-internal reference), then the old lock must 140 * remain valid until the socket and/or protocol control block has been 141 * torn down. 142 * 143 * o If a socket has a non-NULL so_head value (i.e. is in the process of 144 * connecting), then locking the socket must also lock the socket pointed 145 * to by so_head: their lock pointers must match. 146 * 147 * o If a socket has connections in progress (so_q, so_q0 not empty) then 148 * locking the socket must also lock the sockets attached to both queues. 149 * Again, their lock pointers must match. 150 * 151 * o Beyond the initial lock assignment in socreate(), assigning locks to 152 * sockets is the responsibility of the individual protocols / protocol 153 * domains. 154 */ 155 156static pool_cache_t socket_cache; 157u_long sb_max = SB_MAX;/* maximum socket buffer size */ 158static u_long sb_max_adj; /* adjusted sb_max */ 159 160void 161soisconnecting(struct socket *so) 162{ 163 164 KASSERT(solocked(so)); 165 166 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 167 so->so_state |= SS_ISCONNECTING; 168} 169 170void 171soisconnected(struct socket *so) 172{ 173 struct socket *head; 174 175 head = so->so_head; 176 177 KASSERT(solocked(so)); 178 KASSERT(head == NULL || solocked2(so, head)); 179 180 so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING); 181 so->so_state |= SS_ISCONNECTED; 182 if (head && so->so_onq == &head->so_q0) { 183 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 184 /* 185 * Re-enqueue and wake up any waiters, e.g. 186 * processes blocking on accept(). 187 */ 188 soqremque(so, 0); 189 soqinsque(head, so, 1); 190 sorwakeup(head); 191 cv_broadcast(&head->so_cv); 192 } else { 193 so->so_upcall = 194 head->so_accf->so_accept_filter->accf_callback; 195 so->so_upcallarg = head->so_accf->so_accept_filter_arg; 196 so->so_rcv.sb_flags |= SB_UPCALL; 197 so->so_options &= ~SO_ACCEPTFILTER; 198 (*so->so_upcall)(so, so->so_upcallarg, 199 POLLIN|POLLRDNORM, M_DONTWAIT); 200 } 201 } else { 202 cv_broadcast(&so->so_cv); 203 sorwakeup(so); 204 sowwakeup(so); 205 } 206} 207 208void 209soisdisconnecting(struct socket *so) 210{ 211 212 KASSERT(solocked(so)); 213 214 so->so_state &= ~SS_ISCONNECTING; 215 so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE); 216 cv_broadcast(&so->so_cv); 217 sowwakeup(so); 218 sorwakeup(so); 219} 220 221void 222soisdisconnected(struct socket *so) 223{ 224 225 KASSERT(solocked(so)); 226 227 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 228 so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED); 229 cv_broadcast(&so->so_cv); 230 sowwakeup(so); 231 sorwakeup(so); 232} 233 234void 235soinit2(void) 236{ 237 238 socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0, 239 "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL); 240} 241 242/* 243 * sonewconn: accept a new connection. 244 * 245 * When an attempt at a new connection is noted on a socket which accepts 246 * connections, sonewconn(9) is called. If the connection is possible 247 * (subject to space constraints, etc) then we allocate a new structure, 248 * properly linked into the data structure of the original socket. 249 * 250 * => If 'soready' is true, then socket will become ready for accept() i.e. 251 * inserted into the so_q queue, SS_ISCONNECTED set and waiters awoken. 252 * => May be called from soft-interrupt context. 253 * => Listening socket should be locked. 254 * => Returns the new socket locked. 255 */ 256struct socket * 257sonewconn(struct socket *head, bool soready) 258{ 259 struct socket *so; 260 int soqueue, error; 261 262 KASSERT(solocked(head)); 263 264 if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) { 265 /* 266 * Listen queue overflow. If there is an accept filter 267 * active, pass through the oldest cxn it's handling. 268 */ 269 if (head->so_accf == NULL) { 270 return NULL; 271 } else { 272 struct socket *so2, *next; 273 274 /* Pass the oldest connection waiting in the 275 accept filter */ 276 for (so2 = TAILQ_FIRST(&head->so_q0); 277 so2 != NULL; so2 = next) { 278 next = TAILQ_NEXT(so2, so_qe); 279 if (so2->so_upcall == NULL) { 280 continue; 281 } 282 so2->so_upcall = NULL; 283 so2->so_upcallarg = NULL; 284 so2->so_options &= ~SO_ACCEPTFILTER; 285 so2->so_rcv.sb_flags &= ~SB_UPCALL; 286 soisconnected(so2); 287 break; 288 } 289 290 /* If nothing was nudged out of the acept filter, bail 291 * out; otherwise proceed allocating the socket. */ 292 if (so2 == NULL) { 293 return NULL; 294 } 295 } 296 } 297 if ((head->so_options & SO_ACCEPTFILTER) != 0) { 298 soready = false; 299 } 300 soqueue = soready ? 1 : 0; 301 302 if ((so = soget(false)) == NULL) { 303 return NULL; 304 } 305 so->so_type = head->so_type; 306 so->so_options = head->so_options & ~SO_ACCEPTCONN; 307 so->so_linger = head->so_linger; 308 so->so_state = head->so_state | SS_NOFDREF; 309 so->so_proto = head->so_proto; 310 so->so_timeo = head->so_timeo; 311 so->so_pgid = head->so_pgid; 312 so->so_send = head->so_send; 313 so->so_receive = head->so_receive; 314 so->so_uidinfo = head->so_uidinfo; 315 so->so_cpid = head->so_cpid; 316 317 /* 318 * Share the lock with the listening-socket, it may get unshared 319 * once the connection is complete. 320 */ 321 mutex_obj_hold(head->so_lock); 322 so->so_lock = head->so_lock; 323 324 /* 325 * Reserve the space for socket buffers. 326 */ 327#ifdef MBUFTRACE 328 so->so_mowner = head->so_mowner; 329 so->so_rcv.sb_mowner = head->so_rcv.sb_mowner; 330 so->so_snd.sb_mowner = head->so_snd.sb_mowner; 331#endif 332 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 333 goto out; 334 } 335 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 336 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 337 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 338 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 339 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 340 so->so_snd.sb_flags |= head->so_snd.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 341 342 /* 343 * Finally, perform the protocol attach. Note: a new socket 344 * lock may be assigned at this point (if so, it will be held). 345 */ 346 error = (*so->so_proto->pr_usrreqs->pr_attach)(so, 0); 347 if (error) { 348out: 349 KASSERT(solocked(so)); 350 KASSERT(so->so_accf == NULL); 351 soput(so); 352 353 /* Note: the listening socket shall stay locked. */ 354 KASSERT(solocked(head)); 355 return NULL; 356 } 357 KASSERT(solocked2(head, so)); 358 359 /* 360 * Insert into the queue. If ready, update the connection status 361 * and wake up any waiters, e.g. processes blocking on accept(). 362 */ 363 soqinsque(head, so, soqueue); 364 if (soready) { 365 so->so_state |= SS_ISCONNECTED; 366 sorwakeup(head); 367 cv_broadcast(&head->so_cv); 368 } 369 return so; 370} 371 372struct socket * 373soget(bool waitok) 374{ 375 struct socket *so; 376 377 so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT)); 378 if (__predict_false(so == NULL)) 379 return (NULL); 380 memset(so, 0, sizeof(*so)); 381 TAILQ_INIT(&so->so_q0); 382 TAILQ_INIT(&so->so_q); 383 cv_init(&so->so_cv, "socket"); 384 cv_init(&so->so_rcv.sb_cv, "netio"); 385 cv_init(&so->so_snd.sb_cv, "netio"); 386 selinit(&so->so_rcv.sb_sel); 387 selinit(&so->so_snd.sb_sel); 388 so->so_rcv.sb_so = so; 389 so->so_snd.sb_so = so; 390 return so; 391} 392 393void 394soput(struct socket *so) 395{ 396 397 KASSERT(!cv_has_waiters(&so->so_cv)); 398 KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv)); 399 KASSERT(!cv_has_waiters(&so->so_snd.sb_cv)); 400 seldestroy(&so->so_rcv.sb_sel); 401 seldestroy(&so->so_snd.sb_sel); 402 mutex_obj_free(so->so_lock); 403 cv_destroy(&so->so_cv); 404 cv_destroy(&so->so_rcv.sb_cv); 405 cv_destroy(&so->so_snd.sb_cv); 406 pool_cache_put(socket_cache, so); 407} 408 409/* 410 * soqinsque: insert socket of a new connection into the specified 411 * accept queue of the listening socket (head). 412 * 413 * q = 0: queue of partial connections 414 * q = 1: queue of incoming connections 415 */ 416void 417soqinsque(struct socket *head, struct socket *so, int q) 418{ 419 KASSERT(q == 0 || q == 1); 420 KASSERT(solocked2(head, so)); 421 KASSERT(so->so_onq == NULL); 422 KASSERT(so->so_head == NULL); 423 424 so->so_head = head; 425 if (q == 0) { 426 head->so_q0len++; 427 so->so_onq = &head->so_q0; 428 } else { 429 head->so_qlen++; 430 so->so_onq = &head->so_q; 431 } 432 TAILQ_INSERT_TAIL(so->so_onq, so, so_qe); 433} 434 435/* 436 * soqremque: remove socket from the specified queue. 437 * 438 * => Returns true if socket was removed from the specified queue. 439 * => False if socket was not removed (because it was in other queue). 440 */ 441bool 442soqremque(struct socket *so, int q) 443{ 444 struct socket *head = so->so_head; 445 446 KASSERT(q == 0 || q == 1); 447 KASSERT(solocked(so)); 448 KASSERT(so->so_onq != NULL); 449 KASSERT(head != NULL); 450 451 if (q == 0) { 452 if (so->so_onq != &head->so_q0) 453 return false; 454 head->so_q0len--; 455 } else { 456 if (so->so_onq != &head->so_q) 457 return false; 458 head->so_qlen--; 459 } 460 KASSERT(solocked2(so, head)); 461 TAILQ_REMOVE(so->so_onq, so, so_qe); 462 so->so_onq = NULL; 463 so->so_head = NULL; 464 return true; 465} 466 467/* 468 * socantsendmore: indicates that no more data will be sent on the 469 * socket; it would normally be applied to a socket when the user 470 * informs the system that no more data is to be sent, by the protocol 471 * code (in case pr_shutdown()). 472 */ 473void 474socantsendmore(struct socket *so) 475{ 476 KASSERT(solocked(so)); 477 478 so->so_state |= SS_CANTSENDMORE; 479 sowwakeup(so); 480} 481 482/* 483 * socantrcvmore(): indicates that no more data will be received and 484 * will normally be applied to the socket by a protocol when it detects 485 * that the peer will send no more data. Data queued for reading in 486 * the socket may yet be read. 487 */ 488void 489socantrcvmore(struct socket *so) 490{ 491 KASSERT(solocked(so)); 492 493 so->so_state |= SS_CANTRCVMORE; 494 sorwakeup(so); 495} 496 497/* 498 * soroverflow(): indicates that data was attempted to be sent 499 * but the receiving buffer overflowed. 500 */ 501void 502soroverflow(struct socket *so) 503{ 504 KASSERT(solocked(so)); 505 506 so->so_rcv.sb_overflowed++; 507 so->so_error = ENOBUFS; 508 sorwakeup(so); 509} 510 511/* 512 * Wait for data to arrive at/drain from a socket buffer. 513 */ 514int 515sbwait(struct sockbuf *sb) 516{ 517 struct socket *so; 518 kmutex_t *lock; 519 int error; 520 521 so = sb->sb_so; 522 523 KASSERT(solocked(so)); 524 525 sb->sb_flags |= SB_NOTIFY; 526 lock = so->so_lock; 527 if ((sb->sb_flags & SB_NOINTR) != 0) 528 error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo); 529 else 530 error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo); 531 if (__predict_false(lock != so->so_lock)) 532 solockretry(so, lock); 533 return error; 534} 535 536/* 537 * Wakeup processes waiting on a socket buffer. 538 * Do asynchronous notification via SIGIO 539 * if the socket buffer has the SB_ASYNC flag set. 540 */ 541void 542sowakeup(struct socket *so, struct sockbuf *sb, int code) 543{ 544 int band; 545 546 KASSERT(solocked(so)); 547 KASSERT(sb->sb_so == so); 548 549 if (code == POLL_IN) 550 band = POLLIN|POLLRDNORM; 551 else 552 band = POLLOUT|POLLWRNORM; 553 sb->sb_flags &= ~SB_NOTIFY; 554 selnotify(&sb->sb_sel, band, NOTE_SUBMIT); 555 cv_broadcast(&sb->sb_cv); 556 if (sb->sb_flags & SB_ASYNC) 557 fownsignal(so->so_pgid, SIGIO, code, band, so); 558 if (sb->sb_flags & SB_UPCALL) 559 (*so->so_upcall)(so, so->so_upcallarg, band, M_DONTWAIT); 560} 561 562/* 563 * Reset a socket's lock pointer. Wake all threads waiting on the 564 * socket's condition variables so that they can restart their waits 565 * using the new lock. The existing lock must be held. 566 */ 567void 568solockreset(struct socket *so, kmutex_t *lock) 569{ 570 571 KASSERT(solocked(so)); 572 573 so->so_lock = lock; 574 cv_broadcast(&so->so_snd.sb_cv); 575 cv_broadcast(&so->so_rcv.sb_cv); 576 cv_broadcast(&so->so_cv); 577} 578 579/* 580 * Socket buffer (struct sockbuf) utility routines. 581 * 582 * Each socket contains two socket buffers: one for sending data and 583 * one for receiving data. Each buffer contains a queue of mbufs, 584 * information about the number of mbufs and amount of data in the 585 * queue, and other fields allowing poll() statements and notification 586 * on data availability to be implemented. 587 * 588 * Data stored in a socket buffer is maintained as a list of records. 589 * Each record is a list of mbufs chained together with the m_next 590 * field. Records are chained together with the m_nextpkt field. The upper 591 * level routine soreceive() expects the following conventions to be 592 * observed when placing information in the receive buffer: 593 * 594 * 1. If the protocol requires each message be preceded by the sender's 595 * name, then a record containing that name must be present before 596 * any associated data (mbuf's must be of type MT_SONAME). 597 * 2. If the protocol supports the exchange of ``access rights'' (really 598 * just additional data associated with the message), and there are 599 * ``rights'' to be received, then a record containing this data 600 * should be present (mbuf's must be of type MT_CONTROL). 601 * 3. If a name or rights record exists, then it must be followed by 602 * a data record, perhaps of zero length. 603 * 604 * Before using a new socket structure it is first necessary to reserve 605 * buffer space to the socket, by calling sbreserve(). This should commit 606 * some of the available buffer space in the system buffer pool for the 607 * socket (currently, it does nothing but enforce limits). The space 608 * should be released by calling sbrelease() when the socket is destroyed. 609 */ 610 611int 612sb_max_set(u_long new_sbmax) 613{ 614 int s; 615 616 if (new_sbmax < (16 * 1024)) 617 return (EINVAL); 618 619 s = splsoftnet(); 620 sb_max = new_sbmax; 621 sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES); 622 splx(s); 623 624 return (0); 625} 626 627int 628soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 629{ 630 KASSERT(so->so_pcb == NULL || solocked(so)); 631 632 /* 633 * there's at least one application (a configure script of screen) 634 * which expects a fifo is writable even if it has "some" bytes 635 * in its buffer. 636 * so we want to make sure (hiwat - lowat) >= (some bytes). 637 * 638 * PIPE_BUF here is an arbitrary value chosen as (some bytes) above. 639 * we expect it's large enough for such applications. 640 */ 641 u_long lowat = MAX(sock_loan_thresh, MCLBYTES); 642 u_long hiwat = lowat + PIPE_BUF; 643 644 if (sndcc < hiwat) 645 sndcc = hiwat; 646 if (sbreserve(&so->so_snd, sndcc, so) == 0) 647 goto bad; 648 if (sbreserve(&so->so_rcv, rcvcc, so) == 0) 649 goto bad2; 650 if (so->so_rcv.sb_lowat == 0) 651 so->so_rcv.sb_lowat = 1; 652 if (so->so_snd.sb_lowat == 0) 653 so->so_snd.sb_lowat = lowat; 654 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 655 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 656 return (0); 657 bad2: 658 sbrelease(&so->so_snd, so); 659 bad: 660 return (ENOBUFS); 661} 662 663/* 664 * Allot mbufs to a sockbuf. 665 * Attempt to scale mbmax so that mbcnt doesn't become limiting 666 * if buffering efficiency is near the normal case. 667 */ 668int 669sbreserve(struct sockbuf *sb, u_long cc, struct socket *so) 670{ 671 struct lwp *l = curlwp; /* XXX */ 672 rlim_t maxcc; 673 struct uidinfo *uidinfo; 674 675 KASSERT(so->so_pcb == NULL || solocked(so)); 676 KASSERT(sb->sb_so == so); 677 KASSERT(sb_max_adj != 0); 678 679 if (cc == 0 || cc > sb_max_adj) 680 return (0); 681 682 maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur; 683 684 uidinfo = so->so_uidinfo; 685 if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc)) 686 return 0; 687 sb->sb_mbmax = min(cc * 2, sb_max); 688 if (sb->sb_lowat > sb->sb_hiwat) 689 sb->sb_lowat = sb->sb_hiwat; 690 return (1); 691} 692 693/* 694 * Free mbufs held by a socket, and reserved mbuf space. We do not assert 695 * that the socket is held locked here: see sorflush(). 696 */ 697void 698sbrelease(struct sockbuf *sb, struct socket *so) 699{ 700 701 KASSERT(sb->sb_so == so); 702 703 sbflush(sb); 704 (void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY); 705 sb->sb_mbmax = 0; 706} 707 708/* 709 * Routines to add and remove 710 * data from an mbuf queue. 711 * 712 * The routines sbappend() or sbappendrecord() are normally called to 713 * append new mbufs to a socket buffer, after checking that adequate 714 * space is available, comparing the function sbspace() with the amount 715 * of data to be added. sbappendrecord() differs from sbappend() in 716 * that data supplied is treated as the beginning of a new record. 717 * To place a sender's address, optional access rights, and data in a 718 * socket receive buffer, sbappendaddr() should be used. To place 719 * access rights and data in a socket receive buffer, sbappendrights() 720 * should be used. In either case, the new data begins a new record. 721 * Note that unlike sbappend() and sbappendrecord(), these routines check 722 * for the caller that there will be enough space to store the data. 723 * Each fails if there is not enough space, or if it cannot find mbufs 724 * to store additional information in. 725 * 726 * Reliable protocols may use the socket send buffer to hold data 727 * awaiting acknowledgement. Data is normally copied from a socket 728 * send buffer in a protocol with m_copy for output to a peer, 729 * and then removing the data from the socket buffer with sbdrop() 730 * or sbdroprecord() when the data is acknowledged by the peer. 731 */ 732 733#ifdef SOCKBUF_DEBUG 734void 735sblastrecordchk(struct sockbuf *sb, const char *where) 736{ 737 struct mbuf *m = sb->sb_mb; 738 739 KASSERT(solocked(sb->sb_so)); 740 741 while (m && m->m_nextpkt) 742 m = m->m_nextpkt; 743 744 if (m != sb->sb_lastrecord) { 745 printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n", 746 sb->sb_mb, sb->sb_lastrecord, m); 747 printf("packet chain:\n"); 748 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 749 printf("\t%p\n", m); 750 panic("sblastrecordchk from %s", where); 751 } 752} 753 754void 755sblastmbufchk(struct sockbuf *sb, const char *where) 756{ 757 struct mbuf *m = sb->sb_mb; 758 struct mbuf *n; 759 760 KASSERT(solocked(sb->sb_so)); 761 762 while (m && m->m_nextpkt) 763 m = m->m_nextpkt; 764 765 while (m && m->m_next) 766 m = m->m_next; 767 768 if (m != sb->sb_mbtail) { 769 printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n", 770 sb->sb_mb, sb->sb_mbtail, m); 771 printf("packet tree:\n"); 772 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 773 printf("\t"); 774 for (n = m; n != NULL; n = n->m_next) 775 printf("%p ", n); 776 printf("\n"); 777 } 778 panic("sblastmbufchk from %s", where); 779 } 780} 781#endif /* SOCKBUF_DEBUG */ 782 783/* 784 * Link a chain of records onto a socket buffer 785 */ 786#define SBLINKRECORDCHAIN(sb, m0, mlast) \ 787do { \ 788 if ((sb)->sb_lastrecord != NULL) \ 789 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 790 else \ 791 (sb)->sb_mb = (m0); \ 792 (sb)->sb_lastrecord = (mlast); \ 793} while (/*CONSTCOND*/0) 794 795 796#define SBLINKRECORD(sb, m0) \ 797 SBLINKRECORDCHAIN(sb, m0, m0) 798 799/* 800 * Append mbuf chain m to the last record in the 801 * socket buffer sb. The additional space associated 802 * the mbuf chain is recorded in sb. Empty mbufs are 803 * discarded and mbufs are compacted where possible. 804 */ 805void 806sbappend(struct sockbuf *sb, struct mbuf *m) 807{ 808 struct mbuf *n; 809 810 KASSERT(solocked(sb->sb_so)); 811 812 if (m == NULL) 813 return; 814 815#ifdef MBUFTRACE 816 m_claimm(m, sb->sb_mowner); 817#endif 818 819 SBLASTRECORDCHK(sb, "sbappend 1"); 820 821 if ((n = sb->sb_lastrecord) != NULL) { 822 /* 823 * XXX Would like to simply use sb_mbtail here, but 824 * XXX I need to verify that I won't miss an EOR that 825 * XXX way. 826 */ 827 do { 828 if (n->m_flags & M_EOR) { 829 sbappendrecord(sb, m); /* XXXXXX!!!! */ 830 return; 831 } 832 } while (n->m_next && (n = n->m_next)); 833 } else { 834 /* 835 * If this is the first record in the socket buffer, it's 836 * also the last record. 837 */ 838 sb->sb_lastrecord = m; 839 } 840 sbcompress(sb, m, n); 841 SBLASTRECORDCHK(sb, "sbappend 2"); 842} 843 844/* 845 * This version of sbappend() should only be used when the caller 846 * absolutely knows that there will never be more than one record 847 * in the socket buffer, that is, a stream protocol (such as TCP). 848 */ 849void 850sbappendstream(struct sockbuf *sb, struct mbuf *m) 851{ 852 853 KASSERT(solocked(sb->sb_so)); 854 KDASSERT(m->m_nextpkt == NULL); 855 KASSERT(sb->sb_mb == sb->sb_lastrecord); 856 857 SBLASTMBUFCHK(sb, __func__); 858 859#ifdef MBUFTRACE 860 m_claimm(m, sb->sb_mowner); 861#endif 862 863 sbcompress(sb, m, sb->sb_mbtail); 864 865 sb->sb_lastrecord = sb->sb_mb; 866 SBLASTRECORDCHK(sb, __func__); 867} 868 869#ifdef SOCKBUF_DEBUG 870void 871sbcheck(struct sockbuf *sb) 872{ 873 struct mbuf *m, *m2; 874 u_long len, mbcnt; 875 876 KASSERT(solocked(sb->sb_so)); 877 878 len = 0; 879 mbcnt = 0; 880 for (m = sb->sb_mb; m; m = m->m_nextpkt) { 881 for (m2 = m; m2 != NULL; m2 = m2->m_next) { 882 len += m2->m_len; 883 mbcnt += MSIZE; 884 if (m2->m_flags & M_EXT) 885 mbcnt += m2->m_ext.ext_size; 886 if (m2->m_nextpkt != NULL) 887 panic("sbcheck nextpkt"); 888 } 889 } 890 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 891 printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc, 892 mbcnt, sb->sb_mbcnt); 893 panic("sbcheck"); 894 } 895} 896#endif 897 898/* 899 * As above, except the mbuf chain 900 * begins a new record. 901 */ 902void 903sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 904{ 905 struct mbuf *m; 906 907 KASSERT(solocked(sb->sb_so)); 908 909 if (m0 == NULL) 910 return; 911 912#ifdef MBUFTRACE 913 m_claimm(m0, sb->sb_mowner); 914#endif 915 /* 916 * Put the first mbuf on the queue. 917 * Note this permits zero length records. 918 */ 919 sballoc(sb, m0); 920 SBLASTRECORDCHK(sb, "sbappendrecord 1"); 921 SBLINKRECORD(sb, m0); 922 m = m0->m_next; 923 m0->m_next = 0; 924 if (m && (m0->m_flags & M_EOR)) { 925 m0->m_flags &= ~M_EOR; 926 m->m_flags |= M_EOR; 927 } 928 sbcompress(sb, m, m0); 929 SBLASTRECORDCHK(sb, "sbappendrecord 2"); 930} 931 932/* 933 * As above except that OOB data 934 * is inserted at the beginning of the sockbuf, 935 * but after any other OOB data. 936 */ 937void 938sbinsertoob(struct sockbuf *sb, struct mbuf *m0) 939{ 940 struct mbuf *m, **mp; 941 942 KASSERT(solocked(sb->sb_so)); 943 944 if (m0 == NULL) 945 return; 946 947 SBLASTRECORDCHK(sb, "sbinsertoob 1"); 948 949 for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) { 950 again: 951 switch (m->m_type) { 952 953 case MT_OOBDATA: 954 continue; /* WANT next train */ 955 956 case MT_CONTROL: 957 if ((m = m->m_next) != NULL) 958 goto again; /* inspect THIS train further */ 959 } 960 break; 961 } 962 /* 963 * Put the first mbuf on the queue. 964 * Note this permits zero length records. 965 */ 966 sballoc(sb, m0); 967 m0->m_nextpkt = *mp; 968 if (*mp == NULL) { 969 /* m0 is actually the new tail */ 970 sb->sb_lastrecord = m0; 971 } 972 *mp = m0; 973 m = m0->m_next; 974 m0->m_next = 0; 975 if (m && (m0->m_flags & M_EOR)) { 976 m0->m_flags &= ~M_EOR; 977 m->m_flags |= M_EOR; 978 } 979 sbcompress(sb, m, m0); 980 SBLASTRECORDCHK(sb, "sbinsertoob 2"); 981} 982 983/* 984 * Append address and data, and optionally, control (ancillary) data 985 * to the receive queue of a socket. If present, 986 * m0 must include a packet header with total length. 987 * Returns 0 if no space in sockbuf or insufficient mbufs. 988 */ 989int 990sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, 991 struct mbuf *control) 992{ 993 struct mbuf *m, *n, *nlast; 994 int space, len; 995 996 KASSERT(solocked(sb->sb_so)); 997 998 space = asa->sa_len; 999 1000 if (m0 != NULL) { 1001 if ((m0->m_flags & M_PKTHDR) == 0) 1002 panic("sbappendaddr"); 1003 space += m0->m_pkthdr.len; 1004#ifdef MBUFTRACE 1005 m_claimm(m0, sb->sb_mowner); 1006#endif 1007 } 1008 for (n = control; n; n = n->m_next) { 1009 space += n->m_len; 1010 MCLAIM(n, sb->sb_mowner); 1011 if (n->m_next == NULL) /* keep pointer to last control buf */ 1012 break; 1013 } 1014 if (space > sbspace(sb)) 1015 return (0); 1016 m = m_get(M_DONTWAIT, MT_SONAME); 1017 if (m == NULL) 1018 return (0); 1019 MCLAIM(m, sb->sb_mowner); 1020 /* 1021 * XXX avoid 'comparison always true' warning which isn't easily 1022 * avoided. 1023 */ 1024 len = asa->sa_len; 1025 if (len > MLEN) { 1026 MEXTMALLOC(m, asa->sa_len, M_NOWAIT); 1027 if ((m->m_flags & M_EXT) == 0) { 1028 m_free(m); 1029 return (0); 1030 } 1031 } 1032 m->m_len = asa->sa_len; 1033 memcpy(mtod(m, void *), asa, asa->sa_len); 1034 if (n) 1035 n->m_next = m0; /* concatenate data to control */ 1036 else 1037 control = m0; 1038 m->m_next = control; 1039 1040 SBLASTRECORDCHK(sb, "sbappendaddr 1"); 1041 1042 for (n = m; n->m_next != NULL; n = n->m_next) 1043 sballoc(sb, n); 1044 sballoc(sb, n); 1045 nlast = n; 1046 SBLINKRECORD(sb, m); 1047 1048 sb->sb_mbtail = nlast; 1049 SBLASTMBUFCHK(sb, "sbappendaddr"); 1050 SBLASTRECORDCHK(sb, "sbappendaddr 2"); 1051 1052 return (1); 1053} 1054 1055/* 1056 * Helper for sbappendchainaddr: prepend a struct sockaddr* to 1057 * an mbuf chain. 1058 */ 1059static inline struct mbuf * 1060m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0, 1061 const struct sockaddr *asa) 1062{ 1063 struct mbuf *m; 1064 const int salen = asa->sa_len; 1065 1066 KASSERT(solocked(sb->sb_so)); 1067 1068 /* only the first in each chain need be a pkthdr */ 1069 m = m_gethdr(M_DONTWAIT, MT_SONAME); 1070 if (m == NULL) 1071 return NULL; 1072 MCLAIM(m, sb->sb_mowner); 1073#ifdef notyet 1074 if (salen > MHLEN) { 1075 MEXTMALLOC(m, salen, M_NOWAIT); 1076 if ((m->m_flags & M_EXT) == 0) { 1077 m_free(m); 1078 return NULL; 1079 } 1080 } 1081#else 1082 KASSERT(salen <= MHLEN); 1083#endif 1084 m->m_len = salen; 1085 memcpy(mtod(m, void *), asa, salen); 1086 m->m_next = m0; 1087 m->m_pkthdr.len = salen + m0->m_pkthdr.len; 1088 1089 return m; 1090} 1091 1092int 1093sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa, 1094 struct mbuf *m0, int sbprio) 1095{ 1096 struct mbuf *m, *n, *n0, *nlast; 1097 int error; 1098 1099 KASSERT(solocked(sb->sb_so)); 1100 1101 /* 1102 * XXX sbprio reserved for encoding priority of this* request: 1103 * SB_PRIO_NONE --> honour normal sb limits 1104 * SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space, 1105 * take whole chain. Intended for large requests 1106 * that should be delivered atomically (all, or none). 1107 * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow 1108 * over normal socket limits, for messages indicating 1109 * buffer overflow in earlier normal/lower-priority messages 1110 * SB_PRIO_BESTEFFORT --> ignore limits entirely. 1111 * Intended for kernel-generated messages only. 1112 * Up to generator to avoid total mbuf resource exhaustion. 1113 */ 1114 (void)sbprio; 1115 1116 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 1117 panic("sbappendaddrchain"); 1118 1119#ifdef notyet 1120 space = sbspace(sb); 1121 1122 /* 1123 * Enforce SB_PRIO_* limits as described above. 1124 */ 1125#endif 1126 1127 n0 = NULL; 1128 nlast = NULL; 1129 for (m = m0; m; m = m->m_nextpkt) { 1130 struct mbuf *np; 1131 1132#ifdef MBUFTRACE 1133 m_claimm(m, sb->sb_mowner); 1134#endif 1135 1136 /* Prepend sockaddr to this record (m) of input chain m0 */ 1137 n = m_prepend_sockaddr(sb, m, asa); 1138 if (n == NULL) { 1139 error = ENOBUFS; 1140 goto bad; 1141 } 1142 1143 /* Append record (asa+m) to end of new chain n0 */ 1144 if (n0 == NULL) { 1145 n0 = n; 1146 } else { 1147 nlast->m_nextpkt = n; 1148 } 1149 /* Keep track of last record on new chain */ 1150 nlast = n; 1151 1152 for (np = n; np; np = np->m_next) 1153 sballoc(sb, np); 1154 } 1155 1156 SBLASTRECORDCHK(sb, "sbappendaddrchain 1"); 1157 1158 /* Drop the entire chain of (asa+m) records onto the socket */ 1159 SBLINKRECORDCHAIN(sb, n0, nlast); 1160 1161 SBLASTRECORDCHK(sb, "sbappendaddrchain 2"); 1162 1163 for (m = nlast; m->m_next; m = m->m_next) 1164 ; 1165 sb->sb_mbtail = m; 1166 SBLASTMBUFCHK(sb, "sbappendaddrchain"); 1167 1168 return (1); 1169 1170bad: 1171 /* 1172 * On error, free the prepended addreseses. For consistency 1173 * with sbappendaddr(), leave it to our caller to free 1174 * the input record chain passed to us as m0. 1175 */ 1176 while ((n = n0) != NULL) { 1177 struct mbuf *np; 1178 1179 /* Undo the sballoc() of this record */ 1180 for (np = n; np; np = np->m_next) 1181 sbfree(sb, np); 1182 1183 n0 = n->m_nextpkt; /* iterate at next prepended address */ 1184 np = m_free(n); /* free prepended address (not data) */ 1185 } 1186 return error; 1187} 1188 1189 1190int 1191sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 1192{ 1193 struct mbuf *m, *mlast, *n; 1194 int space; 1195 1196 KASSERT(solocked(sb->sb_so)); 1197 1198 space = 0; 1199 if (control == NULL) 1200 panic("sbappendcontrol"); 1201 for (m = control; ; m = m->m_next) { 1202 space += m->m_len; 1203 MCLAIM(m, sb->sb_mowner); 1204 if (m->m_next == NULL) 1205 break; 1206 } 1207 n = m; /* save pointer to last control buffer */ 1208 for (m = m0; m; m = m->m_next) { 1209 MCLAIM(m, sb->sb_mowner); 1210 space += m->m_len; 1211 } 1212 if (space > sbspace(sb)) 1213 return (0); 1214 n->m_next = m0; /* concatenate data to control */ 1215 1216 SBLASTRECORDCHK(sb, "sbappendcontrol 1"); 1217 1218 for (m = control; m->m_next != NULL; m = m->m_next) 1219 sballoc(sb, m); 1220 sballoc(sb, m); 1221 mlast = m; 1222 SBLINKRECORD(sb, control); 1223 1224 sb->sb_mbtail = mlast; 1225 SBLASTMBUFCHK(sb, "sbappendcontrol"); 1226 SBLASTRECORDCHK(sb, "sbappendcontrol 2"); 1227 1228 return (1); 1229} 1230 1231/* 1232 * Compress mbuf chain m into the socket 1233 * buffer sb following mbuf n. If n 1234 * is null, the buffer is presumed empty. 1235 */ 1236void 1237sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 1238{ 1239 int eor; 1240 struct mbuf *o; 1241 1242 KASSERT(solocked(sb->sb_so)); 1243 1244 eor = 0; 1245 while (m) { 1246 eor |= m->m_flags & M_EOR; 1247 if (m->m_len == 0 && 1248 (eor == 0 || 1249 (((o = m->m_next) || (o = n)) && 1250 o->m_type == m->m_type))) { 1251 if (sb->sb_lastrecord == m) 1252 sb->sb_lastrecord = m->m_next; 1253 m = m_free(m); 1254 continue; 1255 } 1256 if (n && (n->m_flags & M_EOR) == 0 && 1257 /* M_TRAILINGSPACE() checks buffer writeability */ 1258 m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */ 1259 m->m_len <= M_TRAILINGSPACE(n) && 1260 n->m_type == m->m_type) { 1261 memcpy(mtod(n, char *) + n->m_len, mtod(m, void *), 1262 (unsigned)m->m_len); 1263 n->m_len += m->m_len; 1264 sb->sb_cc += m->m_len; 1265 m = m_free(m); 1266 continue; 1267 } 1268 if (n) 1269 n->m_next = m; 1270 else 1271 sb->sb_mb = m; 1272 sb->sb_mbtail = m; 1273 sballoc(sb, m); 1274 n = m; 1275 m->m_flags &= ~M_EOR; 1276 m = m->m_next; 1277 n->m_next = 0; 1278 } 1279 if (eor) { 1280 if (n) 1281 n->m_flags |= eor; 1282 else 1283 printf("semi-panic: sbcompress\n"); 1284 } 1285 SBLASTMBUFCHK(sb, __func__); 1286} 1287 1288/* 1289 * Free all mbufs in a sockbuf. 1290 * Check that all resources are reclaimed. 1291 */ 1292void 1293sbflush(struct sockbuf *sb) 1294{ 1295 1296 KASSERT(solocked(sb->sb_so)); 1297 KASSERT((sb->sb_flags & SB_LOCK) == 0); 1298 1299 while (sb->sb_mbcnt) 1300 sbdrop(sb, (int)sb->sb_cc); 1301 1302 KASSERT(sb->sb_cc == 0); 1303 KASSERT(sb->sb_mb == NULL); 1304 KASSERT(sb->sb_mbtail == NULL); 1305 KASSERT(sb->sb_lastrecord == NULL); 1306} 1307 1308/* 1309 * Drop data from (the front of) a sockbuf. 1310 */ 1311void 1312sbdrop(struct sockbuf *sb, int len) 1313{ 1314 struct mbuf *m, *next; 1315 1316 KASSERT(solocked(sb->sb_so)); 1317 1318 next = (m = sb->sb_mb) ? m->m_nextpkt : NULL; 1319 while (len > 0) { 1320 if (m == NULL) { 1321 if (next == NULL) 1322 panic("sbdrop(%p,%d): cc=%lu", 1323 sb, len, sb->sb_cc); 1324 m = next; 1325 next = m->m_nextpkt; 1326 continue; 1327 } 1328 if (m->m_len > len) { 1329 m->m_len -= len; 1330 m->m_data += len; 1331 sb->sb_cc -= len; 1332 break; 1333 } 1334 len -= m->m_len; 1335 sbfree(sb, m); 1336 m = m_free(m); 1337 } 1338 while (m && m->m_len == 0) { 1339 sbfree(sb, m); 1340 m = m_free(m); 1341 } 1342 if (m) { 1343 sb->sb_mb = m; 1344 m->m_nextpkt = next; 1345 } else 1346 sb->sb_mb = next; 1347 /* 1348 * First part is an inline SB_EMPTY_FIXUP(). Second part 1349 * makes sure sb_lastrecord is up-to-date if we dropped 1350 * part of the last record. 1351 */ 1352 m = sb->sb_mb; 1353 if (m == NULL) { 1354 sb->sb_mbtail = NULL; 1355 sb->sb_lastrecord = NULL; 1356 } else if (m->m_nextpkt == NULL) 1357 sb->sb_lastrecord = m; 1358} 1359 1360/* 1361 * Drop a record off the front of a sockbuf 1362 * and move the next record to the front. 1363 */ 1364void 1365sbdroprecord(struct sockbuf *sb) 1366{ 1367 struct mbuf *m, *mn; 1368 1369 KASSERT(solocked(sb->sb_so)); 1370 1371 m = sb->sb_mb; 1372 if (m) { 1373 sb->sb_mb = m->m_nextpkt; 1374 do { 1375 sbfree(sb, m); 1376 mn = m_free(m); 1377 } while ((m = mn) != NULL); 1378 } 1379 SB_EMPTY_FIXUP(sb); 1380} 1381 1382/* 1383 * Create a "control" mbuf containing the specified data 1384 * with the specified type for presentation on a socket buffer. 1385 */ 1386struct mbuf * 1387sbcreatecontrol1(void **p, int size, int type, int level, int flags) 1388{ 1389 struct cmsghdr *cp; 1390 struct mbuf *m; 1391 int space = CMSG_SPACE(size); 1392 1393 if ((flags & M_DONTWAIT) && space > MCLBYTES) { 1394 printf("%s: message too large %d\n", __func__, space); 1395 return NULL; 1396 } 1397 1398 if ((m = m_get(flags, MT_CONTROL)) == NULL) 1399 return NULL; 1400 if (space > MLEN) { 1401 if (space > MCLBYTES) 1402 MEXTMALLOC(m, space, M_WAITOK); 1403 else 1404 MCLGET(m, flags); 1405 if ((m->m_flags & M_EXT) == 0) { 1406 m_free(m); 1407 return NULL; 1408 } 1409 } 1410 cp = mtod(m, struct cmsghdr *); 1411 *p = CMSG_DATA(cp); 1412 m->m_len = space; 1413 cp->cmsg_len = CMSG_LEN(size); 1414 cp->cmsg_level = level; 1415 cp->cmsg_type = type; 1416 return m; 1417} 1418 1419struct mbuf * 1420sbcreatecontrol(void *p, int size, int type, int level) 1421{ 1422 struct mbuf *m; 1423 void *v; 1424 1425 m = sbcreatecontrol1(&v, size, type, level, M_DONTWAIT); 1426 if (m == NULL) 1427 return NULL; 1428 memcpy(v, p, size); 1429 return m; 1430} 1431 1432void 1433solockretry(struct socket *so, kmutex_t *lock) 1434{ 1435 1436 while (lock != so->so_lock) { 1437 mutex_exit(lock); 1438 lock = so->so_lock; 1439 mutex_enter(lock); 1440 } 1441} 1442 1443bool 1444solocked(const struct socket *so) 1445{ 1446 1447 return mutex_owned(so->so_lock); 1448} 1449 1450bool 1451solocked2(const struct socket *so1, const struct socket *so2) 1452{ 1453 const kmutex_t *lock; 1454 1455 lock = so1->so_lock; 1456 if (lock != so2->so_lock) 1457 return false; 1458 return mutex_owned(lock); 1459} 1460 1461/* 1462 * sosetlock: assign a default lock to a new socket. 1463 */ 1464void 1465sosetlock(struct socket *so) 1466{ 1467 if (so->so_lock == NULL) { 1468 kmutex_t *lock = softnet_lock; 1469 1470 so->so_lock = lock; 1471 mutex_obj_hold(lock); 1472 mutex_enter(lock); 1473 } 1474 KASSERT(solocked(so)); 1475} 1476 1477/* 1478 * Set lock on sockbuf sb; sleep if lock is already held. 1479 * Unless SB_NOINTR is set on sockbuf, sleep is interruptible. 1480 * Returns error without lock if sleep is interrupted. 1481 */ 1482int 1483sblock(struct sockbuf *sb, int wf) 1484{ 1485 struct socket *so; 1486 kmutex_t *lock; 1487 int error; 1488 1489 KASSERT(solocked(sb->sb_so)); 1490 1491 for (;;) { 1492 if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) { 1493 sb->sb_flags |= SB_LOCK; 1494 return 0; 1495 } 1496 if (wf != M_WAITOK) 1497 return EWOULDBLOCK; 1498 so = sb->sb_so; 1499 lock = so->so_lock; 1500 if ((sb->sb_flags & SB_NOINTR) != 0) { 1501 cv_wait(&so->so_cv, lock); 1502 error = 0; 1503 } else 1504 error = cv_wait_sig(&so->so_cv, lock); 1505 if (__predict_false(lock != so->so_lock)) 1506 solockretry(so, lock); 1507 if (error != 0) 1508 return error; 1509 } 1510} 1511 1512void 1513sbunlock(struct sockbuf *sb) 1514{ 1515 struct socket *so; 1516 1517 so = sb->sb_so; 1518 1519 KASSERT(solocked(so)); 1520 KASSERT((sb->sb_flags & SB_LOCK) != 0); 1521 1522 sb->sb_flags &= ~SB_LOCK; 1523 cv_broadcast(&so->so_cv); 1524} 1525 1526int 1527sowait(struct socket *so, bool catch_p, int timo) 1528{ 1529 kmutex_t *lock; 1530 int error; 1531 1532 KASSERT(solocked(so)); 1533 KASSERT(catch_p || timo != 0); 1534 1535 lock = so->so_lock; 1536 if (catch_p) 1537 error = cv_timedwait_sig(&so->so_cv, lock, timo); 1538 else 1539 error = cv_timedwait(&so->so_cv, lock, timo); 1540 if (__predict_false(lock != so->so_lock)) 1541 solockretry(so, lock); 1542 return error; 1543} 1544