uipc_socket2.c revision 1.117
1/* $NetBSD: uipc_socket2.c,v 1.117 2014/05/17 23:55:24 rmind Exp $ */ 2 3/*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * Copyright (c) 1982, 1986, 1988, 1990, 1993 31 * The Regents of the University of California. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. Neither the name of the University nor the names of its contributors 42 * may be used to endorse or promote products derived from this software 43 * without specific prior written permission. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * @(#)uipc_socket2.c 8.2 (Berkeley) 2/14/95 58 */ 59 60#include <sys/cdefs.h> 61__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.117 2014/05/17 23:55:24 rmind Exp $"); 62 63#include "opt_mbuftrace.h" 64#include "opt_sb_max.h" 65 66#include <sys/param.h> 67#include <sys/systm.h> 68#include <sys/proc.h> 69#include <sys/file.h> 70#include <sys/buf.h> 71#include <sys/mbuf.h> 72#include <sys/protosw.h> 73#include <sys/domain.h> 74#include <sys/poll.h> 75#include <sys/socket.h> 76#include <sys/socketvar.h> 77#include <sys/signalvar.h> 78#include <sys/kauth.h> 79#include <sys/pool.h> 80#include <sys/uidinfo.h> 81 82/* 83 * Primitive routines for operating on sockets and socket buffers. 84 * 85 * Connection life-cycle: 86 * 87 * Normal sequence from the active (originating) side: 88 * 89 * - soisconnecting() is called during processing of connect() call, 90 * - resulting in an eventual call to soisconnected() if/when the 91 * connection is established. 92 * 93 * When the connection is torn down during processing of disconnect(): 94 * 95 * - soisdisconnecting() is called and, 96 * - soisdisconnected() is called when the connection to the peer 97 * is totally severed. 98 * 99 * The semantics of these routines are such that connectionless protocols 100 * can call soisconnected() and soisdisconnected() only, bypassing the 101 * in-progress calls when setting up a ``connection'' takes no time. 102 * 103 * From the passive side, a socket is created with two queues of sockets: 104 * 105 * - so_q0 (0) for partial connections (i.e. connections in progress) 106 * - so_q (1) for connections already made and awaiting user acceptance. 107 * 108 * As a protocol is preparing incoming connections, it creates a socket 109 * structure queued on so_q0 by calling sonewconn(). When the connection 110 * is established, soisconnected() is called, and transfers the 111 * socket structure to so_q, making it available to accept(). 112 * 113 * If a socket is closed with sockets on either so_q0 or so_q, these 114 * sockets are dropped. 115 * 116 * Locking rules and assumptions: 117 * 118 * o socket::so_lock can change on the fly. The low level routines used 119 * to lock sockets are aware of this. When so_lock is acquired, the 120 * routine locking must check to see if so_lock still points to the 121 * lock that was acquired. If so_lock has changed in the meantime, the 122 * now irrelevant lock that was acquired must be dropped and the lock 123 * operation retried. Although not proven here, this is completely safe 124 * on a multiprocessor system, even with relaxed memory ordering, given 125 * the next two rules: 126 * 127 * o In order to mutate so_lock, the lock pointed to by the current value 128 * of so_lock must be held: i.e., the socket must be held locked by the 129 * changing thread. The thread must issue membar_exit() to prevent 130 * memory accesses being reordered, and can set so_lock to the desired 131 * value. If the lock pointed to by the new value of so_lock is not 132 * held by the changing thread, the socket must then be considered 133 * unlocked. 134 * 135 * o If so_lock is mutated, and the previous lock referred to by so_lock 136 * could still be visible to other threads in the system (e.g. via file 137 * descriptor or protocol-internal reference), then the old lock must 138 * remain valid until the socket and/or protocol control block has been 139 * torn down. 140 * 141 * o If a socket has a non-NULL so_head value (i.e. is in the process of 142 * connecting), then locking the socket must also lock the socket pointed 143 * to by so_head: their lock pointers must match. 144 * 145 * o If a socket has connections in progress (so_q, so_q0 not empty) then 146 * locking the socket must also lock the sockets attached to both queues. 147 * Again, their lock pointers must match. 148 * 149 * o Beyond the initial lock assignment in socreate(), assigning locks to 150 * sockets is the responsibility of the individual protocols / protocol 151 * domains. 152 */ 153 154static pool_cache_t socket_cache; 155u_long sb_max = SB_MAX;/* maximum socket buffer size */ 156static u_long sb_max_adj; /* adjusted sb_max */ 157 158void 159soisconnecting(struct socket *so) 160{ 161 162 KASSERT(solocked(so)); 163 164 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 165 so->so_state |= SS_ISCONNECTING; 166} 167 168void 169soisconnected(struct socket *so) 170{ 171 struct socket *head; 172 173 head = so->so_head; 174 175 KASSERT(solocked(so)); 176 KASSERT(head == NULL || solocked2(so, head)); 177 178 so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING); 179 so->so_state |= SS_ISCONNECTED; 180 if (head && so->so_onq == &head->so_q0) { 181 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 182 /* 183 * Re-enqueue and wake up any waiters, e.g. 184 * processes blocking on accept(). 185 */ 186 soqremque(so, 0); 187 soqinsque(head, so, 1); 188 sorwakeup(head); 189 cv_broadcast(&head->so_cv); 190 } else { 191 so->so_upcall = 192 head->so_accf->so_accept_filter->accf_callback; 193 so->so_upcallarg = head->so_accf->so_accept_filter_arg; 194 so->so_rcv.sb_flags |= SB_UPCALL; 195 so->so_options &= ~SO_ACCEPTFILTER; 196 (*so->so_upcall)(so, so->so_upcallarg, 197 POLLIN|POLLRDNORM, M_DONTWAIT); 198 } 199 } else { 200 cv_broadcast(&so->so_cv); 201 sorwakeup(so); 202 sowwakeup(so); 203 } 204} 205 206void 207soisdisconnecting(struct socket *so) 208{ 209 210 KASSERT(solocked(so)); 211 212 so->so_state &= ~SS_ISCONNECTING; 213 so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE); 214 cv_broadcast(&so->so_cv); 215 sowwakeup(so); 216 sorwakeup(so); 217} 218 219void 220soisdisconnected(struct socket *so) 221{ 222 223 KASSERT(solocked(so)); 224 225 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 226 so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED); 227 cv_broadcast(&so->so_cv); 228 sowwakeup(so); 229 sorwakeup(so); 230} 231 232void 233soinit2(void) 234{ 235 236 socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0, 237 "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL); 238} 239 240/* 241 * sonewconn: accept a new connection. 242 * 243 * When an attempt at a new connection is noted on a socket which accepts 244 * connections, sonewconn(9) is called. If the connection is possible 245 * (subject to space constraints, etc) then we allocate a new structure, 246 * properly linked into the data structure of the original socket. 247 * 248 * => If 'soready' is true, then socket will become ready for accept() i.e. 249 * inserted into the so_q queue, SS_ISCONNECTED set and waiters awoken. 250 * => May be called from soft-interrupt context. 251 * => Listening socket should be locked. 252 * => Returns the new socket locked. 253 */ 254struct socket * 255sonewconn(struct socket *head, bool soready) 256{ 257 struct socket *so; 258 int soqueue, error; 259 260 KASSERT(solocked(head)); 261 262 if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) { 263 /* Listen queue overflow. */ 264 return NULL; 265 } 266 if ((head->so_options & SO_ACCEPTFILTER) != 0) { 267 soready = false; 268 } 269 soqueue = soready ? 1 : 0; 270 271 if ((so = soget(false)) == NULL) { 272 return NULL; 273 } 274 so->so_type = head->so_type; 275 so->so_options = head->so_options & ~SO_ACCEPTCONN; 276 so->so_linger = head->so_linger; 277 so->so_state = head->so_state | SS_NOFDREF; 278 so->so_proto = head->so_proto; 279 so->so_timeo = head->so_timeo; 280 so->so_pgid = head->so_pgid; 281 so->so_send = head->so_send; 282 so->so_receive = head->so_receive; 283 so->so_uidinfo = head->so_uidinfo; 284 so->so_cpid = head->so_cpid; 285#ifdef MBUFTRACE 286 so->so_mowner = head->so_mowner; 287 so->so_rcv.sb_mowner = head->so_rcv.sb_mowner; 288 so->so_snd.sb_mowner = head->so_snd.sb_mowner; 289#endif 290 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) != 0) 291 goto out; 292 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 293 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 294 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 295 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 296 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 297 so->so_snd.sb_flags |= head->so_snd.sb_flags & (SB_AUTOSIZE | SB_ASYNC); 298 299 /* 300 * Share the lock with the listening-socket, it may get unshared 301 * once the connection is complete. 302 */ 303 mutex_obj_hold(head->so_lock); 304 so->so_lock = head->so_lock; 305 306 error = (*so->so_proto->pr_usrreq)(so, PRU_ATTACH, NULL, NULL, 307 NULL, NULL); 308 KASSERT(solocked(so)); 309 if (error) { 310out: 311 KASSERT(so->so_accf == NULL); 312 soput(so); 313 314 /* Note: the listening socket shall stay locked. */ 315 KASSERT(solocked(head)); 316 return NULL; 317 } 318 319 /* 320 * Insert into the queue. If ready, update the connection status 321 * and wake up any waiters, e.g. processes blocking on accept(). 322 */ 323 soqinsque(head, so, soqueue); 324 if (soready) { 325 so->so_state |= SS_ISCONNECTED; 326 sorwakeup(head); 327 cv_broadcast(&head->so_cv); 328 } 329 KASSERT(solocked2(head, so)); 330 return so; 331} 332 333struct socket * 334soget(bool waitok) 335{ 336 struct socket *so; 337 338 so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT)); 339 if (__predict_false(so == NULL)) 340 return (NULL); 341 memset(so, 0, sizeof(*so)); 342 TAILQ_INIT(&so->so_q0); 343 TAILQ_INIT(&so->so_q); 344 cv_init(&so->so_cv, "socket"); 345 cv_init(&so->so_rcv.sb_cv, "netio"); 346 cv_init(&so->so_snd.sb_cv, "netio"); 347 selinit(&so->so_rcv.sb_sel); 348 selinit(&so->so_snd.sb_sel); 349 so->so_rcv.sb_so = so; 350 so->so_snd.sb_so = so; 351 return so; 352} 353 354void 355soput(struct socket *so) 356{ 357 358 KASSERT(!cv_has_waiters(&so->so_cv)); 359 KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv)); 360 KASSERT(!cv_has_waiters(&so->so_snd.sb_cv)); 361 seldestroy(&so->so_rcv.sb_sel); 362 seldestroy(&so->so_snd.sb_sel); 363 mutex_obj_free(so->so_lock); 364 cv_destroy(&so->so_cv); 365 cv_destroy(&so->so_rcv.sb_cv); 366 cv_destroy(&so->so_snd.sb_cv); 367 pool_cache_put(socket_cache, so); 368} 369 370/* 371 * soqinsque: insert socket of a new connection into the specified 372 * accept queue of the listening socket (head). 373 * 374 * q = 0: queue of partial connections 375 * q = 1: queue of incoming connections 376 */ 377void 378soqinsque(struct socket *head, struct socket *so, int q) 379{ 380 KASSERT(q == 0 || q == 1); 381 KASSERT(solocked2(head, so)); 382 KASSERT(so->so_onq == NULL); 383 KASSERT(so->so_head == NULL); 384 385 so->so_head = head; 386 if (q == 0) { 387 head->so_q0len++; 388 so->so_onq = &head->so_q0; 389 } else { 390 head->so_qlen++; 391 so->so_onq = &head->so_q; 392 } 393 TAILQ_INSERT_TAIL(so->so_onq, so, so_qe); 394} 395 396/* 397 * soqremque: remove socket from the specified queue. 398 * 399 * => Returns true if socket was removed from the specified queue. 400 * => False if socket was not removed (because it was in other queue). 401 */ 402bool 403soqremque(struct socket *so, int q) 404{ 405 struct socket *head = so->so_head; 406 407 KASSERT(q == 0 || q == 1); 408 KASSERT(solocked(so)); 409 KASSERT(so->so_onq != NULL); 410 KASSERT(head != NULL); 411 412 if (q == 0) { 413 if (so->so_onq != &head->so_q0) 414 return false; 415 head->so_q0len--; 416 } else { 417 if (so->so_onq != &head->so_q) 418 return false; 419 head->so_qlen--; 420 } 421 KASSERT(solocked2(so, head)); 422 TAILQ_REMOVE(so->so_onq, so, so_qe); 423 so->so_onq = NULL; 424 so->so_head = NULL; 425 return true; 426} 427 428/* 429 * socantsendmore: indicates that no more data will be sent on the 430 * socket; it would normally be applied to a socket when the user 431 * informs the system that no more data is to be sent, by the protocol 432 * code (in case PRU_SHUTDOWN). 433 */ 434void 435socantsendmore(struct socket *so) 436{ 437 KASSERT(solocked(so)); 438 439 so->so_state |= SS_CANTSENDMORE; 440 sowwakeup(so); 441} 442 443/* 444 * socantrcvmore(): indicates that no more data will be received and 445 * will normally be applied to the socket by a protocol when it detects 446 * that the peer will send no more data. Data queued for reading in 447 * the socket may yet be read. 448 */ 449void 450socantrcvmore(struct socket *so) 451{ 452 KASSERT(solocked(so)); 453 454 so->so_state |= SS_CANTRCVMORE; 455 sorwakeup(so); 456} 457 458/* 459 * Wait for data to arrive at/drain from a socket buffer. 460 */ 461int 462sbwait(struct sockbuf *sb) 463{ 464 struct socket *so; 465 kmutex_t *lock; 466 int error; 467 468 so = sb->sb_so; 469 470 KASSERT(solocked(so)); 471 472 sb->sb_flags |= SB_NOTIFY; 473 lock = so->so_lock; 474 if ((sb->sb_flags & SB_NOINTR) != 0) 475 error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo); 476 else 477 error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo); 478 if (__predict_false(lock != so->so_lock)) 479 solockretry(so, lock); 480 return error; 481} 482 483/* 484 * Wakeup processes waiting on a socket buffer. 485 * Do asynchronous notification via SIGIO 486 * if the socket buffer has the SB_ASYNC flag set. 487 */ 488void 489sowakeup(struct socket *so, struct sockbuf *sb, int code) 490{ 491 int band; 492 493 KASSERT(solocked(so)); 494 KASSERT(sb->sb_so == so); 495 496 if (code == POLL_IN) 497 band = POLLIN|POLLRDNORM; 498 else 499 band = POLLOUT|POLLWRNORM; 500 sb->sb_flags &= ~SB_NOTIFY; 501 selnotify(&sb->sb_sel, band, NOTE_SUBMIT); 502 cv_broadcast(&sb->sb_cv); 503 if (sb->sb_flags & SB_ASYNC) 504 fownsignal(so->so_pgid, SIGIO, code, band, so); 505 if (sb->sb_flags & SB_UPCALL) 506 (*so->so_upcall)(so, so->so_upcallarg, band, M_DONTWAIT); 507} 508 509/* 510 * Reset a socket's lock pointer. Wake all threads waiting on the 511 * socket's condition variables so that they can restart their waits 512 * using the new lock. The existing lock must be held. 513 */ 514void 515solockreset(struct socket *so, kmutex_t *lock) 516{ 517 518 KASSERT(solocked(so)); 519 520 so->so_lock = lock; 521 cv_broadcast(&so->so_snd.sb_cv); 522 cv_broadcast(&so->so_rcv.sb_cv); 523 cv_broadcast(&so->so_cv); 524} 525 526/* 527 * Socket buffer (struct sockbuf) utility routines. 528 * 529 * Each socket contains two socket buffers: one for sending data and 530 * one for receiving data. Each buffer contains a queue of mbufs, 531 * information about the number of mbufs and amount of data in the 532 * queue, and other fields allowing poll() statements and notification 533 * on data availability to be implemented. 534 * 535 * Data stored in a socket buffer is maintained as a list of records. 536 * Each record is a list of mbufs chained together with the m_next 537 * field. Records are chained together with the m_nextpkt field. The upper 538 * level routine soreceive() expects the following conventions to be 539 * observed when placing information in the receive buffer: 540 * 541 * 1. If the protocol requires each message be preceded by the sender's 542 * name, then a record containing that name must be present before 543 * any associated data (mbuf's must be of type MT_SONAME). 544 * 2. If the protocol supports the exchange of ``access rights'' (really 545 * just additional data associated with the message), and there are 546 * ``rights'' to be received, then a record containing this data 547 * should be present (mbuf's must be of type MT_CONTROL). 548 * 3. If a name or rights record exists, then it must be followed by 549 * a data record, perhaps of zero length. 550 * 551 * Before using a new socket structure it is first necessary to reserve 552 * buffer space to the socket, by calling sbreserve(). This should commit 553 * some of the available buffer space in the system buffer pool for the 554 * socket (currently, it does nothing but enforce limits). The space 555 * should be released by calling sbrelease() when the socket is destroyed. 556 */ 557 558int 559sb_max_set(u_long new_sbmax) 560{ 561 int s; 562 563 if (new_sbmax < (16 * 1024)) 564 return (EINVAL); 565 566 s = splsoftnet(); 567 sb_max = new_sbmax; 568 sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES); 569 splx(s); 570 571 return (0); 572} 573 574int 575soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 576{ 577 KASSERT(so->so_pcb == NULL || solocked(so)); 578 579 /* 580 * there's at least one application (a configure script of screen) 581 * which expects a fifo is writable even if it has "some" bytes 582 * in its buffer. 583 * so we want to make sure (hiwat - lowat) >= (some bytes). 584 * 585 * PIPE_BUF here is an arbitrary value chosen as (some bytes) above. 586 * we expect it's large enough for such applications. 587 */ 588 u_long lowat = MAX(sock_loan_thresh, MCLBYTES); 589 u_long hiwat = lowat + PIPE_BUF; 590 591 if (sndcc < hiwat) 592 sndcc = hiwat; 593 if (sbreserve(&so->so_snd, sndcc, so) == 0) 594 goto bad; 595 if (sbreserve(&so->so_rcv, rcvcc, so) == 0) 596 goto bad2; 597 if (so->so_rcv.sb_lowat == 0) 598 so->so_rcv.sb_lowat = 1; 599 if (so->so_snd.sb_lowat == 0) 600 so->so_snd.sb_lowat = lowat; 601 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 602 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 603 return (0); 604 bad2: 605 sbrelease(&so->so_snd, so); 606 bad: 607 return (ENOBUFS); 608} 609 610/* 611 * Allot mbufs to a sockbuf. 612 * Attempt to scale mbmax so that mbcnt doesn't become limiting 613 * if buffering efficiency is near the normal case. 614 */ 615int 616sbreserve(struct sockbuf *sb, u_long cc, struct socket *so) 617{ 618 struct lwp *l = curlwp; /* XXX */ 619 rlim_t maxcc; 620 struct uidinfo *uidinfo; 621 622 KASSERT(so->so_pcb == NULL || solocked(so)); 623 KASSERT(sb->sb_so == so); 624 KASSERT(sb_max_adj != 0); 625 626 if (cc == 0 || cc > sb_max_adj) 627 return (0); 628 629 maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur; 630 631 uidinfo = so->so_uidinfo; 632 if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc)) 633 return 0; 634 sb->sb_mbmax = min(cc * 2, sb_max); 635 if (sb->sb_lowat > sb->sb_hiwat) 636 sb->sb_lowat = sb->sb_hiwat; 637 return (1); 638} 639 640/* 641 * Free mbufs held by a socket, and reserved mbuf space. We do not assert 642 * that the socket is held locked here: see sorflush(). 643 */ 644void 645sbrelease(struct sockbuf *sb, struct socket *so) 646{ 647 648 KASSERT(sb->sb_so == so); 649 650 sbflush(sb); 651 (void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY); 652 sb->sb_mbmax = 0; 653} 654 655/* 656 * Routines to add and remove 657 * data from an mbuf queue. 658 * 659 * The routines sbappend() or sbappendrecord() are normally called to 660 * append new mbufs to a socket buffer, after checking that adequate 661 * space is available, comparing the function sbspace() with the amount 662 * of data to be added. sbappendrecord() differs from sbappend() in 663 * that data supplied is treated as the beginning of a new record. 664 * To place a sender's address, optional access rights, and data in a 665 * socket receive buffer, sbappendaddr() should be used. To place 666 * access rights and data in a socket receive buffer, sbappendrights() 667 * should be used. In either case, the new data begins a new record. 668 * Note that unlike sbappend() and sbappendrecord(), these routines check 669 * for the caller that there will be enough space to store the data. 670 * Each fails if there is not enough space, or if it cannot find mbufs 671 * to store additional information in. 672 * 673 * Reliable protocols may use the socket send buffer to hold data 674 * awaiting acknowledgement. Data is normally copied from a socket 675 * send buffer in a protocol with m_copy for output to a peer, 676 * and then removing the data from the socket buffer with sbdrop() 677 * or sbdroprecord() when the data is acknowledged by the peer. 678 */ 679 680#ifdef SOCKBUF_DEBUG 681void 682sblastrecordchk(struct sockbuf *sb, const char *where) 683{ 684 struct mbuf *m = sb->sb_mb; 685 686 KASSERT(solocked(sb->sb_so)); 687 688 while (m && m->m_nextpkt) 689 m = m->m_nextpkt; 690 691 if (m != sb->sb_lastrecord) { 692 printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n", 693 sb->sb_mb, sb->sb_lastrecord, m); 694 printf("packet chain:\n"); 695 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 696 printf("\t%p\n", m); 697 panic("sblastrecordchk from %s", where); 698 } 699} 700 701void 702sblastmbufchk(struct sockbuf *sb, const char *where) 703{ 704 struct mbuf *m = sb->sb_mb; 705 struct mbuf *n; 706 707 KASSERT(solocked(sb->sb_so)); 708 709 while (m && m->m_nextpkt) 710 m = m->m_nextpkt; 711 712 while (m && m->m_next) 713 m = m->m_next; 714 715 if (m != sb->sb_mbtail) { 716 printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n", 717 sb->sb_mb, sb->sb_mbtail, m); 718 printf("packet tree:\n"); 719 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 720 printf("\t"); 721 for (n = m; n != NULL; n = n->m_next) 722 printf("%p ", n); 723 printf("\n"); 724 } 725 panic("sblastmbufchk from %s", where); 726 } 727} 728#endif /* SOCKBUF_DEBUG */ 729 730/* 731 * Link a chain of records onto a socket buffer 732 */ 733#define SBLINKRECORDCHAIN(sb, m0, mlast) \ 734do { \ 735 if ((sb)->sb_lastrecord != NULL) \ 736 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 737 else \ 738 (sb)->sb_mb = (m0); \ 739 (sb)->sb_lastrecord = (mlast); \ 740} while (/*CONSTCOND*/0) 741 742 743#define SBLINKRECORD(sb, m0) \ 744 SBLINKRECORDCHAIN(sb, m0, m0) 745 746/* 747 * Append mbuf chain m to the last record in the 748 * socket buffer sb. The additional space associated 749 * the mbuf chain is recorded in sb. Empty mbufs are 750 * discarded and mbufs are compacted where possible. 751 */ 752void 753sbappend(struct sockbuf *sb, struct mbuf *m) 754{ 755 struct mbuf *n; 756 757 KASSERT(solocked(sb->sb_so)); 758 759 if (m == NULL) 760 return; 761 762#ifdef MBUFTRACE 763 m_claimm(m, sb->sb_mowner); 764#endif 765 766 SBLASTRECORDCHK(sb, "sbappend 1"); 767 768 if ((n = sb->sb_lastrecord) != NULL) { 769 /* 770 * XXX Would like to simply use sb_mbtail here, but 771 * XXX I need to verify that I won't miss an EOR that 772 * XXX way. 773 */ 774 do { 775 if (n->m_flags & M_EOR) { 776 sbappendrecord(sb, m); /* XXXXXX!!!! */ 777 return; 778 } 779 } while (n->m_next && (n = n->m_next)); 780 } else { 781 /* 782 * If this is the first record in the socket buffer, it's 783 * also the last record. 784 */ 785 sb->sb_lastrecord = m; 786 } 787 sbcompress(sb, m, n); 788 SBLASTRECORDCHK(sb, "sbappend 2"); 789} 790 791/* 792 * This version of sbappend() should only be used when the caller 793 * absolutely knows that there will never be more than one record 794 * in the socket buffer, that is, a stream protocol (such as TCP). 795 */ 796void 797sbappendstream(struct sockbuf *sb, struct mbuf *m) 798{ 799 800 KASSERT(solocked(sb->sb_so)); 801 KDASSERT(m->m_nextpkt == NULL); 802 KASSERT(sb->sb_mb == sb->sb_lastrecord); 803 804 SBLASTMBUFCHK(sb, __func__); 805 806#ifdef MBUFTRACE 807 m_claimm(m, sb->sb_mowner); 808#endif 809 810 sbcompress(sb, m, sb->sb_mbtail); 811 812 sb->sb_lastrecord = sb->sb_mb; 813 SBLASTRECORDCHK(sb, __func__); 814} 815 816#ifdef SOCKBUF_DEBUG 817void 818sbcheck(struct sockbuf *sb) 819{ 820 struct mbuf *m, *m2; 821 u_long len, mbcnt; 822 823 KASSERT(solocked(sb->sb_so)); 824 825 len = 0; 826 mbcnt = 0; 827 for (m = sb->sb_mb; m; m = m->m_nextpkt) { 828 for (m2 = m; m2 != NULL; m2 = m2->m_next) { 829 len += m2->m_len; 830 mbcnt += MSIZE; 831 if (m2->m_flags & M_EXT) 832 mbcnt += m2->m_ext.ext_size; 833 if (m2->m_nextpkt != NULL) 834 panic("sbcheck nextpkt"); 835 } 836 } 837 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 838 printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc, 839 mbcnt, sb->sb_mbcnt); 840 panic("sbcheck"); 841 } 842} 843#endif 844 845/* 846 * As above, except the mbuf chain 847 * begins a new record. 848 */ 849void 850sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 851{ 852 struct mbuf *m; 853 854 KASSERT(solocked(sb->sb_so)); 855 856 if (m0 == NULL) 857 return; 858 859#ifdef MBUFTRACE 860 m_claimm(m0, sb->sb_mowner); 861#endif 862 /* 863 * Put the first mbuf on the queue. 864 * Note this permits zero length records. 865 */ 866 sballoc(sb, m0); 867 SBLASTRECORDCHK(sb, "sbappendrecord 1"); 868 SBLINKRECORD(sb, m0); 869 m = m0->m_next; 870 m0->m_next = 0; 871 if (m && (m0->m_flags & M_EOR)) { 872 m0->m_flags &= ~M_EOR; 873 m->m_flags |= M_EOR; 874 } 875 sbcompress(sb, m, m0); 876 SBLASTRECORDCHK(sb, "sbappendrecord 2"); 877} 878 879/* 880 * As above except that OOB data 881 * is inserted at the beginning of the sockbuf, 882 * but after any other OOB data. 883 */ 884void 885sbinsertoob(struct sockbuf *sb, struct mbuf *m0) 886{ 887 struct mbuf *m, **mp; 888 889 KASSERT(solocked(sb->sb_so)); 890 891 if (m0 == NULL) 892 return; 893 894 SBLASTRECORDCHK(sb, "sbinsertoob 1"); 895 896 for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) { 897 again: 898 switch (m->m_type) { 899 900 case MT_OOBDATA: 901 continue; /* WANT next train */ 902 903 case MT_CONTROL: 904 if ((m = m->m_next) != NULL) 905 goto again; /* inspect THIS train further */ 906 } 907 break; 908 } 909 /* 910 * Put the first mbuf on the queue. 911 * Note this permits zero length records. 912 */ 913 sballoc(sb, m0); 914 m0->m_nextpkt = *mp; 915 if (*mp == NULL) { 916 /* m0 is actually the new tail */ 917 sb->sb_lastrecord = m0; 918 } 919 *mp = m0; 920 m = m0->m_next; 921 m0->m_next = 0; 922 if (m && (m0->m_flags & M_EOR)) { 923 m0->m_flags &= ~M_EOR; 924 m->m_flags |= M_EOR; 925 } 926 sbcompress(sb, m, m0); 927 SBLASTRECORDCHK(sb, "sbinsertoob 2"); 928} 929 930/* 931 * Append address and data, and optionally, control (ancillary) data 932 * to the receive queue of a socket. If present, 933 * m0 must include a packet header with total length. 934 * Returns 0 if no space in sockbuf or insufficient mbufs. 935 */ 936int 937sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, 938 struct mbuf *control) 939{ 940 struct mbuf *m, *n, *nlast; 941 int space, len; 942 943 KASSERT(solocked(sb->sb_so)); 944 945 space = asa->sa_len; 946 947 if (m0 != NULL) { 948 if ((m0->m_flags & M_PKTHDR) == 0) 949 panic("sbappendaddr"); 950 space += m0->m_pkthdr.len; 951#ifdef MBUFTRACE 952 m_claimm(m0, sb->sb_mowner); 953#endif 954 } 955 for (n = control; n; n = n->m_next) { 956 space += n->m_len; 957 MCLAIM(n, sb->sb_mowner); 958 if (n->m_next == NULL) /* keep pointer to last control buf */ 959 break; 960 } 961 if (space > sbspace(sb)) 962 return (0); 963 m = m_get(M_DONTWAIT, MT_SONAME); 964 if (m == NULL) 965 return (0); 966 MCLAIM(m, sb->sb_mowner); 967 /* 968 * XXX avoid 'comparison always true' warning which isn't easily 969 * avoided. 970 */ 971 len = asa->sa_len; 972 if (len > MLEN) { 973 MEXTMALLOC(m, asa->sa_len, M_NOWAIT); 974 if ((m->m_flags & M_EXT) == 0) { 975 m_free(m); 976 return (0); 977 } 978 } 979 m->m_len = asa->sa_len; 980 memcpy(mtod(m, void *), asa, asa->sa_len); 981 if (n) 982 n->m_next = m0; /* concatenate data to control */ 983 else 984 control = m0; 985 m->m_next = control; 986 987 SBLASTRECORDCHK(sb, "sbappendaddr 1"); 988 989 for (n = m; n->m_next != NULL; n = n->m_next) 990 sballoc(sb, n); 991 sballoc(sb, n); 992 nlast = n; 993 SBLINKRECORD(sb, m); 994 995 sb->sb_mbtail = nlast; 996 SBLASTMBUFCHK(sb, "sbappendaddr"); 997 SBLASTRECORDCHK(sb, "sbappendaddr 2"); 998 999 return (1); 1000} 1001 1002/* 1003 * Helper for sbappendchainaddr: prepend a struct sockaddr* to 1004 * an mbuf chain. 1005 */ 1006static inline struct mbuf * 1007m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0, 1008 const struct sockaddr *asa) 1009{ 1010 struct mbuf *m; 1011 const int salen = asa->sa_len; 1012 1013 KASSERT(solocked(sb->sb_so)); 1014 1015 /* only the first in each chain need be a pkthdr */ 1016 m = m_gethdr(M_DONTWAIT, MT_SONAME); 1017 if (m == NULL) 1018 return NULL; 1019 MCLAIM(m, sb->sb_mowner); 1020#ifdef notyet 1021 if (salen > MHLEN) { 1022 MEXTMALLOC(m, salen, M_NOWAIT); 1023 if ((m->m_flags & M_EXT) == 0) { 1024 m_free(m); 1025 return NULL; 1026 } 1027 } 1028#else 1029 KASSERT(salen <= MHLEN); 1030#endif 1031 m->m_len = salen; 1032 memcpy(mtod(m, void *), asa, salen); 1033 m->m_next = m0; 1034 m->m_pkthdr.len = salen + m0->m_pkthdr.len; 1035 1036 return m; 1037} 1038 1039int 1040sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa, 1041 struct mbuf *m0, int sbprio) 1042{ 1043 struct mbuf *m, *n, *n0, *nlast; 1044 int error; 1045 1046 KASSERT(solocked(sb->sb_so)); 1047 1048 /* 1049 * XXX sbprio reserved for encoding priority of this* request: 1050 * SB_PRIO_NONE --> honour normal sb limits 1051 * SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space, 1052 * take whole chain. Intended for large requests 1053 * that should be delivered atomically (all, or none). 1054 * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow 1055 * over normal socket limits, for messages indicating 1056 * buffer overflow in earlier normal/lower-priority messages 1057 * SB_PRIO_BESTEFFORT --> ignore limits entirely. 1058 * Intended for kernel-generated messages only. 1059 * Up to generator to avoid total mbuf resource exhaustion. 1060 */ 1061 (void)sbprio; 1062 1063 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 1064 panic("sbappendaddrchain"); 1065 1066#ifdef notyet 1067 space = sbspace(sb); 1068 1069 /* 1070 * Enforce SB_PRIO_* limits as described above. 1071 */ 1072#endif 1073 1074 n0 = NULL; 1075 nlast = NULL; 1076 for (m = m0; m; m = m->m_nextpkt) { 1077 struct mbuf *np; 1078 1079#ifdef MBUFTRACE 1080 m_claimm(m, sb->sb_mowner); 1081#endif 1082 1083 /* Prepend sockaddr to this record (m) of input chain m0 */ 1084 n = m_prepend_sockaddr(sb, m, asa); 1085 if (n == NULL) { 1086 error = ENOBUFS; 1087 goto bad; 1088 } 1089 1090 /* Append record (asa+m) to end of new chain n0 */ 1091 if (n0 == NULL) { 1092 n0 = n; 1093 } else { 1094 nlast->m_nextpkt = n; 1095 } 1096 /* Keep track of last record on new chain */ 1097 nlast = n; 1098 1099 for (np = n; np; np = np->m_next) 1100 sballoc(sb, np); 1101 } 1102 1103 SBLASTRECORDCHK(sb, "sbappendaddrchain 1"); 1104 1105 /* Drop the entire chain of (asa+m) records onto the socket */ 1106 SBLINKRECORDCHAIN(sb, n0, nlast); 1107 1108 SBLASTRECORDCHK(sb, "sbappendaddrchain 2"); 1109 1110 for (m = nlast; m->m_next; m = m->m_next) 1111 ; 1112 sb->sb_mbtail = m; 1113 SBLASTMBUFCHK(sb, "sbappendaddrchain"); 1114 1115 return (1); 1116 1117bad: 1118 /* 1119 * On error, free the prepended addreseses. For consistency 1120 * with sbappendaddr(), leave it to our caller to free 1121 * the input record chain passed to us as m0. 1122 */ 1123 while ((n = n0) != NULL) { 1124 struct mbuf *np; 1125 1126 /* Undo the sballoc() of this record */ 1127 for (np = n; np; np = np->m_next) 1128 sbfree(sb, np); 1129 1130 n0 = n->m_nextpkt; /* iterate at next prepended address */ 1131 MFREE(n, np); /* free prepended address (not data) */ 1132 } 1133 return error; 1134} 1135 1136 1137int 1138sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 1139{ 1140 struct mbuf *m, *mlast, *n; 1141 int space; 1142 1143 KASSERT(solocked(sb->sb_so)); 1144 1145 space = 0; 1146 if (control == NULL) 1147 panic("sbappendcontrol"); 1148 for (m = control; ; m = m->m_next) { 1149 space += m->m_len; 1150 MCLAIM(m, sb->sb_mowner); 1151 if (m->m_next == NULL) 1152 break; 1153 } 1154 n = m; /* save pointer to last control buffer */ 1155 for (m = m0; m; m = m->m_next) { 1156 MCLAIM(m, sb->sb_mowner); 1157 space += m->m_len; 1158 } 1159 if (space > sbspace(sb)) 1160 return (0); 1161 n->m_next = m0; /* concatenate data to control */ 1162 1163 SBLASTRECORDCHK(sb, "sbappendcontrol 1"); 1164 1165 for (m = control; m->m_next != NULL; m = m->m_next) 1166 sballoc(sb, m); 1167 sballoc(sb, m); 1168 mlast = m; 1169 SBLINKRECORD(sb, control); 1170 1171 sb->sb_mbtail = mlast; 1172 SBLASTMBUFCHK(sb, "sbappendcontrol"); 1173 SBLASTRECORDCHK(sb, "sbappendcontrol 2"); 1174 1175 return (1); 1176} 1177 1178/* 1179 * Compress mbuf chain m into the socket 1180 * buffer sb following mbuf n. If n 1181 * is null, the buffer is presumed empty. 1182 */ 1183void 1184sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 1185{ 1186 int eor; 1187 struct mbuf *o; 1188 1189 KASSERT(solocked(sb->sb_so)); 1190 1191 eor = 0; 1192 while (m) { 1193 eor |= m->m_flags & M_EOR; 1194 if (m->m_len == 0 && 1195 (eor == 0 || 1196 (((o = m->m_next) || (o = n)) && 1197 o->m_type == m->m_type))) { 1198 if (sb->sb_lastrecord == m) 1199 sb->sb_lastrecord = m->m_next; 1200 m = m_free(m); 1201 continue; 1202 } 1203 if (n && (n->m_flags & M_EOR) == 0 && 1204 /* M_TRAILINGSPACE() checks buffer writeability */ 1205 m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */ 1206 m->m_len <= M_TRAILINGSPACE(n) && 1207 n->m_type == m->m_type) { 1208 memcpy(mtod(n, char *) + n->m_len, mtod(m, void *), 1209 (unsigned)m->m_len); 1210 n->m_len += m->m_len; 1211 sb->sb_cc += m->m_len; 1212 m = m_free(m); 1213 continue; 1214 } 1215 if (n) 1216 n->m_next = m; 1217 else 1218 sb->sb_mb = m; 1219 sb->sb_mbtail = m; 1220 sballoc(sb, m); 1221 n = m; 1222 m->m_flags &= ~M_EOR; 1223 m = m->m_next; 1224 n->m_next = 0; 1225 } 1226 if (eor) { 1227 if (n) 1228 n->m_flags |= eor; 1229 else 1230 printf("semi-panic: sbcompress\n"); 1231 } 1232 SBLASTMBUFCHK(sb, __func__); 1233} 1234 1235/* 1236 * Free all mbufs in a sockbuf. 1237 * Check that all resources are reclaimed. 1238 */ 1239void 1240sbflush(struct sockbuf *sb) 1241{ 1242 1243 KASSERT(solocked(sb->sb_so)); 1244 KASSERT((sb->sb_flags & SB_LOCK) == 0); 1245 1246 while (sb->sb_mbcnt) 1247 sbdrop(sb, (int)sb->sb_cc); 1248 1249 KASSERT(sb->sb_cc == 0); 1250 KASSERT(sb->sb_mb == NULL); 1251 KASSERT(sb->sb_mbtail == NULL); 1252 KASSERT(sb->sb_lastrecord == NULL); 1253} 1254 1255/* 1256 * Drop data from (the front of) a sockbuf. 1257 */ 1258void 1259sbdrop(struct sockbuf *sb, int len) 1260{ 1261 struct mbuf *m, *mn, *next; 1262 1263 KASSERT(solocked(sb->sb_so)); 1264 1265 next = (m = sb->sb_mb) ? m->m_nextpkt : NULL; 1266 while (len > 0) { 1267 if (m == NULL) { 1268 if (next == NULL) 1269 panic("sbdrop(%p,%d): cc=%lu", 1270 sb, len, sb->sb_cc); 1271 m = next; 1272 next = m->m_nextpkt; 1273 continue; 1274 } 1275 if (m->m_len > len) { 1276 m->m_len -= len; 1277 m->m_data += len; 1278 sb->sb_cc -= len; 1279 break; 1280 } 1281 len -= m->m_len; 1282 sbfree(sb, m); 1283 MFREE(m, mn); 1284 m = mn; 1285 } 1286 while (m && m->m_len == 0) { 1287 sbfree(sb, m); 1288 MFREE(m, mn); 1289 m = mn; 1290 } 1291 if (m) { 1292 sb->sb_mb = m; 1293 m->m_nextpkt = next; 1294 } else 1295 sb->sb_mb = next; 1296 /* 1297 * First part is an inline SB_EMPTY_FIXUP(). Second part 1298 * makes sure sb_lastrecord is up-to-date if we dropped 1299 * part of the last record. 1300 */ 1301 m = sb->sb_mb; 1302 if (m == NULL) { 1303 sb->sb_mbtail = NULL; 1304 sb->sb_lastrecord = NULL; 1305 } else if (m->m_nextpkt == NULL) 1306 sb->sb_lastrecord = m; 1307} 1308 1309/* 1310 * Drop a record off the front of a sockbuf 1311 * and move the next record to the front. 1312 */ 1313void 1314sbdroprecord(struct sockbuf *sb) 1315{ 1316 struct mbuf *m, *mn; 1317 1318 KASSERT(solocked(sb->sb_so)); 1319 1320 m = sb->sb_mb; 1321 if (m) { 1322 sb->sb_mb = m->m_nextpkt; 1323 do { 1324 sbfree(sb, m); 1325 MFREE(m, mn); 1326 } while ((m = mn) != NULL); 1327 } 1328 SB_EMPTY_FIXUP(sb); 1329} 1330 1331/* 1332 * Create a "control" mbuf containing the specified data 1333 * with the specified type for presentation on a socket buffer. 1334 */ 1335struct mbuf * 1336sbcreatecontrol1(void **p, int size, int type, int level, int flags) 1337{ 1338 struct cmsghdr *cp; 1339 struct mbuf *m; 1340 int space = CMSG_SPACE(size); 1341 1342 if ((flags & M_DONTWAIT) && space > MCLBYTES) { 1343 printf("%s: message too large %d\n", __func__, space); 1344 return NULL; 1345 } 1346 1347 if ((m = m_get(flags, MT_CONTROL)) == NULL) 1348 return NULL; 1349 if (space > MLEN) { 1350 if (space > MCLBYTES) 1351 MEXTMALLOC(m, space, M_WAITOK); 1352 else 1353 MCLGET(m, flags); 1354 if ((m->m_flags & M_EXT) == 0) { 1355 m_free(m); 1356 return NULL; 1357 } 1358 } 1359 cp = mtod(m, struct cmsghdr *); 1360 *p = CMSG_DATA(cp); 1361 m->m_len = space; 1362 cp->cmsg_len = CMSG_LEN(size); 1363 cp->cmsg_level = level; 1364 cp->cmsg_type = type; 1365 return m; 1366} 1367 1368struct mbuf * 1369sbcreatecontrol(void *p, int size, int type, int level) 1370{ 1371 struct mbuf *m; 1372 void *v; 1373 1374 m = sbcreatecontrol1(&v, size, type, level, M_DONTWAIT); 1375 if (m == NULL) 1376 return NULL; 1377 memcpy(v, p, size); 1378 return m; 1379} 1380 1381void 1382solockretry(struct socket *so, kmutex_t *lock) 1383{ 1384 1385 while (lock != so->so_lock) { 1386 mutex_exit(lock); 1387 lock = so->so_lock; 1388 mutex_enter(lock); 1389 } 1390} 1391 1392bool 1393solocked(struct socket *so) 1394{ 1395 1396 return mutex_owned(so->so_lock); 1397} 1398 1399bool 1400solocked2(struct socket *so1, struct socket *so2) 1401{ 1402 kmutex_t *lock; 1403 1404 lock = so1->so_lock; 1405 if (lock != so2->so_lock) 1406 return false; 1407 return mutex_owned(lock); 1408} 1409 1410/* 1411 * sosetlock: assign a default lock to a new socket. 1412 */ 1413void 1414sosetlock(struct socket *so) 1415{ 1416 if (so->so_lock == NULL) { 1417 kmutex_t *lock = softnet_lock; 1418 1419 so->so_lock = lock; 1420 mutex_obj_hold(lock); 1421 mutex_enter(lock); 1422 } 1423 1424 /* In all cases, lock must be held on return from PRU_ATTACH. */ 1425 KASSERT(solocked(so)); 1426} 1427 1428/* 1429 * Set lock on sockbuf sb; sleep if lock is already held. 1430 * Unless SB_NOINTR is set on sockbuf, sleep is interruptible. 1431 * Returns error without lock if sleep is interrupted. 1432 */ 1433int 1434sblock(struct sockbuf *sb, int wf) 1435{ 1436 struct socket *so; 1437 kmutex_t *lock; 1438 int error; 1439 1440 KASSERT(solocked(sb->sb_so)); 1441 1442 for (;;) { 1443 if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) { 1444 sb->sb_flags |= SB_LOCK; 1445 return 0; 1446 } 1447 if (wf != M_WAITOK) 1448 return EWOULDBLOCK; 1449 so = sb->sb_so; 1450 lock = so->so_lock; 1451 if ((sb->sb_flags & SB_NOINTR) != 0) { 1452 cv_wait(&so->so_cv, lock); 1453 error = 0; 1454 } else 1455 error = cv_wait_sig(&so->so_cv, lock); 1456 if (__predict_false(lock != so->so_lock)) 1457 solockretry(so, lock); 1458 if (error != 0) 1459 return error; 1460 } 1461} 1462 1463void 1464sbunlock(struct sockbuf *sb) 1465{ 1466 struct socket *so; 1467 1468 so = sb->sb_so; 1469 1470 KASSERT(solocked(so)); 1471 KASSERT((sb->sb_flags & SB_LOCK) != 0); 1472 1473 sb->sb_flags &= ~SB_LOCK; 1474 cv_broadcast(&so->so_cv); 1475} 1476 1477int 1478sowait(struct socket *so, bool catch, int timo) 1479{ 1480 kmutex_t *lock; 1481 int error; 1482 1483 KASSERT(solocked(so)); 1484 KASSERT(catch || timo != 0); 1485 1486 lock = so->so_lock; 1487 if (catch) 1488 error = cv_timedwait_sig(&so->so_cv, lock, timo); 1489 else 1490 error = cv_timedwait(&so->so_cv, lock, timo); 1491 if (__predict_false(lock != so->so_lock)) 1492 solockretry(so, lock); 1493 return error; 1494} 1495