uipc_socket2.c revision 1.95
1/* $NetBSD: uipc_socket2.c,v 1.95 2008/06/10 11:49:11 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * Copyright (c) 1982, 1986, 1988, 1990, 1993 31 * The Regents of the University of California. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. Neither the name of the University nor the names of its contributors 42 * may be used to endorse or promote products derived from this software 43 * without specific prior written permission. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * @(#)uipc_socket2.c 8.2 (Berkeley) 2/14/95 58 */ 59 60#include <sys/cdefs.h> 61__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.95 2008/06/10 11:49:11 ad Exp $"); 62 63#include "opt_mbuftrace.h" 64#include "opt_sb_max.h" 65 66#include <sys/param.h> 67#include <sys/systm.h> 68#include <sys/proc.h> 69#include <sys/file.h> 70#include <sys/buf.h> 71#include <sys/malloc.h> 72#include <sys/mbuf.h> 73#include <sys/protosw.h> 74#include <sys/domain.h> 75#include <sys/poll.h> 76#include <sys/socket.h> 77#include <sys/socketvar.h> 78#include <sys/signalvar.h> 79#include <sys/kauth.h> 80#include <sys/pool.h> 81 82/* 83 * Primitive routines for operating on sockets and socket buffers. 84 * 85 * Locking rules and assumptions: 86 * 87 * o socket::so_lock can change on the fly. The low level routines used 88 * to lock sockets are aware of this. When so_lock is acquired, the 89 * routine locking must check to see if so_lock still points to the 90 * lock that was acquired. If so_lock has changed in the meantime, the 91 * now irellevant lock that was acquired must be dropped and the lock 92 * operation retried. Although not proven here, this is completely safe 93 * on a multiprocessor system, even with relaxed memory ordering, given 94 * the next two rules: 95 * 96 * o In order to mutate so_lock, the lock pointed to by the current value 97 * of so_lock must be held: i.e., the socket must be held locked by the 98 * changing thread. The thread must issue membar_exit() to prevent 99 * memory accesses being reordered, and can set so_lock to the desired 100 * value. If the lock pointed to by the new value of so_lock is not 101 * held by the changing thread, the socket must then be considered 102 * unlocked. 103 * 104 * o If so_lock is mutated, and the previous lock referred to by so_lock 105 * could still be visible to other threads in the system (e.g. via file 106 * descriptor or protocol-internal reference), then the old lock must 107 * remain valid until the socket and/or protocol control block has been 108 * torn down. 109 * 110 * o If a socket has a non-NULL so_head value (i.e. is in the process of 111 * connecting), then locking the socket must also lock the socket pointed 112 * to by so_head: their lock pointers must match. 113 * 114 * o If a socket has connections in progress (so_q, so_q0 not empty) then 115 * locking the socket must also lock the sockets attached to both queues. 116 * Again, their lock pointers must match. 117 * 118 * o Beyond the initial lock assigment in socreate(), assigning locks to 119 * sockets is the responsibility of the individual protocols / protocol 120 * domains. 121 */ 122 123static pool_cache_t socket_cache; 124 125u_long sb_max = SB_MAX; /* maximum socket buffer size */ 126static u_long sb_max_adj; /* adjusted sb_max */ 127 128/* 129 * Procedures to manipulate state flags of socket 130 * and do appropriate wakeups. Normal sequence from the 131 * active (originating) side is that soisconnecting() is 132 * called during processing of connect() call, 133 * resulting in an eventual call to soisconnected() if/when the 134 * connection is established. When the connection is torn down 135 * soisdisconnecting() is called during processing of disconnect() call, 136 * and soisdisconnected() is called when the connection to the peer 137 * is totally severed. The semantics of these routines are such that 138 * connectionless protocols can call soisconnected() and soisdisconnected() 139 * only, bypassing the in-progress calls when setting up a ``connection'' 140 * takes no time. 141 * 142 * From the passive side, a socket is created with 143 * two queues of sockets: so_q0 for connections in progress 144 * and so_q for connections already made and awaiting user acceptance. 145 * As a protocol is preparing incoming connections, it creates a socket 146 * structure queued on so_q0 by calling sonewconn(). When the connection 147 * is established, soisconnected() is called, and transfers the 148 * socket structure to so_q, making it available to accept(). 149 * 150 * If a socket is closed with sockets on either 151 * so_q0 or so_q, these sockets are dropped. 152 * 153 * If higher level protocols are implemented in 154 * the kernel, the wakeups done here will sometimes 155 * cause software-interrupt process scheduling. 156 */ 157 158void 159soisconnecting(struct socket *so) 160{ 161 162 KASSERT(solocked(so)); 163 164 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 165 so->so_state |= SS_ISCONNECTING; 166} 167 168void 169soisconnected(struct socket *so) 170{ 171 struct socket *head; 172 173 head = so->so_head; 174 175 KASSERT(solocked(so)); 176 KASSERT(head == NULL || solocked2(so, head)); 177 178 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING); 179 so->so_state |= SS_ISCONNECTED; 180 if (head && soqremque(so, 0)) { 181 soqinsque(head, so, 1); 182 sorwakeup(head); 183 cv_broadcast(&head->so_cv); 184 } else { 185 cv_broadcast(&so->so_cv); 186 sorwakeup(so); 187 sowwakeup(so); 188 } 189} 190 191void 192soisdisconnecting(struct socket *so) 193{ 194 195 KASSERT(solocked(so)); 196 197 so->so_state &= ~SS_ISCONNECTING; 198 so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE); 199 cv_broadcast(&so->so_cv); 200 sowwakeup(so); 201 sorwakeup(so); 202} 203 204void 205soisdisconnected(struct socket *so) 206{ 207 208 KASSERT(solocked(so)); 209 210 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 211 so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED); 212 cv_broadcast(&so->so_cv); 213 sowwakeup(so); 214 sorwakeup(so); 215} 216 217void 218soinit2(void) 219{ 220 221 socket_cache = pool_cache_init(sizeof(struct socket), 0, 0, 0, 222 "socket", NULL, IPL_SOFTNET, NULL, NULL, NULL); 223} 224 225/* 226 * When an attempt at a new connection is noted on a socket 227 * which accepts connections, sonewconn is called. If the 228 * connection is possible (subject to space constraints, etc.) 229 * then we allocate a new structure, propoerly linked into the 230 * data structure of the original socket, and return this. 231 * Connstatus may be 0, SS_ISCONFIRMING, or SS_ISCONNECTED. 232 */ 233struct socket * 234sonewconn(struct socket *head, int connstatus) 235{ 236 struct socket *so; 237 int soqueue, error; 238 239 KASSERT(solocked(head)); 240 241 soqueue = connstatus ? 1 : 0; 242 if (head->so_qlen + head->so_q0len > 3 * head->so_qlimit / 2) 243 return ((struct socket *)0); 244 so = soget(false); 245 if (so == NULL) 246 return (NULL); 247 mutex_obj_hold(head->so_lock); 248 so->so_lock = head->so_lock; 249 so->so_type = head->so_type; 250 so->so_options = head->so_options &~ SO_ACCEPTCONN; 251 so->so_linger = head->so_linger; 252 so->so_state = head->so_state | SS_NOFDREF; 253 so->so_nbio = head->so_nbio; 254 so->so_proto = head->so_proto; 255 so->so_timeo = head->so_timeo; 256 so->so_pgid = head->so_pgid; 257 so->so_send = head->so_send; 258 so->so_receive = head->so_receive; 259 so->so_uidinfo = head->so_uidinfo; 260#ifdef MBUFTRACE 261 so->so_mowner = head->so_mowner; 262 so->so_rcv.sb_mowner = head->so_rcv.sb_mowner; 263 so->so_snd.sb_mowner = head->so_snd.sb_mowner; 264#endif 265 (void) soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat); 266 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 267 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 268 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 269 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 270 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 271 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 272 soqinsque(head, so, soqueue); 273 error = (*so->so_proto->pr_usrreq)(so, PRU_ATTACH, NULL, NULL, 274 NULL, NULL); 275 KASSERT(solocked(so)); 276 if (error != 0) { 277 (void) soqremque(so, soqueue); 278 soput(so); 279 return (NULL); 280 } 281 if (connstatus) { 282 sorwakeup(head); 283 cv_broadcast(&head->so_cv); 284 so->so_state |= connstatus; 285 } 286 return (so); 287} 288 289struct socket * 290soget(bool waitok) 291{ 292 struct socket *so; 293 294 so = pool_cache_get(socket_cache, (waitok ? PR_WAITOK : PR_NOWAIT)); 295 if (__predict_false(so == NULL)) 296 return (NULL); 297 memset(so, 0, sizeof(*so)); 298 TAILQ_INIT(&so->so_q0); 299 TAILQ_INIT(&so->so_q); 300 cv_init(&so->so_cv, "socket"); 301 cv_init(&so->so_rcv.sb_cv, "netio"); 302 cv_init(&so->so_snd.sb_cv, "netio"); 303 selinit(&so->so_rcv.sb_sel); 304 selinit(&so->so_snd.sb_sel); 305 so->so_rcv.sb_so = so; 306 so->so_snd.sb_so = so; 307 return so; 308} 309 310void 311soput(struct socket *so) 312{ 313 314 KASSERT(!cv_has_waiters(&so->so_cv)); 315 KASSERT(!cv_has_waiters(&so->so_rcv.sb_cv)); 316 KASSERT(!cv_has_waiters(&so->so_snd.sb_cv)); 317 seldestroy(&so->so_rcv.sb_sel); 318 seldestroy(&so->so_snd.sb_sel); 319 mutex_obj_free(so->so_lock); 320 cv_destroy(&so->so_cv); 321 cv_destroy(&so->so_rcv.sb_cv); 322 cv_destroy(&so->so_snd.sb_cv); 323 pool_cache_put(socket_cache, so); 324} 325 326void 327soqinsque(struct socket *head, struct socket *so, int q) 328{ 329 330 KASSERT(solocked2(head, so)); 331 332#ifdef DIAGNOSTIC 333 if (so->so_onq != NULL) 334 panic("soqinsque"); 335#endif 336 337 so->so_head = head; 338 if (q == 0) { 339 head->so_q0len++; 340 so->so_onq = &head->so_q0; 341 } else { 342 head->so_qlen++; 343 so->so_onq = &head->so_q; 344 } 345 TAILQ_INSERT_TAIL(so->so_onq, so, so_qe); 346} 347 348int 349soqremque(struct socket *so, int q) 350{ 351 struct socket *head; 352 353 head = so->so_head; 354 355 KASSERT(solocked(so)); 356 if (q == 0) { 357 if (so->so_onq != &head->so_q0) 358 return (0); 359 head->so_q0len--; 360 } else { 361 if (so->so_onq != &head->so_q) 362 return (0); 363 head->so_qlen--; 364 } 365 KASSERT(solocked2(so, head)); 366 TAILQ_REMOVE(so->so_onq, so, so_qe); 367 so->so_onq = NULL; 368 so->so_head = NULL; 369 return (1); 370} 371 372/* 373 * Socantsendmore indicates that no more data will be sent on the 374 * socket; it would normally be applied to a socket when the user 375 * informs the system that no more data is to be sent, by the protocol 376 * code (in case PRU_SHUTDOWN). Socantrcvmore indicates that no more data 377 * will be received, and will normally be applied to the socket by a 378 * protocol when it detects that the peer will send no more data. 379 * Data queued for reading in the socket may yet be read. 380 */ 381 382void 383socantsendmore(struct socket *so) 384{ 385 386 KASSERT(solocked(so)); 387 388 so->so_state |= SS_CANTSENDMORE; 389 sowwakeup(so); 390} 391 392void 393socantrcvmore(struct socket *so) 394{ 395 396 KASSERT(solocked(so)); 397 398 so->so_state |= SS_CANTRCVMORE; 399 sorwakeup(so); 400} 401 402/* 403 * Wait for data to arrive at/drain from a socket buffer. 404 */ 405int 406sbwait(struct sockbuf *sb) 407{ 408 struct socket *so; 409 kmutex_t *lock; 410 int error; 411 412 so = sb->sb_so; 413 414 KASSERT(solocked(so)); 415 416 sb->sb_flags |= SB_NOTIFY; 417 lock = so->so_lock; 418 if ((sb->sb_flags & SB_NOINTR) != 0) 419 error = cv_timedwait(&sb->sb_cv, lock, sb->sb_timeo); 420 else 421 error = cv_timedwait_sig(&sb->sb_cv, lock, sb->sb_timeo); 422 if (__predict_false(lock != so->so_lock)) 423 solockretry(so, lock); 424 return error; 425} 426 427/* 428 * Wakeup processes waiting on a socket buffer. 429 * Do asynchronous notification via SIGIO 430 * if the socket buffer has the SB_ASYNC flag set. 431 */ 432void 433sowakeup(struct socket *so, struct sockbuf *sb, int code) 434{ 435 int band; 436 437 KASSERT(solocked(so)); 438 KASSERT(sb->sb_so == so); 439 440 if (code == POLL_IN) 441 band = POLLIN|POLLRDNORM; 442 else 443 band = POLLOUT|POLLWRNORM; 444 sb->sb_flags &= ~SB_NOTIFY; 445 selnotify(&sb->sb_sel, band, NOTE_SUBMIT); 446 cv_broadcast(&sb->sb_cv); 447 if (sb->sb_flags & SB_ASYNC) 448 fownsignal(so->so_pgid, SIGIO, code, band, so); 449 if (sb->sb_flags & SB_UPCALL) 450 (*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT); 451} 452 453/* 454 * Reset a socket's lock pointer. Wake all threads waiting on the 455 * socket's condition variables so that they can restart their waits 456 * using the new lock. The existing lock must be held. 457 */ 458void 459solockreset(struct socket *so, kmutex_t *lock) 460{ 461 462 KASSERT(solocked(so)); 463 464 so->so_lock = lock; 465 cv_broadcast(&so->so_snd.sb_cv); 466 cv_broadcast(&so->so_rcv.sb_cv); 467 cv_broadcast(&so->so_cv); 468} 469 470/* 471 * Socket buffer (struct sockbuf) utility routines. 472 * 473 * Each socket contains two socket buffers: one for sending data and 474 * one for receiving data. Each buffer contains a queue of mbufs, 475 * information about the number of mbufs and amount of data in the 476 * queue, and other fields allowing poll() statements and notification 477 * on data availability to be implemented. 478 * 479 * Data stored in a socket buffer is maintained as a list of records. 480 * Each record is a list of mbufs chained together with the m_next 481 * field. Records are chained together with the m_nextpkt field. The upper 482 * level routine soreceive() expects the following conventions to be 483 * observed when placing information in the receive buffer: 484 * 485 * 1. If the protocol requires each message be preceded by the sender's 486 * name, then a record containing that name must be present before 487 * any associated data (mbuf's must be of type MT_SONAME). 488 * 2. If the protocol supports the exchange of ``access rights'' (really 489 * just additional data associated with the message), and there are 490 * ``rights'' to be received, then a record containing this data 491 * should be present (mbuf's must be of type MT_CONTROL). 492 * 3. If a name or rights record exists, then it must be followed by 493 * a data record, perhaps of zero length. 494 * 495 * Before using a new socket structure it is first necessary to reserve 496 * buffer space to the socket, by calling sbreserve(). This should commit 497 * some of the available buffer space in the system buffer pool for the 498 * socket (currently, it does nothing but enforce limits). The space 499 * should be released by calling sbrelease() when the socket is destroyed. 500 */ 501 502int 503sb_max_set(u_long new_sbmax) 504{ 505 int s; 506 507 if (new_sbmax < (16 * 1024)) 508 return (EINVAL); 509 510 s = splsoftnet(); 511 sb_max = new_sbmax; 512 sb_max_adj = (u_quad_t)new_sbmax * MCLBYTES / (MSIZE + MCLBYTES); 513 splx(s); 514 515 return (0); 516} 517 518int 519soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 520{ 521 522 KASSERT(so->so_lock == NULL || solocked(so)); 523 524 /* 525 * there's at least one application (a configure script of screen) 526 * which expects a fifo is writable even if it has "some" bytes 527 * in its buffer. 528 * so we want to make sure (hiwat - lowat) >= (some bytes). 529 * 530 * PIPE_BUF here is an arbitrary value chosen as (some bytes) above. 531 * we expect it's large enough for such applications. 532 */ 533 u_long lowat = MAX(sock_loan_thresh, MCLBYTES); 534 u_long hiwat = lowat + PIPE_BUF; 535 536 if (sndcc < hiwat) 537 sndcc = hiwat; 538 if (sbreserve(&so->so_snd, sndcc, so) == 0) 539 goto bad; 540 if (sbreserve(&so->so_rcv, rcvcc, so) == 0) 541 goto bad2; 542 if (so->so_rcv.sb_lowat == 0) 543 so->so_rcv.sb_lowat = 1; 544 if (so->so_snd.sb_lowat == 0) 545 so->so_snd.sb_lowat = lowat; 546 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 547 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 548 return (0); 549 bad2: 550 sbrelease(&so->so_snd, so); 551 bad: 552 return (ENOBUFS); 553} 554 555/* 556 * Allot mbufs to a sockbuf. 557 * Attempt to scale mbmax so that mbcnt doesn't become limiting 558 * if buffering efficiency is near the normal case. 559 */ 560int 561sbreserve(struct sockbuf *sb, u_long cc, struct socket *so) 562{ 563 struct lwp *l = curlwp; /* XXX */ 564 rlim_t maxcc; 565 struct uidinfo *uidinfo; 566 567 KASSERT(so->so_lock == NULL || solocked(so)); 568 KASSERT(sb->sb_so == so); 569 KASSERT(sb_max_adj != 0); 570 571 if (cc == 0 || cc > sb_max_adj) 572 return (0); 573 574 if (kauth_cred_geteuid(l->l_cred) == so->so_uidinfo->ui_uid) 575 maxcc = l->l_proc->p_rlimit[RLIMIT_SBSIZE].rlim_cur; 576 else 577 maxcc = RLIM_INFINITY; 578 579 uidinfo = so->so_uidinfo; 580 if (!chgsbsize(uidinfo, &sb->sb_hiwat, cc, maxcc)) 581 return 0; 582 sb->sb_mbmax = min(cc * 2, sb_max); 583 if (sb->sb_lowat > sb->sb_hiwat) 584 sb->sb_lowat = sb->sb_hiwat; 585 return (1); 586} 587 588/* 589 * Free mbufs held by a socket, and reserved mbuf space. We do not assert 590 * that the socket is held locked here: see sorflush(). 591 */ 592void 593sbrelease(struct sockbuf *sb, struct socket *so) 594{ 595 596 KASSERT(sb->sb_so == so); 597 598 sbflush(sb); 599 (void)chgsbsize(so->so_uidinfo, &sb->sb_hiwat, 0, RLIM_INFINITY); 600 sb->sb_mbmax = 0; 601} 602 603/* 604 * Routines to add and remove 605 * data from an mbuf queue. 606 * 607 * The routines sbappend() or sbappendrecord() are normally called to 608 * append new mbufs to a socket buffer, after checking that adequate 609 * space is available, comparing the function sbspace() with the amount 610 * of data to be added. sbappendrecord() differs from sbappend() in 611 * that data supplied is treated as the beginning of a new record. 612 * To place a sender's address, optional access rights, and data in a 613 * socket receive buffer, sbappendaddr() should be used. To place 614 * access rights and data in a socket receive buffer, sbappendrights() 615 * should be used. In either case, the new data begins a new record. 616 * Note that unlike sbappend() and sbappendrecord(), these routines check 617 * for the caller that there will be enough space to store the data. 618 * Each fails if there is not enough space, or if it cannot find mbufs 619 * to store additional information in. 620 * 621 * Reliable protocols may use the socket send buffer to hold data 622 * awaiting acknowledgement. Data is normally copied from a socket 623 * send buffer in a protocol with m_copy for output to a peer, 624 * and then removing the data from the socket buffer with sbdrop() 625 * or sbdroprecord() when the data is acknowledged by the peer. 626 */ 627 628#ifdef SOCKBUF_DEBUG 629void 630sblastrecordchk(struct sockbuf *sb, const char *where) 631{ 632 struct mbuf *m = sb->sb_mb; 633 634 KASSERT(solocked(sb->sb_so)); 635 636 while (m && m->m_nextpkt) 637 m = m->m_nextpkt; 638 639 if (m != sb->sb_lastrecord) { 640 printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n", 641 sb->sb_mb, sb->sb_lastrecord, m); 642 printf("packet chain:\n"); 643 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 644 printf("\t%p\n", m); 645 panic("sblastrecordchk from %s", where); 646 } 647} 648 649void 650sblastmbufchk(struct sockbuf *sb, const char *where) 651{ 652 struct mbuf *m = sb->sb_mb; 653 struct mbuf *n; 654 655 KASSERT(solocked(sb->sb_so)); 656 657 while (m && m->m_nextpkt) 658 m = m->m_nextpkt; 659 660 while (m && m->m_next) 661 m = m->m_next; 662 663 if (m != sb->sb_mbtail) { 664 printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n", 665 sb->sb_mb, sb->sb_mbtail, m); 666 printf("packet tree:\n"); 667 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 668 printf("\t"); 669 for (n = m; n != NULL; n = n->m_next) 670 printf("%p ", n); 671 printf("\n"); 672 } 673 panic("sblastmbufchk from %s", where); 674 } 675} 676#endif /* SOCKBUF_DEBUG */ 677 678/* 679 * Link a chain of records onto a socket buffer 680 */ 681#define SBLINKRECORDCHAIN(sb, m0, mlast) \ 682do { \ 683 if ((sb)->sb_lastrecord != NULL) \ 684 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 685 else \ 686 (sb)->sb_mb = (m0); \ 687 (sb)->sb_lastrecord = (mlast); \ 688} while (/*CONSTCOND*/0) 689 690 691#define SBLINKRECORD(sb, m0) \ 692 SBLINKRECORDCHAIN(sb, m0, m0) 693 694/* 695 * Append mbuf chain m to the last record in the 696 * socket buffer sb. The additional space associated 697 * the mbuf chain is recorded in sb. Empty mbufs are 698 * discarded and mbufs are compacted where possible. 699 */ 700void 701sbappend(struct sockbuf *sb, struct mbuf *m) 702{ 703 struct mbuf *n; 704 705 KASSERT(solocked(sb->sb_so)); 706 707 if (m == 0) 708 return; 709 710#ifdef MBUFTRACE 711 m_claimm(m, sb->sb_mowner); 712#endif 713 714 SBLASTRECORDCHK(sb, "sbappend 1"); 715 716 if ((n = sb->sb_lastrecord) != NULL) { 717 /* 718 * XXX Would like to simply use sb_mbtail here, but 719 * XXX I need to verify that I won't miss an EOR that 720 * XXX way. 721 */ 722 do { 723 if (n->m_flags & M_EOR) { 724 sbappendrecord(sb, m); /* XXXXXX!!!! */ 725 return; 726 } 727 } while (n->m_next && (n = n->m_next)); 728 } else { 729 /* 730 * If this is the first record in the socket buffer, it's 731 * also the last record. 732 */ 733 sb->sb_lastrecord = m; 734 } 735 sbcompress(sb, m, n); 736 SBLASTRECORDCHK(sb, "sbappend 2"); 737} 738 739/* 740 * This version of sbappend() should only be used when the caller 741 * absolutely knows that there will never be more than one record 742 * in the socket buffer, that is, a stream protocol (such as TCP). 743 */ 744void 745sbappendstream(struct sockbuf *sb, struct mbuf *m) 746{ 747 748 KASSERT(solocked(sb->sb_so)); 749 KDASSERT(m->m_nextpkt == NULL); 750 KASSERT(sb->sb_mb == sb->sb_lastrecord); 751 752 SBLASTMBUFCHK(sb, __func__); 753 754#ifdef MBUFTRACE 755 m_claimm(m, sb->sb_mowner); 756#endif 757 758 sbcompress(sb, m, sb->sb_mbtail); 759 760 sb->sb_lastrecord = sb->sb_mb; 761 SBLASTRECORDCHK(sb, __func__); 762} 763 764#ifdef SOCKBUF_DEBUG 765void 766sbcheck(struct sockbuf *sb) 767{ 768 struct mbuf *m, *m2; 769 u_long len, mbcnt; 770 771 KASSERT(solocked(sb->sb_so)); 772 773 len = 0; 774 mbcnt = 0; 775 for (m = sb->sb_mb; m; m = m->m_nextpkt) { 776 for (m2 = m; m2 != NULL; m2 = m2->m_next) { 777 len += m2->m_len; 778 mbcnt += MSIZE; 779 if (m2->m_flags & M_EXT) 780 mbcnt += m2->m_ext.ext_size; 781 if (m2->m_nextpkt != NULL) 782 panic("sbcheck nextpkt"); 783 } 784 } 785 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 786 printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc, 787 mbcnt, sb->sb_mbcnt); 788 panic("sbcheck"); 789 } 790} 791#endif 792 793/* 794 * As above, except the mbuf chain 795 * begins a new record. 796 */ 797void 798sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 799{ 800 struct mbuf *m; 801 802 KASSERT(solocked(sb->sb_so)); 803 804 if (m0 == 0) 805 return; 806 807#ifdef MBUFTRACE 808 m_claimm(m0, sb->sb_mowner); 809#endif 810 /* 811 * Put the first mbuf on the queue. 812 * Note this permits zero length records. 813 */ 814 sballoc(sb, m0); 815 SBLASTRECORDCHK(sb, "sbappendrecord 1"); 816 SBLINKRECORD(sb, m0); 817 m = m0->m_next; 818 m0->m_next = 0; 819 if (m && (m0->m_flags & M_EOR)) { 820 m0->m_flags &= ~M_EOR; 821 m->m_flags |= M_EOR; 822 } 823 sbcompress(sb, m, m0); 824 SBLASTRECORDCHK(sb, "sbappendrecord 2"); 825} 826 827/* 828 * As above except that OOB data 829 * is inserted at the beginning of the sockbuf, 830 * but after any other OOB data. 831 */ 832void 833sbinsertoob(struct sockbuf *sb, struct mbuf *m0) 834{ 835 struct mbuf *m, **mp; 836 837 KASSERT(solocked(sb->sb_so)); 838 839 if (m0 == 0) 840 return; 841 842 SBLASTRECORDCHK(sb, "sbinsertoob 1"); 843 844 for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) { 845 again: 846 switch (m->m_type) { 847 848 case MT_OOBDATA: 849 continue; /* WANT next train */ 850 851 case MT_CONTROL: 852 if ((m = m->m_next) != NULL) 853 goto again; /* inspect THIS train further */ 854 } 855 break; 856 } 857 /* 858 * Put the first mbuf on the queue. 859 * Note this permits zero length records. 860 */ 861 sballoc(sb, m0); 862 m0->m_nextpkt = *mp; 863 if (*mp == NULL) { 864 /* m0 is actually the new tail */ 865 sb->sb_lastrecord = m0; 866 } 867 *mp = m0; 868 m = m0->m_next; 869 m0->m_next = 0; 870 if (m && (m0->m_flags & M_EOR)) { 871 m0->m_flags &= ~M_EOR; 872 m->m_flags |= M_EOR; 873 } 874 sbcompress(sb, m, m0); 875 SBLASTRECORDCHK(sb, "sbinsertoob 2"); 876} 877 878/* 879 * Append address and data, and optionally, control (ancillary) data 880 * to the receive queue of a socket. If present, 881 * m0 must include a packet header with total length. 882 * Returns 0 if no space in sockbuf or insufficient mbufs. 883 */ 884int 885sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, 886 struct mbuf *control) 887{ 888 struct mbuf *m, *n, *nlast; 889 int space, len; 890 891 KASSERT(solocked(sb->sb_so)); 892 893 space = asa->sa_len; 894 895 if (m0 != NULL) { 896 if ((m0->m_flags & M_PKTHDR) == 0) 897 panic("sbappendaddr"); 898 space += m0->m_pkthdr.len; 899#ifdef MBUFTRACE 900 m_claimm(m0, sb->sb_mowner); 901#endif 902 } 903 for (n = control; n; n = n->m_next) { 904 space += n->m_len; 905 MCLAIM(n, sb->sb_mowner); 906 if (n->m_next == 0) /* keep pointer to last control buf */ 907 break; 908 } 909 if (space > sbspace(sb)) 910 return (0); 911 MGET(m, M_DONTWAIT, MT_SONAME); 912 if (m == 0) 913 return (0); 914 MCLAIM(m, sb->sb_mowner); 915 /* 916 * XXX avoid 'comparison always true' warning which isn't easily 917 * avoided. 918 */ 919 len = asa->sa_len; 920 if (len > MLEN) { 921 MEXTMALLOC(m, asa->sa_len, M_NOWAIT); 922 if ((m->m_flags & M_EXT) == 0) { 923 m_free(m); 924 return (0); 925 } 926 } 927 m->m_len = asa->sa_len; 928 memcpy(mtod(m, void *), asa, asa->sa_len); 929 if (n) 930 n->m_next = m0; /* concatenate data to control */ 931 else 932 control = m0; 933 m->m_next = control; 934 935 SBLASTRECORDCHK(sb, "sbappendaddr 1"); 936 937 for (n = m; n->m_next != NULL; n = n->m_next) 938 sballoc(sb, n); 939 sballoc(sb, n); 940 nlast = n; 941 SBLINKRECORD(sb, m); 942 943 sb->sb_mbtail = nlast; 944 SBLASTMBUFCHK(sb, "sbappendaddr"); 945 SBLASTRECORDCHK(sb, "sbappendaddr 2"); 946 947 return (1); 948} 949 950/* 951 * Helper for sbappendchainaddr: prepend a struct sockaddr* to 952 * an mbuf chain. 953 */ 954static inline struct mbuf * 955m_prepend_sockaddr(struct sockbuf *sb, struct mbuf *m0, 956 const struct sockaddr *asa) 957{ 958 struct mbuf *m; 959 const int salen = asa->sa_len; 960 961 KASSERT(solocked(sb->sb_so)); 962 963 /* only the first in each chain need be a pkthdr */ 964 MGETHDR(m, M_DONTWAIT, MT_SONAME); 965 if (m == 0) 966 return (0); 967 MCLAIM(m, sb->sb_mowner); 968#ifdef notyet 969 if (salen > MHLEN) { 970 MEXTMALLOC(m, salen, M_NOWAIT); 971 if ((m->m_flags & M_EXT) == 0) { 972 m_free(m); 973 return (0); 974 } 975 } 976#else 977 KASSERT(salen <= MHLEN); 978#endif 979 m->m_len = salen; 980 memcpy(mtod(m, void *), asa, salen); 981 m->m_next = m0; 982 m->m_pkthdr.len = salen + m0->m_pkthdr.len; 983 984 return m; 985} 986 987int 988sbappendaddrchain(struct sockbuf *sb, const struct sockaddr *asa, 989 struct mbuf *m0, int sbprio) 990{ 991 int space; 992 struct mbuf *m, *n, *n0, *nlast; 993 int error; 994 995 KASSERT(solocked(sb->sb_so)); 996 997 /* 998 * XXX sbprio reserved for encoding priority of this* request: 999 * SB_PRIO_NONE --> honour normal sb limits 1000 * SB_PRIO_ONESHOT_OVERFLOW --> if socket has any space, 1001 * take whole chain. Intended for large requests 1002 * that should be delivered atomically (all, or none). 1003 * SB_PRIO_OVERDRAFT -- allow a small (2*MLEN) overflow 1004 * over normal socket limits, for messages indicating 1005 * buffer overflow in earlier normal/lower-priority messages 1006 * SB_PRIO_BESTEFFORT --> ignore limits entirely. 1007 * Intended for kernel-generated messages only. 1008 * Up to generator to avoid total mbuf resource exhaustion. 1009 */ 1010 (void)sbprio; 1011 1012 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 1013 panic("sbappendaddrchain"); 1014 1015 space = sbspace(sb); 1016 1017#ifdef notyet 1018 /* 1019 * Enforce SB_PRIO_* limits as described above. 1020 */ 1021#endif 1022 1023 n0 = NULL; 1024 nlast = NULL; 1025 for (m = m0; m; m = m->m_nextpkt) { 1026 struct mbuf *np; 1027 1028#ifdef MBUFTRACE 1029 m_claimm(m, sb->sb_mowner); 1030#endif 1031 1032 /* Prepend sockaddr to this record (m) of input chain m0 */ 1033 n = m_prepend_sockaddr(sb, m, asa); 1034 if (n == NULL) { 1035 error = ENOBUFS; 1036 goto bad; 1037 } 1038 1039 /* Append record (asa+m) to end of new chain n0 */ 1040 if (n0 == NULL) { 1041 n0 = n; 1042 } else { 1043 nlast->m_nextpkt = n; 1044 } 1045 /* Keep track of last record on new chain */ 1046 nlast = n; 1047 1048 for (np = n; np; np = np->m_next) 1049 sballoc(sb, np); 1050 } 1051 1052 SBLASTRECORDCHK(sb, "sbappendaddrchain 1"); 1053 1054 /* Drop the entire chain of (asa+m) records onto the socket */ 1055 SBLINKRECORDCHAIN(sb, n0, nlast); 1056 1057 SBLASTRECORDCHK(sb, "sbappendaddrchain 2"); 1058 1059 for (m = nlast; m->m_next; m = m->m_next) 1060 ; 1061 sb->sb_mbtail = m; 1062 SBLASTMBUFCHK(sb, "sbappendaddrchain"); 1063 1064 return (1); 1065 1066bad: 1067 /* 1068 * On error, free the prepended addreseses. For consistency 1069 * with sbappendaddr(), leave it to our caller to free 1070 * the input record chain passed to us as m0. 1071 */ 1072 while ((n = n0) != NULL) { 1073 struct mbuf *np; 1074 1075 /* Undo the sballoc() of this record */ 1076 for (np = n; np; np = np->m_next) 1077 sbfree(sb, np); 1078 1079 n0 = n->m_nextpkt; /* iterate at next prepended address */ 1080 MFREE(n, np); /* free prepended address (not data) */ 1081 } 1082 return 0; 1083} 1084 1085 1086int 1087sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 1088{ 1089 struct mbuf *m, *mlast, *n; 1090 int space; 1091 1092 KASSERT(solocked(sb->sb_so)); 1093 1094 space = 0; 1095 if (control == 0) 1096 panic("sbappendcontrol"); 1097 for (m = control; ; m = m->m_next) { 1098 space += m->m_len; 1099 MCLAIM(m, sb->sb_mowner); 1100 if (m->m_next == 0) 1101 break; 1102 } 1103 n = m; /* save pointer to last control buffer */ 1104 for (m = m0; m; m = m->m_next) { 1105 MCLAIM(m, sb->sb_mowner); 1106 space += m->m_len; 1107 } 1108 if (space > sbspace(sb)) 1109 return (0); 1110 n->m_next = m0; /* concatenate data to control */ 1111 1112 SBLASTRECORDCHK(sb, "sbappendcontrol 1"); 1113 1114 for (m = control; m->m_next != NULL; m = m->m_next) 1115 sballoc(sb, m); 1116 sballoc(sb, m); 1117 mlast = m; 1118 SBLINKRECORD(sb, control); 1119 1120 sb->sb_mbtail = mlast; 1121 SBLASTMBUFCHK(sb, "sbappendcontrol"); 1122 SBLASTRECORDCHK(sb, "sbappendcontrol 2"); 1123 1124 return (1); 1125} 1126 1127/* 1128 * Compress mbuf chain m into the socket 1129 * buffer sb following mbuf n. If n 1130 * is null, the buffer is presumed empty. 1131 */ 1132void 1133sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 1134{ 1135 int eor; 1136 struct mbuf *o; 1137 1138 KASSERT(solocked(sb->sb_so)); 1139 1140 eor = 0; 1141 while (m) { 1142 eor |= m->m_flags & M_EOR; 1143 if (m->m_len == 0 && 1144 (eor == 0 || 1145 (((o = m->m_next) || (o = n)) && 1146 o->m_type == m->m_type))) { 1147 if (sb->sb_lastrecord == m) 1148 sb->sb_lastrecord = m->m_next; 1149 m = m_free(m); 1150 continue; 1151 } 1152 if (n && (n->m_flags & M_EOR) == 0 && 1153 /* M_TRAILINGSPACE() checks buffer writeability */ 1154 m->m_len <= MCLBYTES / 4 && /* XXX Don't copy too much */ 1155 m->m_len <= M_TRAILINGSPACE(n) && 1156 n->m_type == m->m_type) { 1157 memcpy(mtod(n, char *) + n->m_len, mtod(m, void *), 1158 (unsigned)m->m_len); 1159 n->m_len += m->m_len; 1160 sb->sb_cc += m->m_len; 1161 m = m_free(m); 1162 continue; 1163 } 1164 if (n) 1165 n->m_next = m; 1166 else 1167 sb->sb_mb = m; 1168 sb->sb_mbtail = m; 1169 sballoc(sb, m); 1170 n = m; 1171 m->m_flags &= ~M_EOR; 1172 m = m->m_next; 1173 n->m_next = 0; 1174 } 1175 if (eor) { 1176 if (n) 1177 n->m_flags |= eor; 1178 else 1179 printf("semi-panic: sbcompress\n"); 1180 } 1181 SBLASTMBUFCHK(sb, __func__); 1182} 1183 1184/* 1185 * Free all mbufs in a sockbuf. 1186 * Check that all resources are reclaimed. 1187 */ 1188void 1189sbflush(struct sockbuf *sb) 1190{ 1191 1192 KASSERT(solocked(sb->sb_so)); 1193 KASSERT((sb->sb_flags & SB_LOCK) == 0); 1194 1195 while (sb->sb_mbcnt) 1196 sbdrop(sb, (int)sb->sb_cc); 1197 1198 KASSERT(sb->sb_cc == 0); 1199 KASSERT(sb->sb_mb == NULL); 1200 KASSERT(sb->sb_mbtail == NULL); 1201 KASSERT(sb->sb_lastrecord == NULL); 1202} 1203 1204/* 1205 * Drop data from (the front of) a sockbuf. 1206 */ 1207void 1208sbdrop(struct sockbuf *sb, int len) 1209{ 1210 struct mbuf *m, *mn, *next; 1211 1212 KASSERT(solocked(sb->sb_so)); 1213 1214 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 1215 while (len > 0) { 1216 if (m == 0) { 1217 if (next == 0) 1218 panic("sbdrop"); 1219 m = next; 1220 next = m->m_nextpkt; 1221 continue; 1222 } 1223 if (m->m_len > len) { 1224 m->m_len -= len; 1225 m->m_data += len; 1226 sb->sb_cc -= len; 1227 break; 1228 } 1229 len -= m->m_len; 1230 sbfree(sb, m); 1231 MFREE(m, mn); 1232 m = mn; 1233 } 1234 while (m && m->m_len == 0) { 1235 sbfree(sb, m); 1236 MFREE(m, mn); 1237 m = mn; 1238 } 1239 if (m) { 1240 sb->sb_mb = m; 1241 m->m_nextpkt = next; 1242 } else 1243 sb->sb_mb = next; 1244 /* 1245 * First part is an inline SB_EMPTY_FIXUP(). Second part 1246 * makes sure sb_lastrecord is up-to-date if we dropped 1247 * part of the last record. 1248 */ 1249 m = sb->sb_mb; 1250 if (m == NULL) { 1251 sb->sb_mbtail = NULL; 1252 sb->sb_lastrecord = NULL; 1253 } else if (m->m_nextpkt == NULL) 1254 sb->sb_lastrecord = m; 1255} 1256 1257/* 1258 * Drop a record off the front of a sockbuf 1259 * and move the next record to the front. 1260 */ 1261void 1262sbdroprecord(struct sockbuf *sb) 1263{ 1264 struct mbuf *m, *mn; 1265 1266 KASSERT(solocked(sb->sb_so)); 1267 1268 m = sb->sb_mb; 1269 if (m) { 1270 sb->sb_mb = m->m_nextpkt; 1271 do { 1272 sbfree(sb, m); 1273 MFREE(m, mn); 1274 } while ((m = mn) != NULL); 1275 } 1276 SB_EMPTY_FIXUP(sb); 1277} 1278 1279/* 1280 * Create a "control" mbuf containing the specified data 1281 * with the specified type for presentation on a socket buffer. 1282 */ 1283struct mbuf * 1284sbcreatecontrol(void *p, int size, int type, int level) 1285{ 1286 struct cmsghdr *cp; 1287 struct mbuf *m; 1288 1289 if (CMSG_SPACE(size) > MCLBYTES) { 1290 printf("sbcreatecontrol: message too large %d\n", size); 1291 return NULL; 1292 } 1293 1294 if ((m = m_get(M_DONTWAIT, MT_CONTROL)) == NULL) 1295 return ((struct mbuf *) NULL); 1296 if (CMSG_SPACE(size) > MLEN) { 1297 MCLGET(m, M_DONTWAIT); 1298 if ((m->m_flags & M_EXT) == 0) { 1299 m_free(m); 1300 return NULL; 1301 } 1302 } 1303 cp = mtod(m, struct cmsghdr *); 1304 memcpy(CMSG_DATA(cp), p, size); 1305 m->m_len = CMSG_SPACE(size); 1306 cp->cmsg_len = CMSG_LEN(size); 1307 cp->cmsg_level = level; 1308 cp->cmsg_type = type; 1309 return (m); 1310} 1311 1312void 1313solockretry(struct socket *so, kmutex_t *lock) 1314{ 1315 1316 while (lock != so->so_lock) { 1317 mutex_exit(lock); 1318 lock = so->so_lock; 1319 mutex_enter(lock); 1320 } 1321} 1322 1323bool 1324solocked(struct socket *so) 1325{ 1326 1327 return mutex_owned(so->so_lock); 1328} 1329 1330bool 1331solocked2(struct socket *so1, struct socket *so2) 1332{ 1333 kmutex_t *lock; 1334 1335 lock = so1->so_lock; 1336 if (lock != so2->so_lock) 1337 return false; 1338 return mutex_owned(lock); 1339} 1340 1341/* 1342 * Assign a default lock to a new socket. For PRU_ATTACH, and done by 1343 * protocols that do not have special locking requirements. 1344 */ 1345void 1346sosetlock(struct socket *so) 1347{ 1348 kmutex_t *lock; 1349 1350 if (so->so_lock == NULL) { 1351 lock = softnet_lock; 1352 so->so_lock = lock; 1353 mutex_obj_hold(lock); 1354 mutex_enter(lock); 1355 } 1356 1357 /* In all cases, lock must be held on return from PRU_ATTACH. */ 1358 KASSERT(solocked(so)); 1359} 1360 1361/* 1362 * Set lock on sockbuf sb; sleep if lock is already held. 1363 * Unless SB_NOINTR is set on sockbuf, sleep is interruptible. 1364 * Returns error without lock if sleep is interrupted. 1365 */ 1366int 1367sblock(struct sockbuf *sb, int wf) 1368{ 1369 struct socket *so; 1370 kmutex_t *lock; 1371 int error; 1372 1373 KASSERT(solocked(sb->sb_so)); 1374 1375 for (;;) { 1376 if (__predict_true((sb->sb_flags & SB_LOCK) == 0)) { 1377 sb->sb_flags |= SB_LOCK; 1378 return 0; 1379 } 1380 if (wf != M_WAITOK) 1381 return EWOULDBLOCK; 1382 so = sb->sb_so; 1383 lock = so->so_lock; 1384 if ((sb->sb_flags & SB_NOINTR) != 0) { 1385 cv_wait(&so->so_cv, lock); 1386 error = 0; 1387 } else 1388 error = cv_wait_sig(&so->so_cv, lock); 1389 if (__predict_false(lock != so->so_lock)) 1390 solockretry(so, lock); 1391 if (error != 0) 1392 return error; 1393 } 1394} 1395 1396void 1397sbunlock(struct sockbuf *sb) 1398{ 1399 struct socket *so; 1400 1401 so = sb->sb_so; 1402 1403 KASSERT(solocked(so)); 1404 KASSERT((sb->sb_flags & SB_LOCK) != 0); 1405 1406 sb->sb_flags &= ~SB_LOCK; 1407 cv_broadcast(&so->so_cv); 1408} 1409 1410int 1411sowait(struct socket *so, int timo) 1412{ 1413 kmutex_t *lock; 1414 int error; 1415 1416 KASSERT(solocked(so)); 1417 1418 lock = so->so_lock; 1419 error = cv_timedwait_sig(&so->so_cv, lock, timo); 1420 if (__predict_false(lock != so->so_lock)) 1421 solockretry(so, lock); 1422 return error; 1423} 1424