34
35#include "opt_param.h"
36
37#include <sys/param.h>
38#include <sys/aio.h> /* for aio_swake proto */
39#include <sys/kernel.h>
40#include <sys/lock.h>
41#include <sys/mbuf.h>
42#include <sys/mutex.h>
43#include <sys/proc.h>
44#include <sys/protosw.h>
45#include <sys/resourcevar.h>
46#include <sys/signalvar.h>
47#include <sys/socket.h>
48#include <sys/socketvar.h>
49#include <sys/sysctl.h>
50
51/*
52 * Function pointer set by the AIO routines so that the socket buffer code
53 * can call back into the AIO module if it is loaded.
54 */
55void (*aio_swake)(struct socket *, struct sockbuf *);
56
57/*
58 * Primitive routines for operating on socket buffers
59 */
60
61u_long sb_max = SB_MAX;
62static u_long sb_max_adj =
63 SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */
64
65static u_long sb_efficiency = 8; /* parameter for sbreserve() */
66
67static void sbdrop_internal(struct sockbuf *sb, int len);
68static void sbflush_internal(struct sockbuf *sb);
69static void sbrelease_internal(struct sockbuf *sb, struct socket *so);
70
71/*
72 * Socantsendmore indicates that no more data will be sent on the socket; it
73 * would normally be applied to a socket when the user informs the system
74 * that no more data is to be sent, by the protocol code (in case
75 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be
76 * received, and will normally be applied to the socket by a protocol when it
77 * detects that the peer will send no more data. Data queued for reading in
78 * the socket may yet be read.
79 */
80void
81socantsendmore_locked(struct socket *so)
82{
83
84 SOCKBUF_LOCK_ASSERT(&so->so_snd);
85
86 so->so_snd.sb_state |= SBS_CANTSENDMORE;
87 sowwakeup_locked(so);
88 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
89}
90
91void
92socantsendmore(struct socket *so)
93{
94
95 SOCKBUF_LOCK(&so->so_snd);
96 socantsendmore_locked(so);
97 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
98}
99
100void
101socantrcvmore_locked(struct socket *so)
102{
103
104 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
105
106 so->so_rcv.sb_state |= SBS_CANTRCVMORE;
107 sorwakeup_locked(so);
108 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
109}
110
111void
112socantrcvmore(struct socket *so)
113{
114
115 SOCKBUF_LOCK(&so->so_rcv);
116 socantrcvmore_locked(so);
117 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
118}
119
120/*
121 * Wait for data to arrive at/drain from a socket buffer.
122 */
123int
124sbwait(struct sockbuf *sb)
125{
126
127 SOCKBUF_LOCK_ASSERT(sb);
128
129 sb->sb_flags |= SB_WAIT;
130 return (msleep(&sb->sb_cc, &sb->sb_mtx,
131 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
132 sb->sb_timeo));
133}
134
135/*
136 * Lock a sockbuf already known to be locked; return any error returned from
137 * sleep (EINTR).
138 */
139int
140sb_lock(struct sockbuf *sb)
141{
142 int error;
143
144 SOCKBUF_LOCK_ASSERT(sb);
145
146 while (sb->sb_flags & SB_LOCK) {
147 sb->sb_flags |= SB_WANT;
148 error = msleep(&sb->sb_flags, &sb->sb_mtx,
149 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH,
150 "sblock", 0);
151 if (error)
152 return (error);
153 }
154 sb->sb_flags |= SB_LOCK;
155 return (0);
156}
157
158/*
159 * Wakeup processes waiting on a socket buffer. Do asynchronous notification
160 * via SIGIO if the socket has the SS_ASYNC flag set.
161 *
162 * Called with the socket buffer lock held; will release the lock by the end
163 * of the function. This allows the caller to acquire the socket buffer lock
164 * while testing for the need for various sorts of wakeup and hold it through
165 * to the point where it's no longer required. We currently hold the lock
166 * through calls out to other subsystems (with the exception of kqueue), and
167 * then release it to avoid lock order issues. It's not clear that's
168 * correct.
169 */
170void
171sowakeup(struct socket *so, struct sockbuf *sb)
172{
173
174 SOCKBUF_LOCK_ASSERT(sb);
175
176 selwakeuppri(&sb->sb_sel, PSOCK);
177 sb->sb_flags &= ~SB_SEL;
178 if (sb->sb_flags & SB_WAIT) {
179 sb->sb_flags &= ~SB_WAIT;
180 wakeup(&sb->sb_cc);
181 }
182 KNOTE_LOCKED(&sb->sb_sel.si_note, 0);
183 SOCKBUF_UNLOCK(sb);
184 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
185 pgsigio(&so->so_sigio, SIGIO, 0);
186 if (sb->sb_flags & SB_UPCALL)
187 (*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
188 if (sb->sb_flags & SB_AIO)
189 aio_swake(so, sb);
190 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED);
191}
192
193/*
194 * Socket buffer (struct sockbuf) utility routines.
195 *
196 * Each socket contains two socket buffers: one for sending data and one for
197 * receiving data. Each buffer contains a queue of mbufs, information about
198 * the number of mbufs and amount of data in the queue, and other fields
199 * allowing select() statements and notification on data availability to be
200 * implemented.
201 *
202 * Data stored in a socket buffer is maintained as a list of records. Each
203 * record is a list of mbufs chained together with the m_next field. Records
204 * are chained together with the m_nextpkt field. The upper level routine
205 * soreceive() expects the following conventions to be observed when placing
206 * information in the receive buffer:
207 *
208 * 1. If the protocol requires each message be preceded by the sender's name,
209 * then a record containing that name must be present before any
210 * associated data (mbuf's must be of type MT_SONAME).
211 * 2. If the protocol supports the exchange of ``access rights'' (really just
212 * additional data associated with the message), and there are ``rights''
213 * to be received, then a record containing this data should be present
214 * (mbuf's must be of type MT_RIGHTS).
215 * 3. If a name or rights record exists, then it must be followed by a data
216 * record, perhaps of zero length.
217 *
218 * Before using a new socket structure it is first necessary to reserve
219 * buffer space to the socket, by calling sbreserve(). This should commit
220 * some of the available buffer space in the system buffer pool for the
221 * socket (currently, it does nothing but enforce limits). The space should
222 * be released by calling sbrelease() when the socket is destroyed.
223 */
224int
225soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
226{
227 struct thread *td = curthread;
228
229 SOCKBUF_LOCK(&so->so_snd);
230 SOCKBUF_LOCK(&so->so_rcv);
231 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0)
232 goto bad;
233 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0)
234 goto bad2;
235 if (so->so_rcv.sb_lowat == 0)
236 so->so_rcv.sb_lowat = 1;
237 if (so->so_snd.sb_lowat == 0)
238 so->so_snd.sb_lowat = MCLBYTES;
239 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
240 so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
241 SOCKBUF_UNLOCK(&so->so_rcv);
242 SOCKBUF_UNLOCK(&so->so_snd);
243 return (0);
244bad2:
245 sbrelease_locked(&so->so_snd, so);
246bad:
247 SOCKBUF_UNLOCK(&so->so_rcv);
248 SOCKBUF_UNLOCK(&so->so_snd);
249 return (ENOBUFS);
250}
251
252static int
253sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS)
254{
255 int error = 0;
256 u_long tmp_sb_max = sb_max;
257
258 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req);
259 if (error || !req->newptr)
260 return (error);
261 if (tmp_sb_max < MSIZE + MCLBYTES)
262 return (EINVAL);
263 sb_max = tmp_sb_max;
264 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES);
265 return (0);
266}
267
268/*
269 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't
270 * become limiting if buffering efficiency is near the normal case.
271 */
272int
273sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so,
274 struct thread *td)
275{
276 rlim_t sbsize_limit;
277
278 SOCKBUF_LOCK_ASSERT(sb);
279
280 /*
281 * td will only be NULL when we're in an interrupt (e.g. in
282 * tcp_input()).
283 *
284 * XXXRW: This comment needs updating, as might the code.
285 */
286 if (cc > sb_max_adj)
287 return (0);
288 if (td != NULL) {
289 PROC_LOCK(td->td_proc);
290 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE);
291 PROC_UNLOCK(td->td_proc);
292 } else
293 sbsize_limit = RLIM_INFINITY;
294 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc,
295 sbsize_limit))
296 return (0);
297 sb->sb_mbmax = min(cc * sb_efficiency, sb_max);
298 if (sb->sb_lowat > sb->sb_hiwat)
299 sb->sb_lowat = sb->sb_hiwat;
300 return (1);
301}
302
303int
304sbreserve(struct sockbuf *sb, u_long cc, struct socket *so,
305 struct thread *td)
306{
307 int error;
308
309 SOCKBUF_LOCK(sb);
310 error = sbreserve_locked(sb, cc, so, td);
311 SOCKBUF_UNLOCK(sb);
312 return (error);
313}
314
315/*
316 * Free mbufs held by a socket, and reserved mbuf space.
317 */
318static void
319sbrelease_internal(struct sockbuf *sb, struct socket *so)
320{
321
322 sbflush_internal(sb);
323 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0,
324 RLIM_INFINITY);
325 sb->sb_mbmax = 0;
326}
327
328void
329sbrelease_locked(struct sockbuf *sb, struct socket *so)
330{
331
332 SOCKBUF_LOCK_ASSERT(sb);
333
334 sbrelease_internal(sb, so);
335}
336
337void
338sbrelease(struct sockbuf *sb, struct socket *so)
339{
340
341 SOCKBUF_LOCK(sb);
342 sbrelease_locked(sb, so);
343 SOCKBUF_UNLOCK(sb);
344}
345
346void
347sbdestroy(struct sockbuf *sb, struct socket *so)
348{
349
350 sbrelease_internal(sb, so);
351}
352
353
354/*
355 * Routines to add and remove data from an mbuf queue.
356 *
357 * The routines sbappend() or sbappendrecord() are normally called to append
358 * new mbufs to a socket buffer, after checking that adequate space is
359 * available, comparing the function sbspace() with the amount of data to be
360 * added. sbappendrecord() differs from sbappend() in that data supplied is
361 * treated as the beginning of a new record. To place a sender's address,
362 * optional access rights, and data in a socket receive buffer,
363 * sbappendaddr() should be used. To place access rights and data in a
364 * socket receive buffer, sbappendrights() should be used. In either case,
365 * the new data begins a new record. Note that unlike sbappend() and
366 * sbappendrecord(), these routines check for the caller that there will be
367 * enough space to store the data. Each fails if there is not enough space,
368 * or if it cannot find mbufs to store additional information in.
369 *
370 * Reliable protocols may use the socket send buffer to hold data awaiting
371 * acknowledgement. Data is normally copied from a socket send buffer in a
372 * protocol with m_copy for output to a peer, and then removing the data from
373 * the socket buffer with sbdrop() or sbdroprecord() when the data is
374 * acknowledged by the peer.
375 */
376#ifdef SOCKBUF_DEBUG
377void
378sblastrecordchk(struct sockbuf *sb, const char *file, int line)
379{
380 struct mbuf *m = sb->sb_mb;
381
382 SOCKBUF_LOCK_ASSERT(sb);
383
384 while (m && m->m_nextpkt)
385 m = m->m_nextpkt;
386
387 if (m != sb->sb_lastrecord) {
388 printf("%s: sb_mb %p sb_lastrecord %p last %p\n",
389 __func__, sb->sb_mb, sb->sb_lastrecord, m);
390 printf("packet chain:\n");
391 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
392 printf("\t%p\n", m);
393 panic("%s from %s:%u", __func__, file, line);
394 }
395}
396
397void
398sblastmbufchk(struct sockbuf *sb, const char *file, int line)
399{
400 struct mbuf *m = sb->sb_mb;
401 struct mbuf *n;
402
403 SOCKBUF_LOCK_ASSERT(sb);
404
405 while (m && m->m_nextpkt)
406 m = m->m_nextpkt;
407
408 while (m && m->m_next)
409 m = m->m_next;
410
411 if (m != sb->sb_mbtail) {
412 printf("%s: sb_mb %p sb_mbtail %p last %p\n",
413 __func__, sb->sb_mb, sb->sb_mbtail, m);
414 printf("packet tree:\n");
415 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
416 printf("\t");
417 for (n = m; n != NULL; n = n->m_next)
418 printf("%p ", n);
419 printf("\n");
420 }
421 panic("%s from %s:%u", __func__, file, line);
422 }
423}
424#endif /* SOCKBUF_DEBUG */
425
426#define SBLINKRECORD(sb, m0) do { \
427 SOCKBUF_LOCK_ASSERT(sb); \
428 if ((sb)->sb_lastrecord != NULL) \
429 (sb)->sb_lastrecord->m_nextpkt = (m0); \
430 else \
431 (sb)->sb_mb = (m0); \
432 (sb)->sb_lastrecord = (m0); \
433} while (/*CONSTCOND*/0)
434
435/*
436 * Append mbuf chain m to the last record in the socket buffer sb. The
437 * additional space associated the mbuf chain is recorded in sb. Empty mbufs
438 * are discarded and mbufs are compacted where possible.
439 */
440void
441sbappend_locked(struct sockbuf *sb, struct mbuf *m)
442{
443 struct mbuf *n;
444
445 SOCKBUF_LOCK_ASSERT(sb);
446
447 if (m == 0)
448 return;
449
450 SBLASTRECORDCHK(sb);
451 n = sb->sb_mb;
452 if (n) {
453 while (n->m_nextpkt)
454 n = n->m_nextpkt;
455 do {
456 if (n->m_flags & M_EOR) {
457 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
458 return;
459 }
460 } while (n->m_next && (n = n->m_next));
461 } else {
462 /*
463 * XXX Would like to simply use sb_mbtail here, but
464 * XXX I need to verify that I won't miss an EOR that
465 * XXX way.
466 */
467 if ((n = sb->sb_lastrecord) != NULL) {
468 do {
469 if (n->m_flags & M_EOR) {
470 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
471 return;
472 }
473 } while (n->m_next && (n = n->m_next));
474 } else {
475 /*
476 * If this is the first record in the socket buffer,
477 * it's also the last record.
478 */
479 sb->sb_lastrecord = m;
480 }
481 }
482 sbcompress(sb, m, n);
483 SBLASTRECORDCHK(sb);
484}
485
486/*
487 * Append mbuf chain m to the last record in the socket buffer sb. The
488 * additional space associated the mbuf chain is recorded in sb. Empty mbufs
489 * are discarded and mbufs are compacted where possible.
490 */
491void
492sbappend(struct sockbuf *sb, struct mbuf *m)
493{
494
495 SOCKBUF_LOCK(sb);
496 sbappend_locked(sb, m);
497 SOCKBUF_UNLOCK(sb);
498}
499
500/*
501 * This version of sbappend() should only be used when the caller absolutely
502 * knows that there will never be more than one record in the socket buffer,
503 * that is, a stream protocol (such as TCP).
504 */
505void
506sbappendstream_locked(struct sockbuf *sb, struct mbuf *m)
507{
508 SOCKBUF_LOCK_ASSERT(sb);
509
510 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0"));
511 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1"));
512
513 SBLASTMBUFCHK(sb);
514
515 sbcompress(sb, m, sb->sb_mbtail);
516
517 sb->sb_lastrecord = sb->sb_mb;
518 SBLASTRECORDCHK(sb);
519}
520
521/*
522 * This version of sbappend() should only be used when the caller absolutely
523 * knows that there will never be more than one record in the socket buffer,
524 * that is, a stream protocol (such as TCP).
525 */
526void
527sbappendstream(struct sockbuf *sb, struct mbuf *m)
528{
529
530 SOCKBUF_LOCK(sb);
531 sbappendstream_locked(sb, m);
532 SOCKBUF_UNLOCK(sb);
533}
534
535#ifdef SOCKBUF_DEBUG
536void
537sbcheck(struct sockbuf *sb)
538{
539 struct mbuf *m;
540 struct mbuf *n = 0;
541 u_long len = 0, mbcnt = 0;
542
543 SOCKBUF_LOCK_ASSERT(sb);
544
545 for (m = sb->sb_mb; m; m = n) {
546 n = m->m_nextpkt;
547 for (; m; m = m->m_next) {
548 len += m->m_len;
549 mbcnt += MSIZE;
550 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */
551 mbcnt += m->m_ext.ext_size;
552 }
553 }
554 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
555 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc,
556 mbcnt, sb->sb_mbcnt);
557 panic("sbcheck");
558 }
559}
560#endif
561
562/*
563 * As above, except the mbuf chain begins a new record.
564 */
565void
566sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0)
567{
568 struct mbuf *m;
569
570 SOCKBUF_LOCK_ASSERT(sb);
571
572 if (m0 == 0)
573 return;
574 m = sb->sb_mb;
575 if (m)
576 while (m->m_nextpkt)
577 m = m->m_nextpkt;
578 /*
579 * Put the first mbuf on the queue. Note this permits zero length
580 * records.
581 */
582 sballoc(sb, m0);
583 SBLASTRECORDCHK(sb);
584 SBLINKRECORD(sb, m0);
585 if (m)
586 m->m_nextpkt = m0;
587 else
588 sb->sb_mb = m0;
589 m = m0->m_next;
590 m0->m_next = 0;
591 if (m && (m0->m_flags & M_EOR)) {
592 m0->m_flags &= ~M_EOR;
593 m->m_flags |= M_EOR;
594 }
595 sbcompress(sb, m, m0);
596}
597
598/*
599 * As above, except the mbuf chain begins a new record.
600 */
601void
602sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
603{
604
605 SOCKBUF_LOCK(sb);
606 sbappendrecord_locked(sb, m0);
607 SOCKBUF_UNLOCK(sb);
608}
609
610/*
611 * Append address and data, and optionally, control (ancillary) data to the
612 * receive queue of a socket. If present, m0 must include a packet header
613 * with total length. Returns 0 if no space in sockbuf or insufficient
614 * mbufs.
615 */
616int
617sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa,
618 struct mbuf *m0, struct mbuf *control)
619{
620 struct mbuf *m, *n, *nlast;
621 int space = asa->sa_len;
622
623 SOCKBUF_LOCK_ASSERT(sb);
624
625 if (m0 && (m0->m_flags & M_PKTHDR) == 0)
626 panic("sbappendaddr_locked");
627 if (m0)
628 space += m0->m_pkthdr.len;
629 space += m_length(control, &n);
630
631 if (space > sbspace(sb))
632 return (0);
633#if MSIZE <= 256
634 if (asa->sa_len > MLEN)
635 return (0);
636#endif
637 MGET(m, M_DONTWAIT, MT_SONAME);
638 if (m == 0)
639 return (0);
640 m->m_len = asa->sa_len;
641 bcopy(asa, mtod(m, caddr_t), asa->sa_len);
642 if (n)
643 n->m_next = m0; /* concatenate data to control */
644 else
645 control = m0;
646 m->m_next = control;
647 for (n = m; n->m_next != NULL; n = n->m_next)
648 sballoc(sb, n);
649 sballoc(sb, n);
650 nlast = n;
651 SBLINKRECORD(sb, m);
652
653 sb->sb_mbtail = nlast;
654 SBLASTMBUFCHK(sb);
655
656 SBLASTRECORDCHK(sb);
657 return (1);
658}
659
660/*
661 * Append address and data, and optionally, control (ancillary) data to the
662 * receive queue of a socket. If present, m0 must include a packet header
663 * with total length. Returns 0 if no space in sockbuf or insufficient
664 * mbufs.
665 */
666int
667sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa,
668 struct mbuf *m0, struct mbuf *control)
669{
670 int retval;
671
672 SOCKBUF_LOCK(sb);
673 retval = sbappendaddr_locked(sb, asa, m0, control);
674 SOCKBUF_UNLOCK(sb);
675 return (retval);
676}
677
678int
679sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0,
680 struct mbuf *control)
681{
682 struct mbuf *m, *n, *mlast;
683 int space;
684
685 SOCKBUF_LOCK_ASSERT(sb);
686
687 if (control == 0)
688 panic("sbappendcontrol_locked");
689 space = m_length(control, &n) + m_length(m0, NULL);
690
691 if (space > sbspace(sb))
692 return (0);
693 n->m_next = m0; /* concatenate data to control */
694
695 SBLASTRECORDCHK(sb);
696
697 for (m = control; m->m_next; m = m->m_next)
698 sballoc(sb, m);
699 sballoc(sb, m);
700 mlast = m;
701 SBLINKRECORD(sb, control);
702
703 sb->sb_mbtail = mlast;
704 SBLASTMBUFCHK(sb);
705
706 SBLASTRECORDCHK(sb);
707 return (1);
708}
709
710int
711sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
712{
713 int retval;
714
715 SOCKBUF_LOCK(sb);
716 retval = sbappendcontrol_locked(sb, m0, control);
717 SOCKBUF_UNLOCK(sb);
718 return (retval);
719}
720
721/*
722 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf
723 * (n). If (n) is NULL, the buffer is presumed empty.
724 *
725 * When the data is compressed, mbufs in the chain may be handled in one of
726 * three ways:
727 *
728 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no
729 * record boundary, and no change in data type).
730 *
731 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into
732 * an mbuf already in the socket buffer. This can occur if an
733 * appropriate mbuf exists, there is room, and no merging of data types
734 * will occur.
735 *
736 * (3) The mbuf may be appended to the end of the existing mbuf chain.
737 *
738 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as
739 * end-of-record.
740 */
741void
742sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
743{
744 int eor = 0;
745 struct mbuf *o;
746
747 SOCKBUF_LOCK_ASSERT(sb);
748
749 while (m) {
750 eor |= m->m_flags & M_EOR;
751 if (m->m_len == 0 &&
752 (eor == 0 ||
753 (((o = m->m_next) || (o = n)) &&
754 o->m_type == m->m_type))) {
755 if (sb->sb_lastrecord == m)
756 sb->sb_lastrecord = m->m_next;
757 m = m_free(m);
758 continue;
759 }
760 if (n && (n->m_flags & M_EOR) == 0 &&
761 M_WRITABLE(n) &&
762 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
763 m->m_len <= M_TRAILINGSPACE(n) &&
764 n->m_type == m->m_type) {
765 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len,
766 (unsigned)m->m_len);
767 n->m_len += m->m_len;
768 sb->sb_cc += m->m_len;
769 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
770 /* XXX: Probably don't need.*/
771 sb->sb_ctl += m->m_len;
772 m = m_free(m);
773 continue;
774 }
775 if (n)
776 n->m_next = m;
777 else
778 sb->sb_mb = m;
779 sb->sb_mbtail = m;
780 sballoc(sb, m);
781 n = m;
782 m->m_flags &= ~M_EOR;
783 m = m->m_next;
784 n->m_next = 0;
785 }
786 if (eor) {
787 KASSERT(n != NULL, ("sbcompress: eor && n == NULL"));
788 n->m_flags |= eor;
789 }
790 SBLASTMBUFCHK(sb);
791}
792
793/*
794 * Free all mbufs in a sockbuf. Check that all resources are reclaimed.
795 */
796static void
797sbflush_internal(struct sockbuf *sb)
798{
799
800 if (sb->sb_flags & SB_LOCK)
801 panic("sbflush_internal: locked");
802 while (sb->sb_mbcnt) {
803 /*
804 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
805 * we would loop forever. Panic instead.
806 */
807 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len))
808 break;
809 sbdrop_internal(sb, (int)sb->sb_cc);
810 }
811 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt)
812 panic("sbflush_internal: cc %u || mb %p || mbcnt %u",
813 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt);
814}
815
816void
817sbflush_locked(struct sockbuf *sb)
818{
819
820 SOCKBUF_LOCK_ASSERT(sb);
821 sbflush_internal(sb);
822}
823
824void
825sbflush(struct sockbuf *sb)
826{
827
828 SOCKBUF_LOCK(sb);
829 sbflush_locked(sb);
830 SOCKBUF_UNLOCK(sb);
831}
832
833/*
834 * Drop data from (the front of) a sockbuf.
835 */
836static void
837sbdrop_internal(struct sockbuf *sb, int len)
838{
839 struct mbuf *m;
840 struct mbuf *next;
841
842 next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
843 while (len > 0) {
844 if (m == 0) {
845 if (next == 0)
846 panic("sbdrop");
847 m = next;
848 next = m->m_nextpkt;
849 continue;
850 }
851 if (m->m_len > len) {
852 m->m_len -= len;
853 m->m_data += len;
854 sb->sb_cc -= len;
855 if (sb->sb_sndptroff != 0)
856 sb->sb_sndptroff -= len;
857 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
858 sb->sb_ctl -= len;
859 break;
860 }
861 len -= m->m_len;
862 sbfree(sb, m);
863 m = m_free(m);
864 }
865 while (m && m->m_len == 0) {
866 sbfree(sb, m);
867 m = m_free(m);
868 }
869 if (m) {
870 sb->sb_mb = m;
871 m->m_nextpkt = next;
872 } else
873 sb->sb_mb = next;
874 /*
875 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure
876 * sb_lastrecord is up-to-date if we dropped part of the last record.
877 */
878 m = sb->sb_mb;
879 if (m == NULL) {
880 sb->sb_mbtail = NULL;
881 sb->sb_lastrecord = NULL;
882 } else if (m->m_nextpkt == NULL) {
883 sb->sb_lastrecord = m;
884 }
885}
886
887/*
888 * Drop data from (the front of) a sockbuf.
889 */
890void
891sbdrop_locked(struct sockbuf *sb, int len)
892{
893
894 SOCKBUF_LOCK_ASSERT(sb);
895
896 sbdrop_internal(sb, len);
897}
898
899void
900sbdrop(struct sockbuf *sb, int len)
901{
902
903 SOCKBUF_LOCK(sb);
904 sbdrop_locked(sb, len);
905 SOCKBUF_UNLOCK(sb);
906}
907
908
909/*
910 * Maintain a pointer and offset pair into the socket buffer mbuf chain to
911 * avoid traversal of the entire socket buffer for larger offsets.
912 */
913struct mbuf *
914sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff)
915{
916 struct mbuf *m, *ret;
917
918 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__));
919 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__));
920 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__));
921
922 /*
923 * Is off below stored offset? Happens on retransmits.
924 * Just return, we can't help here.
925 */
926 if (sb->sb_sndptroff > off) {
927 *moff = off;
928 return (sb->sb_mb);
929 }
930
931 /* Return closest mbuf in chain for current offset. */
932 *moff = off - sb->sb_sndptroff;
933 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb;
934
935 /* Advance by len to be as close as possible for the next transmit. */
936 for (off = off - sb->sb_sndptroff + len - 1;
937 off > 0 && off >= m->m_len;
938 m = m->m_next) {
939 sb->sb_sndptroff += m->m_len;
940 off -= m->m_len;
941 }
942 sb->sb_sndptr = m;
943
944 return (ret);
945}
946
947/*
948 * Drop a record off the front of a sockbuf and move the next record to the
949 * front.
950 */
951void
952sbdroprecord_locked(struct sockbuf *sb)
953{
954 struct mbuf *m;
955
956 SOCKBUF_LOCK_ASSERT(sb);
957
958 m = sb->sb_mb;
959 if (m) {
960 sb->sb_mb = m->m_nextpkt;
961 do {
962 sbfree(sb, m);
963 m = m_free(m);
964 } while (m);
965 }
966 SB_EMPTY_FIXUP(sb);
967}
968
969/*
970 * Drop a record off the front of a sockbuf and move the next record to the
971 * front.
972 */
973void
974sbdroprecord(struct sockbuf *sb)
975{
976
977 SOCKBUF_LOCK(sb);
978 sbdroprecord_locked(sb);
979 SOCKBUF_UNLOCK(sb);
980}
981
| 34
35#include "opt_param.h"
36
37#include <sys/param.h>
38#include <sys/aio.h> /* for aio_swake proto */
39#include <sys/kernel.h>
40#include <sys/lock.h>
41#include <sys/mbuf.h>
42#include <sys/mutex.h>
43#include <sys/proc.h>
44#include <sys/protosw.h>
45#include <sys/resourcevar.h>
46#include <sys/signalvar.h>
47#include <sys/socket.h>
48#include <sys/socketvar.h>
49#include <sys/sysctl.h>
50
51/*
52 * Function pointer set by the AIO routines so that the socket buffer code
53 * can call back into the AIO module if it is loaded.
54 */
55void (*aio_swake)(struct socket *, struct sockbuf *);
56
57/*
58 * Primitive routines for operating on socket buffers
59 */
60
61u_long sb_max = SB_MAX;
62static u_long sb_max_adj =
63 SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */
64
65static u_long sb_efficiency = 8; /* parameter for sbreserve() */
66
67static void sbdrop_internal(struct sockbuf *sb, int len);
68static void sbflush_internal(struct sockbuf *sb);
69static void sbrelease_internal(struct sockbuf *sb, struct socket *so);
70
71/*
72 * Socantsendmore indicates that no more data will be sent on the socket; it
73 * would normally be applied to a socket when the user informs the system
74 * that no more data is to be sent, by the protocol code (in case
75 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be
76 * received, and will normally be applied to the socket by a protocol when it
77 * detects that the peer will send no more data. Data queued for reading in
78 * the socket may yet be read.
79 */
80void
81socantsendmore_locked(struct socket *so)
82{
83
84 SOCKBUF_LOCK_ASSERT(&so->so_snd);
85
86 so->so_snd.sb_state |= SBS_CANTSENDMORE;
87 sowwakeup_locked(so);
88 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
89}
90
91void
92socantsendmore(struct socket *so)
93{
94
95 SOCKBUF_LOCK(&so->so_snd);
96 socantsendmore_locked(so);
97 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
98}
99
100void
101socantrcvmore_locked(struct socket *so)
102{
103
104 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
105
106 so->so_rcv.sb_state |= SBS_CANTRCVMORE;
107 sorwakeup_locked(so);
108 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
109}
110
111void
112socantrcvmore(struct socket *so)
113{
114
115 SOCKBUF_LOCK(&so->so_rcv);
116 socantrcvmore_locked(so);
117 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
118}
119
120/*
121 * Wait for data to arrive at/drain from a socket buffer.
122 */
123int
124sbwait(struct sockbuf *sb)
125{
126
127 SOCKBUF_LOCK_ASSERT(sb);
128
129 sb->sb_flags |= SB_WAIT;
130 return (msleep(&sb->sb_cc, &sb->sb_mtx,
131 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
132 sb->sb_timeo));
133}
134
135/*
136 * Lock a sockbuf already known to be locked; return any error returned from
137 * sleep (EINTR).
138 */
139int
140sb_lock(struct sockbuf *sb)
141{
142 int error;
143
144 SOCKBUF_LOCK_ASSERT(sb);
145
146 while (sb->sb_flags & SB_LOCK) {
147 sb->sb_flags |= SB_WANT;
148 error = msleep(&sb->sb_flags, &sb->sb_mtx,
149 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK|PCATCH,
150 "sblock", 0);
151 if (error)
152 return (error);
153 }
154 sb->sb_flags |= SB_LOCK;
155 return (0);
156}
157
158/*
159 * Wakeup processes waiting on a socket buffer. Do asynchronous notification
160 * via SIGIO if the socket has the SS_ASYNC flag set.
161 *
162 * Called with the socket buffer lock held; will release the lock by the end
163 * of the function. This allows the caller to acquire the socket buffer lock
164 * while testing for the need for various sorts of wakeup and hold it through
165 * to the point where it's no longer required. We currently hold the lock
166 * through calls out to other subsystems (with the exception of kqueue), and
167 * then release it to avoid lock order issues. It's not clear that's
168 * correct.
169 */
170void
171sowakeup(struct socket *so, struct sockbuf *sb)
172{
173
174 SOCKBUF_LOCK_ASSERT(sb);
175
176 selwakeuppri(&sb->sb_sel, PSOCK);
177 sb->sb_flags &= ~SB_SEL;
178 if (sb->sb_flags & SB_WAIT) {
179 sb->sb_flags &= ~SB_WAIT;
180 wakeup(&sb->sb_cc);
181 }
182 KNOTE_LOCKED(&sb->sb_sel.si_note, 0);
183 SOCKBUF_UNLOCK(sb);
184 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
185 pgsigio(&so->so_sigio, SIGIO, 0);
186 if (sb->sb_flags & SB_UPCALL)
187 (*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
188 if (sb->sb_flags & SB_AIO)
189 aio_swake(so, sb);
190 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED);
191}
192
193/*
194 * Socket buffer (struct sockbuf) utility routines.
195 *
196 * Each socket contains two socket buffers: one for sending data and one for
197 * receiving data. Each buffer contains a queue of mbufs, information about
198 * the number of mbufs and amount of data in the queue, and other fields
199 * allowing select() statements and notification on data availability to be
200 * implemented.
201 *
202 * Data stored in a socket buffer is maintained as a list of records. Each
203 * record is a list of mbufs chained together with the m_next field. Records
204 * are chained together with the m_nextpkt field. The upper level routine
205 * soreceive() expects the following conventions to be observed when placing
206 * information in the receive buffer:
207 *
208 * 1. If the protocol requires each message be preceded by the sender's name,
209 * then a record containing that name must be present before any
210 * associated data (mbuf's must be of type MT_SONAME).
211 * 2. If the protocol supports the exchange of ``access rights'' (really just
212 * additional data associated with the message), and there are ``rights''
213 * to be received, then a record containing this data should be present
214 * (mbuf's must be of type MT_RIGHTS).
215 * 3. If a name or rights record exists, then it must be followed by a data
216 * record, perhaps of zero length.
217 *
218 * Before using a new socket structure it is first necessary to reserve
219 * buffer space to the socket, by calling sbreserve(). This should commit
220 * some of the available buffer space in the system buffer pool for the
221 * socket (currently, it does nothing but enforce limits). The space should
222 * be released by calling sbrelease() when the socket is destroyed.
223 */
224int
225soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
226{
227 struct thread *td = curthread;
228
229 SOCKBUF_LOCK(&so->so_snd);
230 SOCKBUF_LOCK(&so->so_rcv);
231 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0)
232 goto bad;
233 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0)
234 goto bad2;
235 if (so->so_rcv.sb_lowat == 0)
236 so->so_rcv.sb_lowat = 1;
237 if (so->so_snd.sb_lowat == 0)
238 so->so_snd.sb_lowat = MCLBYTES;
239 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
240 so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
241 SOCKBUF_UNLOCK(&so->so_rcv);
242 SOCKBUF_UNLOCK(&so->so_snd);
243 return (0);
244bad2:
245 sbrelease_locked(&so->so_snd, so);
246bad:
247 SOCKBUF_UNLOCK(&so->so_rcv);
248 SOCKBUF_UNLOCK(&so->so_snd);
249 return (ENOBUFS);
250}
251
252static int
253sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS)
254{
255 int error = 0;
256 u_long tmp_sb_max = sb_max;
257
258 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req);
259 if (error || !req->newptr)
260 return (error);
261 if (tmp_sb_max < MSIZE + MCLBYTES)
262 return (EINVAL);
263 sb_max = tmp_sb_max;
264 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES);
265 return (0);
266}
267
268/*
269 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't
270 * become limiting if buffering efficiency is near the normal case.
271 */
272int
273sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so,
274 struct thread *td)
275{
276 rlim_t sbsize_limit;
277
278 SOCKBUF_LOCK_ASSERT(sb);
279
280 /*
281 * td will only be NULL when we're in an interrupt (e.g. in
282 * tcp_input()).
283 *
284 * XXXRW: This comment needs updating, as might the code.
285 */
286 if (cc > sb_max_adj)
287 return (0);
288 if (td != NULL) {
289 PROC_LOCK(td->td_proc);
290 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE);
291 PROC_UNLOCK(td->td_proc);
292 } else
293 sbsize_limit = RLIM_INFINITY;
294 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc,
295 sbsize_limit))
296 return (0);
297 sb->sb_mbmax = min(cc * sb_efficiency, sb_max);
298 if (sb->sb_lowat > sb->sb_hiwat)
299 sb->sb_lowat = sb->sb_hiwat;
300 return (1);
301}
302
303int
304sbreserve(struct sockbuf *sb, u_long cc, struct socket *so,
305 struct thread *td)
306{
307 int error;
308
309 SOCKBUF_LOCK(sb);
310 error = sbreserve_locked(sb, cc, so, td);
311 SOCKBUF_UNLOCK(sb);
312 return (error);
313}
314
315/*
316 * Free mbufs held by a socket, and reserved mbuf space.
317 */
318static void
319sbrelease_internal(struct sockbuf *sb, struct socket *so)
320{
321
322 sbflush_internal(sb);
323 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0,
324 RLIM_INFINITY);
325 sb->sb_mbmax = 0;
326}
327
328void
329sbrelease_locked(struct sockbuf *sb, struct socket *so)
330{
331
332 SOCKBUF_LOCK_ASSERT(sb);
333
334 sbrelease_internal(sb, so);
335}
336
337void
338sbrelease(struct sockbuf *sb, struct socket *so)
339{
340
341 SOCKBUF_LOCK(sb);
342 sbrelease_locked(sb, so);
343 SOCKBUF_UNLOCK(sb);
344}
345
346void
347sbdestroy(struct sockbuf *sb, struct socket *so)
348{
349
350 sbrelease_internal(sb, so);
351}
352
353
354/*
355 * Routines to add and remove data from an mbuf queue.
356 *
357 * The routines sbappend() or sbappendrecord() are normally called to append
358 * new mbufs to a socket buffer, after checking that adequate space is
359 * available, comparing the function sbspace() with the amount of data to be
360 * added. sbappendrecord() differs from sbappend() in that data supplied is
361 * treated as the beginning of a new record. To place a sender's address,
362 * optional access rights, and data in a socket receive buffer,
363 * sbappendaddr() should be used. To place access rights and data in a
364 * socket receive buffer, sbappendrights() should be used. In either case,
365 * the new data begins a new record. Note that unlike sbappend() and
366 * sbappendrecord(), these routines check for the caller that there will be
367 * enough space to store the data. Each fails if there is not enough space,
368 * or if it cannot find mbufs to store additional information in.
369 *
370 * Reliable protocols may use the socket send buffer to hold data awaiting
371 * acknowledgement. Data is normally copied from a socket send buffer in a
372 * protocol with m_copy for output to a peer, and then removing the data from
373 * the socket buffer with sbdrop() or sbdroprecord() when the data is
374 * acknowledged by the peer.
375 */
376#ifdef SOCKBUF_DEBUG
377void
378sblastrecordchk(struct sockbuf *sb, const char *file, int line)
379{
380 struct mbuf *m = sb->sb_mb;
381
382 SOCKBUF_LOCK_ASSERT(sb);
383
384 while (m && m->m_nextpkt)
385 m = m->m_nextpkt;
386
387 if (m != sb->sb_lastrecord) {
388 printf("%s: sb_mb %p sb_lastrecord %p last %p\n",
389 __func__, sb->sb_mb, sb->sb_lastrecord, m);
390 printf("packet chain:\n");
391 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
392 printf("\t%p\n", m);
393 panic("%s from %s:%u", __func__, file, line);
394 }
395}
396
397void
398sblastmbufchk(struct sockbuf *sb, const char *file, int line)
399{
400 struct mbuf *m = sb->sb_mb;
401 struct mbuf *n;
402
403 SOCKBUF_LOCK_ASSERT(sb);
404
405 while (m && m->m_nextpkt)
406 m = m->m_nextpkt;
407
408 while (m && m->m_next)
409 m = m->m_next;
410
411 if (m != sb->sb_mbtail) {
412 printf("%s: sb_mb %p sb_mbtail %p last %p\n",
413 __func__, sb->sb_mb, sb->sb_mbtail, m);
414 printf("packet tree:\n");
415 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
416 printf("\t");
417 for (n = m; n != NULL; n = n->m_next)
418 printf("%p ", n);
419 printf("\n");
420 }
421 panic("%s from %s:%u", __func__, file, line);
422 }
423}
424#endif /* SOCKBUF_DEBUG */
425
426#define SBLINKRECORD(sb, m0) do { \
427 SOCKBUF_LOCK_ASSERT(sb); \
428 if ((sb)->sb_lastrecord != NULL) \
429 (sb)->sb_lastrecord->m_nextpkt = (m0); \
430 else \
431 (sb)->sb_mb = (m0); \
432 (sb)->sb_lastrecord = (m0); \
433} while (/*CONSTCOND*/0)
434
435/*
436 * Append mbuf chain m to the last record in the socket buffer sb. The
437 * additional space associated the mbuf chain is recorded in sb. Empty mbufs
438 * are discarded and mbufs are compacted where possible.
439 */
440void
441sbappend_locked(struct sockbuf *sb, struct mbuf *m)
442{
443 struct mbuf *n;
444
445 SOCKBUF_LOCK_ASSERT(sb);
446
447 if (m == 0)
448 return;
449
450 SBLASTRECORDCHK(sb);
451 n = sb->sb_mb;
452 if (n) {
453 while (n->m_nextpkt)
454 n = n->m_nextpkt;
455 do {
456 if (n->m_flags & M_EOR) {
457 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
458 return;
459 }
460 } while (n->m_next && (n = n->m_next));
461 } else {
462 /*
463 * XXX Would like to simply use sb_mbtail here, but
464 * XXX I need to verify that I won't miss an EOR that
465 * XXX way.
466 */
467 if ((n = sb->sb_lastrecord) != NULL) {
468 do {
469 if (n->m_flags & M_EOR) {
470 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */
471 return;
472 }
473 } while (n->m_next && (n = n->m_next));
474 } else {
475 /*
476 * If this is the first record in the socket buffer,
477 * it's also the last record.
478 */
479 sb->sb_lastrecord = m;
480 }
481 }
482 sbcompress(sb, m, n);
483 SBLASTRECORDCHK(sb);
484}
485
486/*
487 * Append mbuf chain m to the last record in the socket buffer sb. The
488 * additional space associated the mbuf chain is recorded in sb. Empty mbufs
489 * are discarded and mbufs are compacted where possible.
490 */
491void
492sbappend(struct sockbuf *sb, struct mbuf *m)
493{
494
495 SOCKBUF_LOCK(sb);
496 sbappend_locked(sb, m);
497 SOCKBUF_UNLOCK(sb);
498}
499
500/*
501 * This version of sbappend() should only be used when the caller absolutely
502 * knows that there will never be more than one record in the socket buffer,
503 * that is, a stream protocol (such as TCP).
504 */
505void
506sbappendstream_locked(struct sockbuf *sb, struct mbuf *m)
507{
508 SOCKBUF_LOCK_ASSERT(sb);
509
510 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0"));
511 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1"));
512
513 SBLASTMBUFCHK(sb);
514
515 sbcompress(sb, m, sb->sb_mbtail);
516
517 sb->sb_lastrecord = sb->sb_mb;
518 SBLASTRECORDCHK(sb);
519}
520
521/*
522 * This version of sbappend() should only be used when the caller absolutely
523 * knows that there will never be more than one record in the socket buffer,
524 * that is, a stream protocol (such as TCP).
525 */
526void
527sbappendstream(struct sockbuf *sb, struct mbuf *m)
528{
529
530 SOCKBUF_LOCK(sb);
531 sbappendstream_locked(sb, m);
532 SOCKBUF_UNLOCK(sb);
533}
534
535#ifdef SOCKBUF_DEBUG
536void
537sbcheck(struct sockbuf *sb)
538{
539 struct mbuf *m;
540 struct mbuf *n = 0;
541 u_long len = 0, mbcnt = 0;
542
543 SOCKBUF_LOCK_ASSERT(sb);
544
545 for (m = sb->sb_mb; m; m = n) {
546 n = m->m_nextpkt;
547 for (; m; m = m->m_next) {
548 len += m->m_len;
549 mbcnt += MSIZE;
550 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */
551 mbcnt += m->m_ext.ext_size;
552 }
553 }
554 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
555 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc,
556 mbcnt, sb->sb_mbcnt);
557 panic("sbcheck");
558 }
559}
560#endif
561
562/*
563 * As above, except the mbuf chain begins a new record.
564 */
565void
566sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0)
567{
568 struct mbuf *m;
569
570 SOCKBUF_LOCK_ASSERT(sb);
571
572 if (m0 == 0)
573 return;
574 m = sb->sb_mb;
575 if (m)
576 while (m->m_nextpkt)
577 m = m->m_nextpkt;
578 /*
579 * Put the first mbuf on the queue. Note this permits zero length
580 * records.
581 */
582 sballoc(sb, m0);
583 SBLASTRECORDCHK(sb);
584 SBLINKRECORD(sb, m0);
585 if (m)
586 m->m_nextpkt = m0;
587 else
588 sb->sb_mb = m0;
589 m = m0->m_next;
590 m0->m_next = 0;
591 if (m && (m0->m_flags & M_EOR)) {
592 m0->m_flags &= ~M_EOR;
593 m->m_flags |= M_EOR;
594 }
595 sbcompress(sb, m, m0);
596}
597
598/*
599 * As above, except the mbuf chain begins a new record.
600 */
601void
602sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
603{
604
605 SOCKBUF_LOCK(sb);
606 sbappendrecord_locked(sb, m0);
607 SOCKBUF_UNLOCK(sb);
608}
609
610/*
611 * Append address and data, and optionally, control (ancillary) data to the
612 * receive queue of a socket. If present, m0 must include a packet header
613 * with total length. Returns 0 if no space in sockbuf or insufficient
614 * mbufs.
615 */
616int
617sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa,
618 struct mbuf *m0, struct mbuf *control)
619{
620 struct mbuf *m, *n, *nlast;
621 int space = asa->sa_len;
622
623 SOCKBUF_LOCK_ASSERT(sb);
624
625 if (m0 && (m0->m_flags & M_PKTHDR) == 0)
626 panic("sbappendaddr_locked");
627 if (m0)
628 space += m0->m_pkthdr.len;
629 space += m_length(control, &n);
630
631 if (space > sbspace(sb))
632 return (0);
633#if MSIZE <= 256
634 if (asa->sa_len > MLEN)
635 return (0);
636#endif
637 MGET(m, M_DONTWAIT, MT_SONAME);
638 if (m == 0)
639 return (0);
640 m->m_len = asa->sa_len;
641 bcopy(asa, mtod(m, caddr_t), asa->sa_len);
642 if (n)
643 n->m_next = m0; /* concatenate data to control */
644 else
645 control = m0;
646 m->m_next = control;
647 for (n = m; n->m_next != NULL; n = n->m_next)
648 sballoc(sb, n);
649 sballoc(sb, n);
650 nlast = n;
651 SBLINKRECORD(sb, m);
652
653 sb->sb_mbtail = nlast;
654 SBLASTMBUFCHK(sb);
655
656 SBLASTRECORDCHK(sb);
657 return (1);
658}
659
660/*
661 * Append address and data, and optionally, control (ancillary) data to the
662 * receive queue of a socket. If present, m0 must include a packet header
663 * with total length. Returns 0 if no space in sockbuf or insufficient
664 * mbufs.
665 */
666int
667sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa,
668 struct mbuf *m0, struct mbuf *control)
669{
670 int retval;
671
672 SOCKBUF_LOCK(sb);
673 retval = sbappendaddr_locked(sb, asa, m0, control);
674 SOCKBUF_UNLOCK(sb);
675 return (retval);
676}
677
678int
679sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0,
680 struct mbuf *control)
681{
682 struct mbuf *m, *n, *mlast;
683 int space;
684
685 SOCKBUF_LOCK_ASSERT(sb);
686
687 if (control == 0)
688 panic("sbappendcontrol_locked");
689 space = m_length(control, &n) + m_length(m0, NULL);
690
691 if (space > sbspace(sb))
692 return (0);
693 n->m_next = m0; /* concatenate data to control */
694
695 SBLASTRECORDCHK(sb);
696
697 for (m = control; m->m_next; m = m->m_next)
698 sballoc(sb, m);
699 sballoc(sb, m);
700 mlast = m;
701 SBLINKRECORD(sb, control);
702
703 sb->sb_mbtail = mlast;
704 SBLASTMBUFCHK(sb);
705
706 SBLASTRECORDCHK(sb);
707 return (1);
708}
709
710int
711sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
712{
713 int retval;
714
715 SOCKBUF_LOCK(sb);
716 retval = sbappendcontrol_locked(sb, m0, control);
717 SOCKBUF_UNLOCK(sb);
718 return (retval);
719}
720
721/*
722 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf
723 * (n). If (n) is NULL, the buffer is presumed empty.
724 *
725 * When the data is compressed, mbufs in the chain may be handled in one of
726 * three ways:
727 *
728 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no
729 * record boundary, and no change in data type).
730 *
731 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into
732 * an mbuf already in the socket buffer. This can occur if an
733 * appropriate mbuf exists, there is room, and no merging of data types
734 * will occur.
735 *
736 * (3) The mbuf may be appended to the end of the existing mbuf chain.
737 *
738 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as
739 * end-of-record.
740 */
741void
742sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
743{
744 int eor = 0;
745 struct mbuf *o;
746
747 SOCKBUF_LOCK_ASSERT(sb);
748
749 while (m) {
750 eor |= m->m_flags & M_EOR;
751 if (m->m_len == 0 &&
752 (eor == 0 ||
753 (((o = m->m_next) || (o = n)) &&
754 o->m_type == m->m_type))) {
755 if (sb->sb_lastrecord == m)
756 sb->sb_lastrecord = m->m_next;
757 m = m_free(m);
758 continue;
759 }
760 if (n && (n->m_flags & M_EOR) == 0 &&
761 M_WRITABLE(n) &&
762 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
763 m->m_len <= M_TRAILINGSPACE(n) &&
764 n->m_type == m->m_type) {
765 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len,
766 (unsigned)m->m_len);
767 n->m_len += m->m_len;
768 sb->sb_cc += m->m_len;
769 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
770 /* XXX: Probably don't need.*/
771 sb->sb_ctl += m->m_len;
772 m = m_free(m);
773 continue;
774 }
775 if (n)
776 n->m_next = m;
777 else
778 sb->sb_mb = m;
779 sb->sb_mbtail = m;
780 sballoc(sb, m);
781 n = m;
782 m->m_flags &= ~M_EOR;
783 m = m->m_next;
784 n->m_next = 0;
785 }
786 if (eor) {
787 KASSERT(n != NULL, ("sbcompress: eor && n == NULL"));
788 n->m_flags |= eor;
789 }
790 SBLASTMBUFCHK(sb);
791}
792
793/*
794 * Free all mbufs in a sockbuf. Check that all resources are reclaimed.
795 */
796static void
797sbflush_internal(struct sockbuf *sb)
798{
799
800 if (sb->sb_flags & SB_LOCK)
801 panic("sbflush_internal: locked");
802 while (sb->sb_mbcnt) {
803 /*
804 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
805 * we would loop forever. Panic instead.
806 */
807 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len))
808 break;
809 sbdrop_internal(sb, (int)sb->sb_cc);
810 }
811 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt)
812 panic("sbflush_internal: cc %u || mb %p || mbcnt %u",
813 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt);
814}
815
816void
817sbflush_locked(struct sockbuf *sb)
818{
819
820 SOCKBUF_LOCK_ASSERT(sb);
821 sbflush_internal(sb);
822}
823
824void
825sbflush(struct sockbuf *sb)
826{
827
828 SOCKBUF_LOCK(sb);
829 sbflush_locked(sb);
830 SOCKBUF_UNLOCK(sb);
831}
832
833/*
834 * Drop data from (the front of) a sockbuf.
835 */
836static void
837sbdrop_internal(struct sockbuf *sb, int len)
838{
839 struct mbuf *m;
840 struct mbuf *next;
841
842 next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
843 while (len > 0) {
844 if (m == 0) {
845 if (next == 0)
846 panic("sbdrop");
847 m = next;
848 next = m->m_nextpkt;
849 continue;
850 }
851 if (m->m_len > len) {
852 m->m_len -= len;
853 m->m_data += len;
854 sb->sb_cc -= len;
855 if (sb->sb_sndptroff != 0)
856 sb->sb_sndptroff -= len;
857 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
858 sb->sb_ctl -= len;
859 break;
860 }
861 len -= m->m_len;
862 sbfree(sb, m);
863 m = m_free(m);
864 }
865 while (m && m->m_len == 0) {
866 sbfree(sb, m);
867 m = m_free(m);
868 }
869 if (m) {
870 sb->sb_mb = m;
871 m->m_nextpkt = next;
872 } else
873 sb->sb_mb = next;
874 /*
875 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure
876 * sb_lastrecord is up-to-date if we dropped part of the last record.
877 */
878 m = sb->sb_mb;
879 if (m == NULL) {
880 sb->sb_mbtail = NULL;
881 sb->sb_lastrecord = NULL;
882 } else if (m->m_nextpkt == NULL) {
883 sb->sb_lastrecord = m;
884 }
885}
886
887/*
888 * Drop data from (the front of) a sockbuf.
889 */
890void
891sbdrop_locked(struct sockbuf *sb, int len)
892{
893
894 SOCKBUF_LOCK_ASSERT(sb);
895
896 sbdrop_internal(sb, len);
897}
898
899void
900sbdrop(struct sockbuf *sb, int len)
901{
902
903 SOCKBUF_LOCK(sb);
904 sbdrop_locked(sb, len);
905 SOCKBUF_UNLOCK(sb);
906}
907
908
909/*
910 * Maintain a pointer and offset pair into the socket buffer mbuf chain to
911 * avoid traversal of the entire socket buffer for larger offsets.
912 */
913struct mbuf *
914sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff)
915{
916 struct mbuf *m, *ret;
917
918 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__));
919 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__));
920 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__));
921
922 /*
923 * Is off below stored offset? Happens on retransmits.
924 * Just return, we can't help here.
925 */
926 if (sb->sb_sndptroff > off) {
927 *moff = off;
928 return (sb->sb_mb);
929 }
930
931 /* Return closest mbuf in chain for current offset. */
932 *moff = off - sb->sb_sndptroff;
933 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb;
934
935 /* Advance by len to be as close as possible for the next transmit. */
936 for (off = off - sb->sb_sndptroff + len - 1;
937 off > 0 && off >= m->m_len;
938 m = m->m_next) {
939 sb->sb_sndptroff += m->m_len;
940 off -= m->m_len;
941 }
942 sb->sb_sndptr = m;
943
944 return (ret);
945}
946
947/*
948 * Drop a record off the front of a sockbuf and move the next record to the
949 * front.
950 */
951void
952sbdroprecord_locked(struct sockbuf *sb)
953{
954 struct mbuf *m;
955
956 SOCKBUF_LOCK_ASSERT(sb);
957
958 m = sb->sb_mb;
959 if (m) {
960 sb->sb_mb = m->m_nextpkt;
961 do {
962 sbfree(sb, m);
963 m = m_free(m);
964 } while (m);
965 }
966 SB_EMPTY_FIXUP(sb);
967}
968
969/*
970 * Drop a record off the front of a sockbuf and move the next record to the
971 * front.
972 */
973void
974sbdroprecord(struct sockbuf *sb)
975{
976
977 SOCKBUF_LOCK(sb);
978 sbdroprecord_locked(sb);
979 SOCKBUF_UNLOCK(sb);
980}
981
|