1/*- 2 * Copyright (c) 1982, 1986, 1988, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 16 unchanged lines hidden (view full) --- 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 30 */ 31 32#include <sys/cdefs.h> |
33__FBSDID("$FreeBSD: head/sys/kern/uipc_mbuf.c 151976 2005-11-02 16:20:36Z andre $"); |
34 35#include "opt_mac.h" 36#include "opt_param.h" 37#include "opt_mbuf_stress_test.h" 38 39#include <sys/param.h> 40#include <sys/systm.h> 41#include <sys/kernel.h> --- 39 unchanged lines hidden (view full) --- 81 &m_defraguseless, 0, ""); 82SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD, 83 &m_defragfailure, 0, ""); 84SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, 85 &m_defragrandomfailures, 0, ""); 86#endif 87 88/* |
89 * Allocate a given length worth of mbufs and/or clusters (whatever fits 90 * best) and return a pointer to the top of the allocated chain. If an 91 * existing mbuf chain is provided, then we will append the new chain 92 * to the existing one but still return the top of the newly allocated 93 * chain. 94 */ 95struct mbuf * 96m_getm(struct mbuf *m, int len, int how, short type) --- 85 unchanged lines hidden (view full) --- 182 * 183 * Returns: 184 * Nothing. 185 */ 186void 187m_extadd(struct mbuf *mb, caddr_t buf, u_int size, 188 void (*freef)(void *, void *), void *args, int flags, int type) 189{ |
190 KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); |
191 |
192 if (type != EXT_EXTREF) 193 mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT); |
194 if (mb->m_ext.ref_cnt != NULL) { 195 *(mb->m_ext.ref_cnt) = 1; 196 mb->m_flags |= (M_EXT | flags); 197 mb->m_ext.ext_buf = buf; 198 mb->m_data = mb->m_ext.ext_buf; 199 mb->m_ext.ext_size = size; 200 mb->m_ext.ext_free = freef; 201 mb->m_ext.ext_args = args; 202 mb->m_ext.ext_type = type; 203 } 204} 205 206/* 207 * Non-directly-exported function to clean up after mbufs with M_EXT |
208 * storage attached to them if the reference count hits 1. |
209 */ 210void 211mb_free_ext(struct mbuf *m) 212{ |
213 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 214 KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); |
215 |
216 /* Free attached storage if this mbuf is the only reference to it. */ 217 if (*(m->m_ext.ref_cnt) == 1 || 218 atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) { 219 switch (m->m_ext.ext_type) { 220 case EXT_CLUSTER: |
221 uma_zfree(zone_pack, m); |
222 return; /* Job done. */ 223 break; 224 case EXT_JUMBO9: 225 uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 226 break; 227 case EXT_JUMBO16: 228 uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 229 break; 230 case EXT_SFBUF: 231 case EXT_NET_DRV: 232 case EXT_MOD_TYPE: 233 case EXT_DISPOSABLE: 234 *(m->m_ext.ref_cnt) = 0; 235 uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, 236 m->m_ext.ref_cnt)); 237 /* FALLTHROUGH */ 238 case EXT_EXTREF: 239 KASSERT(m->m_ext.ext_free != NULL, 240 ("%s: ext_free not set", __func__)); |
241 (*(m->m_ext.ext_free))(m->m_ext.ext_buf, 242 m->m_ext.ext_args); |
243 break; 244 default: 245 KASSERT(m->m_ext.ext_type == 0, 246 ("%s: unknown ext_type", __func__)); |
247 } 248 } |
249 /* 250 * Free this mbuf back to the mbuf zone with all m_ext 251 * information purged. 252 */ 253 m->m_ext.ext_buf = NULL; 254 m->m_ext.ext_free = NULL; 255 m->m_ext.ext_args = NULL; 256 m->m_ext.ref_cnt = NULL; 257 m->m_ext.ext_size = 0; 258 m->m_ext.ext_type = 0; 259 m->m_flags &= ~M_EXT; |
260 uma_zfree(zone_mbuf, m); 261} 262 263/* |
264 * Attach the the cluster from *m to *n, set up m_ext in *n 265 * and bump the refcount of the cluster. 266 */ 267static void 268mb_dupcl(struct mbuf *n, struct mbuf *m) 269{ 270 KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 271 KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); 272 KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 273 274 if (*(m->m_ext.ref_cnt) == 1) 275 *(m->m_ext.ref_cnt) += 1; 276 else 277 atomic_add_int(m->m_ext.ref_cnt, 1); 278 n->m_ext.ext_buf = m->m_ext.ext_buf; 279 n->m_ext.ext_free = m->m_ext.ext_free; 280 n->m_ext.ext_args = m->m_ext.ext_args; 281 n->m_ext.ext_size = m->m_ext.ext_size; 282 n->m_ext.ref_cnt = m->m_ext.ref_cnt; 283 n->m_ext.ext_type = m->m_ext.ext_type; 284 n->m_flags |= M_EXT; 285} 286 287/* |
288 * Clean up mbuf (chain) from any tags and packet headers. 289 * If "all" is set then the first mbuf in the chain will be 290 * cleaned too. 291 */ 292void 293m_demote(struct mbuf *m0, int all) 294{ 295 struct mbuf *m; --- 247 unchanged lines hidden (view full) --- 543 n->m_pkthdr.len -= off0; 544 else 545 n->m_pkthdr.len = len; 546 copyhdr = 0; 547 } 548 n->m_len = min(len, m->m_len - off); 549 if (m->m_flags & M_EXT) { 550 n->m_data = m->m_data + off; |
551 mb_dupcl(n, m); |
552 } else 553 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 554 (u_int)n->m_len); 555 if (len != M_COPYALL) 556 len -= n->m_len; 557 off = 0; 558 m = m->m_next; 559 np = &n->m_next; --- 25 unchanged lines hidden (view full) --- 585 bcopy(s, t, (size_t)len); 586 return 0; 587} 588 589struct mbuf * 590m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, 591 int prep, int how) 592{ |
593 struct mbuf *mm, *x, *z, *prev = NULL; |
594 caddr_t p; |
595 int i, nlen = 0; |
596 caddr_t buf[MLEN]; 597 598 KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); 599 KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); 600 KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); 601 KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); 602 |
603 mm = m; 604 if (!prep) { 605 while(mm->m_next) { 606 prev = mm; 607 mm = mm->m_next; |
608 } 609 } |
610 for (z = n; z != NULL; z = z->m_next) 611 nlen += z->m_len; 612 if (len == M_COPYALL) 613 len = nlen - off; 614 if (off + len > nlen || len < 1) 615 return NULL; 616 |
617 if (!M_WRITABLE(mm)) { 618 /* XXX: Use proper m_xxx function instead. */ 619 x = m_getcl(how, MT_DATA, mm->m_flags); 620 if (x == NULL) 621 return NULL; 622 bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); 623 p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf); 624 x->m_data = p; 625 mm->m_next = NULL; 626 if (mm != m) 627 prev->m_next = x; 628 m_free(mm); 629 mm = x; 630 } 631 |
632 /* 633 * Append/prepend the data. Allocating mbufs as necessary. 634 */ 635 /* Shortcut if enough free space in first/last mbuf. */ 636 if (!prep && M_TRAILINGSPACE(mm) >= len) { 637 m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + 638 mm->m_len); 639 mm->m_len += len; --- 96 unchanged lines hidden (view full) --- 736 if (n == NULL) 737 goto nospace; 738 739 if (!m_dup_pkthdr(n, m, how)) 740 goto nospace; 741 n->m_len = m->m_len; 742 if (m->m_flags & M_EXT) { 743 n->m_data = m->m_data; |
744 mb_dupcl(n, m); |
745 } else { 746 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 747 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 748 } 749 750 m = m->m_next; 751 while (m) { 752 MGET(o, how, m->m_type); 753 if (o == NULL) 754 goto nospace; 755 756 n->m_next = o; 757 n = n->m_next; 758 759 n->m_len = m->m_len; 760 if (m->m_flags & M_EXT) { 761 n->m_data = m->m_data; |
762 mb_dupcl(n, m); |
763 } else { 764 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 765 } 766 767 m = m->m_next; 768 } 769 return top; 770nospace: --- 366 unchanged lines hidden (view full) --- 1137 } else { 1138 MGET(n, wait, m->m_type); 1139 if (n == NULL) 1140 return (NULL); 1141 M_ALIGN(n, remain); 1142 } 1143extpacket: 1144 if (m->m_flags & M_EXT) { |
1145 n->m_data = m->m_data + len; |
1146 mb_dupcl(n, m); |
1147 } else { 1148 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 1149 } 1150 n->m_len = remain; 1151 m->m_len = len; 1152 n->m_next = m->m_next; 1153 m->m_next = NULL; 1154 return (n); --- 518 unchanged lines hidden --- |