1/* 2 * Copyright 2007, Hugo Santos. All Rights Reserved. 3 * Copyright 2004, Marcus Overhagen. All Rights Reserved. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8#include "device.h" 9 10#include <stdint.h> 11#include <string.h> 12#include <slab/Slab.h> 13 14#include <compat/sys/malloc.h> 15#include <compat/sys/mbuf.h> 16#include <compat/sys/kernel.h> 17 18 19static object_cache *sMBufCache; 20static object_cache *sChunkCache; 21static object_cache *sJumbo9ChunkCache; 22static object_cache *sJumboPageSizeCache; 23 24 25int max_linkhdr = 16; 26int max_protohdr = 40 + 20; /* ip6 + tcp */ 27 28/* max_linkhdr + max_protohdr, but that's not allowed by gcc. */ 29int max_hdr = 16 + 40 + 20; 30 31/* MHLEN - max_hdr */ 32int max_datalen = MHLEN - (16 + 40 + 20); 33 34 35static int 36m_to_oc_flags(int how) 37{ 38 if (how & M_NOWAIT) 39 return CACHE_DONT_WAIT_FOR_MEMORY; 40 41 return 0; 42} 43 44 45int 46m_init(struct mbuf *m, int how, short type, int flags) 47{ 48 int error; 49 50 if (type == MT_NOINIT) 51 return 0; 52 53 m->m_next = NULL; 54 m->m_nextpkt = NULL; 55 m->m_data = m->m_dat; 56 m->m_len = 0; 57 m->m_flags = flags; 58 m->m_type = type; 59 if (flags & M_PKTHDR) 60 error = m_pkthdr_init(m, how); 61 else 62 error = 0; 63 64 return (error); 65} 66 67 68static void* 69allocate_ext_buf(int how, int size, int* ext_type) 70{ 71 object_cache *cache; 72 int extType; 73 if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE) 74 panic("unsupported size"); 75 76 if (size == MCLBYTES) { 77 cache = sChunkCache; 78 extType = EXT_CLUSTER; 79 } else if (size == MJUM9BYTES) { 80 cache = sJumbo9ChunkCache; 81 extType = EXT_JUMBO9; 82 } else { 83 cache = sJumboPageSizeCache; 84 extType = EXT_JUMBOP; 85 } 86 87 if (ext_type != NULL) 88 *ext_type = extType; 89 return object_cache_alloc(cache, m_to_oc_flags(how)); 90} 91 92 93static int 94construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size) 95{ 96 int extType; 97 98 memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType); 99 if (memoryBuffer->m_ext.ext_buf == NULL) 100 return B_NO_MEMORY; 101 102 memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf; 103 memoryBuffer->m_flags |= M_EXT; 104 memoryBuffer->m_ext.ext_size = size; 105 memoryBuffer->m_ext.ext_type = extType; 106 memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF; 107 memoryBuffer->m_ext.ext_count = 1; 108 109 return 0; 110} 111 112 113static inline int 114construct_ext_mbuf(struct mbuf *memoryBuffer, int how) 115{ 116 return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES); 117} 118 119 120static int 121construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags) 122{ 123 if (m_init(memoryBuffer, how, type, flags) < 0) 124 return -1; 125 if (construct_ext_mbuf(memoryBuffer, how) < 0) 126 return -1; 127 memoryBuffer->m_ext.ext_type = EXT_CLUSTER; 128 return 0; 129} 130 131 132struct mbuf * 133m_getcl(int how, short type, int flags) 134{ 135 struct mbuf *memoryBuffer = 136 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 137 if (memoryBuffer == NULL) 138 return NULL; 139 140 if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) { 141 object_cache_free(sMBufCache, memoryBuffer, 0); 142 return NULL; 143 } 144 145 return memoryBuffer; 146} 147 148 149static struct mbuf * 150_m_get(int how, short type, int flags) 151{ 152 struct mbuf *memoryBuffer = 153 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 154 if (memoryBuffer == NULL) 155 return NULL; 156 157 m_init(memoryBuffer, how, type, flags); 158 159 return memoryBuffer; 160} 161 162 163struct mbuf * 164m_get(int how, short type) 165{ 166 return _m_get(how, type, 0); 167} 168 169 170struct mbuf * 171m_get2(int size, int how, short type, int flags) 172{ 173 if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) { 174 return _m_get(how, type, flags); 175 } else if (size <= MCLBYTES) { 176 size = MCLBYTES; 177 } else if (size <= MJUMPAGESIZE) { 178 size = MJUMPAGESIZE; 179 } else if (size <= MJUM9BYTES) { 180 size = MJUM9BYTES; 181 } else /* (size > MJUM9BYTES) */ { 182 return NULL; 183 } 184 185 return m_getjcl(how, type, flags, size); 186} 187 188 189struct mbuf * 190m_gethdr(int how, short type) 191{ 192 return _m_get(how, type, M_PKTHDR); 193} 194 195 196struct mbuf * 197m_getjcl(int how, short type, int flags, int size) 198{ 199 struct mbuf *memoryBuffer = 200 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 201 if (memoryBuffer == NULL) 202 return NULL; 203 if (m_init(memoryBuffer, how, type, flags) < 0) { 204 object_cache_free(sMBufCache, memoryBuffer, 0); 205 return NULL; 206 } 207 if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) { 208 object_cache_free(sMBufCache, memoryBuffer, 0); 209 return NULL; 210 } 211 return memoryBuffer; 212} 213 214 215int 216m_clget(struct mbuf *memoryBuffer, int how) 217{ 218 memoryBuffer->m_ext.ext_buf = NULL; 219 /* called checks for errors by looking for M_EXT */ 220 construct_ext_mbuf(memoryBuffer, how); 221 return memoryBuffer->m_flags & M_EXT; 222} 223 224 225void* 226m_cljget(struct mbuf* memoryBuffer, int how, int size) 227{ 228 if (memoryBuffer == NULL) 229 return allocate_ext_buf(how, size, NULL); 230 231 memoryBuffer->m_ext.ext_buf = NULL; 232 construct_ext_sized_mbuf(memoryBuffer, how, size); 233 return memoryBuffer->m_ext.ext_buf; 234} 235 236 237static void 238mb_free_ext(struct mbuf *memoryBuffer) 239{ 240 volatile u_int *refcnt; 241 struct mbuf *mref; 242 int freembuf; 243 244 KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p", 245 __func__, memoryBuffer)); 246 247 /* See if this is the mbuf that holds the embedded refcount. */ 248 if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) { 249 refcnt = &memoryBuffer->m_ext.ext_count; 250 mref = memoryBuffer; 251 } else { 252 KASSERT(memoryBuffer->m_ext.ext_cnt != NULL, 253 ("%s: no refcounting pointer on %p", __func__, memoryBuffer)); 254 refcnt = memoryBuffer->m_ext.ext_cnt; 255 mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 256 } 257 258 /* 259 * Check if the header is embedded in the cluster. It is 260 * important that we can't touch any of the mbuf fields 261 * after we have freed the external storage, since mbuf 262 * could have been embedded in it. For now, the mbufs 263 * embedded into the cluster are always of type EXT_EXTREF, 264 * and for this type we won't free the mref. 265 */ 266 if (memoryBuffer->m_flags & M_NOFREE) { 267 freembuf = 0; 268 KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF, 269 ("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer)); 270 } else 271 freembuf = 1; 272 273 /* Free attached storage only if this mbuf is the only reference to it. */ 274 if (*refcnt == 1 || atomic_add((int32*)refcnt, -1) == 1) { 275 object_cache *cache = NULL; 276 277 if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER) 278 cache = sChunkCache; 279 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9) 280 cache = sJumbo9ChunkCache; 281 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP) 282 cache = sJumboPageSizeCache; 283 else 284 panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type); 285 286 object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0); 287 object_cache_free(sMBufCache, mref, 0); 288 } 289 290 if (freembuf && memoryBuffer != mref) 291 object_cache_free(sMBufCache, memoryBuffer, 0); 292} 293 294 295struct mbuf * 296m_free(struct mbuf* memoryBuffer) 297{ 298 struct mbuf* next = memoryBuffer->m_next; 299 300 if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) 301 m_tag_delete_chain(memoryBuffer, NULL); 302 if (memoryBuffer->m_flags & M_EXT) 303 mb_free_ext(memoryBuffer); 304 else if ((memoryBuffer->m_flags & M_NOFREE) == 0) 305 object_cache_free(sMBufCache, memoryBuffer, 0); 306 307 return next; 308} 309 310 311status_t 312init_mbufs() 313{ 314 sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL); 315 if (sMBufCache == NULL) 316 goto clean; 317 sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL, 318 NULL); 319 if (sChunkCache == NULL) 320 goto clean; 321 sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0, 322 NULL, NULL, NULL); 323 if (sJumbo9ChunkCache == NULL) 324 goto clean; 325 sJumboPageSizeCache = create_object_cache("mbuf jumbo page size chunks", 326 MJUMPAGESIZE, 0, NULL, NULL, NULL); 327 if (sJumboPageSizeCache == NULL) 328 goto clean; 329 return B_OK; 330 331clean: 332 if (sJumbo9ChunkCache != NULL) 333 delete_object_cache(sJumbo9ChunkCache); 334 if (sChunkCache != NULL) 335 delete_object_cache(sChunkCache); 336 if (sMBufCache != NULL) 337 delete_object_cache(sMBufCache); 338 return B_NO_MEMORY; 339} 340 341 342void 343uninit_mbufs() 344{ 345 delete_object_cache(sMBufCache); 346 delete_object_cache(sChunkCache); 347 delete_object_cache(sJumbo9ChunkCache); 348 delete_object_cache(sJumboPageSizeCache); 349} 350