1/*
2 * Copyright 2007, Hugo Santos. All Rights Reserved.
3 * Copyright 2004, Marcus Overhagen. All Rights Reserved.
4 * Distributed under the terms of the MIT License.
5 */
6
7
8#include "device.h"
9
10#include <stdint.h>
11#include <string.h>
12#include <slab/Slab.h>
13
14#include <compat/sys/malloc.h>
15#include <compat/sys/mbuf.h>
16#include <compat/sys/kernel.h>
17
18
19static object_cache *sMBufCache;
20static object_cache *sChunkCache;
21static object_cache *sJumbo9ChunkCache;
22static object_cache *sJumboPageSizeCache;
23
24
25int max_linkhdr = 16;
26int max_protohdr = 40 + 20; /* ip6 + tcp */
27
28/* max_linkhdr + max_protohdr, but that's not allowed by gcc. */
29int max_hdr = 16 + 40 + 20;
30
31/* MHLEN - max_hdr */
32int max_datalen = MHLEN - (16 + 40 + 20);
33
34
35static int
36m_to_oc_flags(int how)
37{
38	if (how & M_NOWAIT)
39		return CACHE_DONT_WAIT_FOR_MEMORY;
40
41	return 0;
42}
43
44
45int
46m_init(struct mbuf *m, int how, short type, int flags)
47{
48	int error;
49
50	if (type == MT_NOINIT)
51		return 0;
52
53	m->m_next = NULL;
54	m->m_nextpkt = NULL;
55	m->m_data = m->m_dat;
56	m->m_len = 0;
57	m->m_flags = flags;
58	m->m_type = type;
59	if (flags & M_PKTHDR)
60		error = m_pkthdr_init(m, how);
61	else
62		error = 0;
63
64	return (error);
65}
66
67
68static void*
69allocate_ext_buf(int how, int size, int* ext_type)
70{
71	object_cache *cache;
72	int extType;
73	if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE)
74		panic("unsupported size");
75
76	if (size == MCLBYTES) {
77		cache = sChunkCache;
78		extType = EXT_CLUSTER;
79	} else if (size == MJUM9BYTES) {
80		cache = sJumbo9ChunkCache;
81		extType = EXT_JUMBO9;
82	} else {
83		cache = sJumboPageSizeCache;
84		extType = EXT_JUMBOP;
85	}
86
87	if (ext_type != NULL)
88		*ext_type = extType;
89	return object_cache_alloc(cache, m_to_oc_flags(how));
90}
91
92
93static int
94construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size)
95{
96	int extType;
97
98	memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType);
99	if (memoryBuffer->m_ext.ext_buf == NULL)
100		return B_NO_MEMORY;
101
102	memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf;
103	memoryBuffer->m_flags |= M_EXT;
104	memoryBuffer->m_ext.ext_size = size;
105	memoryBuffer->m_ext.ext_type = extType;
106	memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF;
107	memoryBuffer->m_ext.ext_count = 1;
108
109	return 0;
110}
111
112
113static inline int
114construct_ext_mbuf(struct mbuf *memoryBuffer, int how)
115{
116	return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES);
117}
118
119
120static int
121construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags)
122{
123	if (m_init(memoryBuffer, how, type, flags) < 0)
124		return -1;
125	if (construct_ext_mbuf(memoryBuffer, how) < 0)
126		return -1;
127	memoryBuffer->m_ext.ext_type = EXT_CLUSTER;
128	return 0;
129}
130
131
132struct mbuf *
133m_getcl(int how, short type, int flags)
134{
135	struct mbuf *memoryBuffer =
136		(struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how));
137	if (memoryBuffer == NULL)
138		return NULL;
139
140	if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) {
141		object_cache_free(sMBufCache, memoryBuffer, 0);
142		return NULL;
143	}
144
145	return memoryBuffer;
146}
147
148
149static struct mbuf *
150_m_get(int how, short type, int flags)
151{
152	struct mbuf *memoryBuffer =
153		(struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how));
154	if (memoryBuffer == NULL)
155		return NULL;
156
157	m_init(memoryBuffer, how, type, flags);
158
159	return memoryBuffer;
160}
161
162
163struct mbuf *
164m_get(int how, short type)
165{
166	return _m_get(how, type, 0);
167}
168
169
170struct mbuf *
171m_get2(int size, int how, short type, int flags)
172{
173	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) {
174		return _m_get(how, type, flags);
175	} else if (size <= MCLBYTES) {
176		size = MCLBYTES;
177	} else if (size <= MJUMPAGESIZE) {
178		size = MJUMPAGESIZE;
179	} else if (size <= MJUM9BYTES) {
180		size = MJUM9BYTES;
181	} else /* (size > MJUM9BYTES) */ {
182		return NULL;
183	}
184
185	return m_getjcl(how, type, flags, size);
186}
187
188
189struct mbuf *
190m_gethdr(int how, short type)
191{
192	return _m_get(how, type, M_PKTHDR);
193}
194
195
196struct mbuf *
197m_getjcl(int how, short type, int flags, int size)
198{
199	struct mbuf *memoryBuffer =
200		(struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how));
201	if (memoryBuffer == NULL)
202		return NULL;
203	if (m_init(memoryBuffer, how, type, flags) < 0) {
204		object_cache_free(sMBufCache, memoryBuffer, 0);
205		return NULL;
206	}
207	if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) {
208		object_cache_free(sMBufCache, memoryBuffer, 0);
209		return NULL;
210	}
211	return memoryBuffer;
212}
213
214
215int
216m_clget(struct mbuf *memoryBuffer, int how)
217{
218	memoryBuffer->m_ext.ext_buf = NULL;
219	/* called checks for errors by looking for M_EXT */
220	construct_ext_mbuf(memoryBuffer, how);
221	return memoryBuffer->m_flags & M_EXT;
222}
223
224
225void*
226m_cljget(struct mbuf* memoryBuffer, int how, int size)
227{
228	if (memoryBuffer == NULL)
229		return allocate_ext_buf(how, size, NULL);
230
231	memoryBuffer->m_ext.ext_buf = NULL;
232	construct_ext_sized_mbuf(memoryBuffer, how, size);
233	return memoryBuffer->m_ext.ext_buf;
234}
235
236
237static void
238mb_free_ext(struct mbuf *memoryBuffer)
239{
240	volatile u_int *refcnt;
241	struct mbuf *mref;
242	int freembuf;
243
244	KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p",
245		__func__, memoryBuffer));
246
247	/* See if this is the mbuf that holds the embedded refcount. */
248	if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) {
249		refcnt = &memoryBuffer->m_ext.ext_count;
250		mref = memoryBuffer;
251	} else {
252		KASSERT(memoryBuffer->m_ext.ext_cnt != NULL,
253			("%s: no refcounting pointer on %p", __func__, memoryBuffer));
254		refcnt = memoryBuffer->m_ext.ext_cnt;
255		mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
256	}
257
258	/*
259	 * Check if the header is embedded in the cluster.  It is
260	 * important that we can't touch any of the mbuf fields
261	 * after we have freed the external storage, since mbuf
262	 * could have been embedded in it.  For now, the mbufs
263	 * embedded into the cluster are always of type EXT_EXTREF,
264	 * and for this type we won't free the mref.
265	 */
266	if (memoryBuffer->m_flags & M_NOFREE) {
267		freembuf = 0;
268		KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF,
269			("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer));
270	} else
271		freembuf = 1;
272
273	/* Free attached storage only if this mbuf is the only reference to it. */
274	if (*refcnt == 1 || atomic_add((int32*)refcnt, -1) == 1) {
275		object_cache *cache = NULL;
276
277		if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER)
278			cache = sChunkCache;
279		else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9)
280			cache = sJumbo9ChunkCache;
281		else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP)
282			cache = sJumboPageSizeCache;
283		else
284			panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type);
285
286		object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0);
287		object_cache_free(sMBufCache, mref, 0);
288	}
289
290	if (freembuf && memoryBuffer != mref)
291		object_cache_free(sMBufCache, memoryBuffer, 0);
292}
293
294
295struct mbuf *
296m_free(struct mbuf* memoryBuffer)
297{
298	struct mbuf* next = memoryBuffer->m_next;
299
300	if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE))
301		m_tag_delete_chain(memoryBuffer, NULL);
302	if (memoryBuffer->m_flags & M_EXT)
303		mb_free_ext(memoryBuffer);
304	else if ((memoryBuffer->m_flags & M_NOFREE) == 0)
305		object_cache_free(sMBufCache, memoryBuffer, 0);
306
307	return next;
308}
309
310
311status_t
312init_mbufs()
313{
314	sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL);
315	if (sMBufCache == NULL)
316		goto clean;
317	sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL,
318		NULL);
319	if (sChunkCache == NULL)
320		goto clean;
321	sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0,
322		NULL, NULL, NULL);
323	if (sJumbo9ChunkCache == NULL)
324		goto clean;
325	sJumboPageSizeCache = create_object_cache("mbuf jumbo page size chunks",
326		MJUMPAGESIZE, 0, NULL, NULL, NULL);
327	if (sJumboPageSizeCache == NULL)
328		goto clean;
329	return B_OK;
330
331clean:
332	if (sJumbo9ChunkCache != NULL)
333		delete_object_cache(sJumbo9ChunkCache);
334	if (sChunkCache != NULL)
335		delete_object_cache(sChunkCache);
336	if (sMBufCache != NULL)
337		delete_object_cache(sMBufCache);
338	return B_NO_MEMORY;
339}
340
341
342void
343uninit_mbufs()
344{
345	delete_object_cache(sMBufCache);
346	delete_object_cache(sChunkCache);
347	delete_object_cache(sJumbo9ChunkCache);
348	delete_object_cache(sJumboPageSizeCache);
349}
350