ifq.c revision 1.1 
1/*	$OpenBSD: ifq.c,v 1.1 2015/12/08 10:06:12 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24#include <sys/atomic.h>
25
26#include <net/if.h>
27#include <net/if_var.h>
28
29/*
30 * priq glue
31 */
32void		*priq_alloc(void *);
33void		 priq_free(void *);
34int		 priq_enq(struct ifqueue *, struct mbuf *);
35struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
36void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
37void		 priq_purge(struct ifqueue *, struct mbuf_list *);
38
39const struct ifq_ops priq_ops = {
40	priq_alloc,
41	priq_free,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46};
47
48const struct ifq_ops * const ifq_priq_ops = &priq_ops;
49
50/*
51 * priq internal structures
52 */
53
54struct priq_list {
55	struct mbuf		*head;
56	struct mbuf		*tail;
57};
58
59struct priq {
60	struct priq_list	 pq_lists[IFQ_NQUEUES];
61};
62
63/*
64 * ifqueue serialiser
65 */
66
67static inline unsigned int
68ifq_enter(struct ifqueue *ifq)
69{
70	return (atomic_inc_int_nv(&ifq->ifq_serializer) == 1);
71}
72
73static inline unsigned int
74ifq_leave(struct ifqueue *ifq)
75{
76	if (atomic_cas_uint(&ifq->ifq_serializer, 1, 0) == 1)
77		return (1);
78
79	ifq->ifq_serializer = 1;
80
81	return (0);
82}
83
84void
85if_start_mpsafe(struct ifnet *ifp)
86{
87	struct ifqueue *ifq = &ifp->if_snd;
88
89	if (!ifq_enter(ifq))
90		return;
91
92	do {
93		if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) {
94			ifq->ifq_serializer = 0;
95			wakeup_one(&ifq->ifq_serializer);
96			return;
97		}
98
99		if (ifq_empty(ifq) || ifq_is_oactive(ifq))
100			continue;
101
102		ifp->if_start(ifp);
103
104	} while (!ifq_leave(ifq));
105}
106
107void
108if_start_barrier(struct ifnet *ifp)
109{
110	struct sleep_state sls;
111	struct ifqueue *ifq = &ifp->if_snd;
112
113	/* this should only be called from converted drivers */
114	KASSERT(ISSET(ifp->if_xflags, IFXF_MPSAFE));
115
116	/* drivers should only call this on the way down */
117	KASSERT(!ISSET(ifp->if_flags, IFF_RUNNING));
118
119	if (ifq->ifq_serializer == 0)
120		return;
121
122	if_start_mpsafe(ifp); /* spin the wheel to guarantee a wakeup */
123	do {
124		sleep_setup(&sls, &ifq->ifq_serializer, PWAIT, "ifbar");
125		sleep_finish(&sls, ifq->ifq_serializer != 0);
126	} while (ifq->ifq_serializer != 0);
127}
128
129/*
130 * ifqueue mbuf queue API
131 */
132
133void
134ifq_init(struct ifqueue *ifq)
135{
136	mtx_init(&ifq->ifq_mtx, IPL_NET);
137	ifq->ifq_drops = 0;
138
139	/* default to priq */
140	ifq->ifq_ops = &priq_ops;
141	ifq->ifq_q = priq_ops.ifqop_alloc(NULL);
142
143	ifq->ifq_serializer = 0;
144	ifq->ifq_len = 0;
145
146	if (ifq->ifq_maxlen == 0)
147		ifq_set_maxlen(ifq, IFQ_MAXLEN);
148}
149
150void
151ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
152{
153	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
154	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
155	struct mbuf *m;
156	const struct ifq_ops *oldops;
157	void *newq, *oldq;
158
159	newq = newops->ifqop_alloc(opsarg);
160
161	mtx_enter(&ifq->ifq_mtx);
162	ifq->ifq_ops->ifqop_purge(ifq, &ml);
163	ifq->ifq_len = 0;
164
165	oldops = ifq->ifq_ops;
166	oldq = ifq->ifq_q;
167
168	ifq->ifq_ops = newops;
169	ifq->ifq_q = newq;
170
171	while ((m = ml_dequeue(&ml)) != NULL) {
172		if (ifq->ifq_ops->ifqop_enq(ifq, m) != 0) {
173			ifq->ifq_drops++;
174			ml_enqueue(&free_ml, m);
175		} else
176			ifq->ifq_len++;
177	}
178	mtx_leave(&ifq->ifq_mtx);
179
180	oldops->ifqop_free(oldq);
181
182	ml_purge(&free_ml);
183}
184
185void
186ifq_destroy(struct ifqueue *ifq)
187{
188	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
189
190	/* don't need to lock because this is the last use of the ifq */
191
192	ifq->ifq_ops->ifqop_purge(ifq, &ml);
193	ifq->ifq_ops->ifqop_free(ifq->ifq_q);
194
195	ml_purge(&ml);
196}
197
198int
199ifq_enqueue_try(struct ifqueue *ifq, struct mbuf *m)
200{
201	int rv;
202
203	mtx_enter(&ifq->ifq_mtx);
204	rv = ifq->ifq_ops->ifqop_enq(ifq, m);
205	if (rv == 0)
206		ifq->ifq_len++;
207	else
208		ifq->ifq_drops++;
209	mtx_leave(&ifq->ifq_mtx);
210
211	return (rv);
212}
213
214int
215ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
216{
217	int err;
218
219	err = ifq_enqueue_try(ifq, m);
220	if (err != 0)
221		m_freem(m);
222
223	return (err);
224}
225
226struct mbuf *
227ifq_deq_begin(struct ifqueue *ifq)
228{
229	struct mbuf *m = NULL;
230	void *cookie;
231
232	mtx_enter(&ifq->ifq_mtx);
233	if (ifq->ifq_len == 0 ||
234	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
235		mtx_leave(&ifq->ifq_mtx);
236		return (NULL);
237	}
238
239	m->m_pkthdr.ph_cookie = cookie;
240
241	return (m);
242}
243
244void
245ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
246{
247	void *cookie;
248
249	KASSERT(m != NULL);
250	cookie = m->m_pkthdr.ph_cookie;
251
252	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
253	ifq->ifq_len--;
254	mtx_leave(&ifq->ifq_mtx);
255}
256
257void
258ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
259{
260	KASSERT(m != NULL);
261
262	mtx_leave(&ifq->ifq_mtx);
263}
264
265struct mbuf *
266ifq_dequeue(struct ifqueue *ifq)
267{
268	struct mbuf *m;
269
270	m = ifq_deq_begin(ifq);
271	if (m == NULL)
272		return (NULL);
273
274	ifq_deq_commit(ifq, m);
275
276	return (m);
277}
278
279unsigned int
280ifq_purge(struct ifqueue *ifq)
281{
282	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
283	unsigned int rv;
284
285	mtx_enter(&ifq->ifq_mtx);
286	ifq->ifq_ops->ifqop_purge(ifq, &ml);
287	rv = ifq->ifq_len;
288	ifq->ifq_len = 0;
289	ifq->ifq_drops += rv;
290	mtx_leave(&ifq->ifq_mtx);
291
292	KASSERT(rv == ml_len(&ml));
293
294	ml_purge(&ml);
295
296	return (rv);
297}
298
299void *
300ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
301{
302	mtx_enter(&ifq->ifq_mtx);
303	if (ifq->ifq_ops == ops)
304		return (ifq->ifq_q);
305
306	mtx_leave(&ifq->ifq_mtx);
307
308	return (NULL);
309}
310
311void
312ifq_q_leave(struct ifqueue *ifq, void *q)
313{
314	KASSERT(q == ifq->ifq_q);
315	mtx_leave(&ifq->ifq_mtx);
316}
317
318/*
319 * priq implementation
320 */
321
322void *
323priq_alloc(void *null)
324{
325	return (malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK | M_ZERO));
326}
327
328void
329priq_free(void *pq)
330{
331	free(pq, M_DEVBUF, sizeof(struct priq));
332}
333
334int
335priq_enq(struct ifqueue *ifq, struct mbuf *m)
336{
337	struct priq *pq;
338	struct priq_list *pl;
339
340	if (ifq_len(ifq) >= ifq->ifq_maxlen)
341		return (ENOBUFS);
342
343	pq = ifq->ifq_q;
344	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
345	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
346
347	m->m_nextpkt = NULL;
348	if (pl->tail == NULL)
349		pl->head = m;
350	else
351		pl->tail->m_nextpkt = m;
352	pl->tail = m;
353
354	return (0);
355}
356
357struct mbuf *
358priq_deq_begin(struct ifqueue *ifq, void **cookiep)
359{
360	struct priq *pq = ifq->ifq_q;
361	struct priq_list *pl;
362	unsigned int prio = nitems(pq->pq_lists);
363	struct mbuf *m;
364
365	do {
366		pl = &pq->pq_lists[--prio];
367		m = pl->head;
368		if (m != NULL) {
369			*cookiep = pl;
370			return (m);
371		}
372	} while (prio > 0);
373
374	return (NULL);
375}
376
377void
378priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
379{
380	struct priq_list *pl = cookie;
381
382	KASSERT(pl->head == m);
383
384	pl->head = m->m_nextpkt;
385	m->m_nextpkt = NULL;
386
387	if (pl->head == NULL)
388		pl->tail = NULL;
389}
390
391void
392priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
393{
394	struct priq *pq = ifq->ifq_q;
395	struct priq_list *pl;
396	unsigned int prio = nitems(pq->pq_lists);
397	struct mbuf *m, *n;
398
399	do {
400		pl = &pq->pq_lists[--prio];
401
402		for (m = pl->head; m != NULL; m = n) {
403			n = m->m_nextpkt;
404			ml_enqueue(ml, m);
405		}
406
407		pl->head = pl->tail = NULL;
408	} while (prio > 0);
409}
410