ifq.c revision 1.10 
1/*	$OpenBSD: ifq.c,v 1.10 2017/05/03 03:14:32 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq {
57	struct mbuf_list	 pq_lists[IFQ_NQUEUES];
58};
59
60/*
61 * ifqueue serialiser
62 */
63
64void	ifq_start_task(void *);
65void	ifq_restart_task(void *);
66void	ifq_barrier_task(void *);
67
68#define TASK_ONQUEUE 0x1
69
70void
71ifq_serialize(struct ifqueue *ifq, struct task *t)
72{
73	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
74	struct task work;
75
76	if (ISSET(t->t_flags, TASK_ONQUEUE))
77		return;
78
79	mtx_enter(&ifq->ifq_task_mtx);
80	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
81		SET(t->t_flags, TASK_ONQUEUE);
82		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
83	}
84
85	if (ifq->ifq_serializer == NULL) {
86		ifq->ifq_serializer = curcpu();
87
88		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
89			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
90			CLR(t->t_flags, TASK_ONQUEUE);
91			work = *t; /* copy to caller to avoid races */
92
93			mtx_leave(&ifq->ifq_task_mtx);
94
95			(*work.t_func)(work.t_arg);
96
97			mtx_enter(&ifq->ifq_task_mtx);
98		}
99
100		/*
101		 * ifq->ifq_free is only modified by dequeue, which
102		 * is only called from within this serialization
103		 * context. it is therefore safe to access and modify
104		 * here without taking ifq->ifq_mtx.
105		 */
106		ml = ifq->ifq_free;
107		ml_init(&ifq->ifq_free);
108
109		ifq->ifq_serializer = NULL;
110	}
111	mtx_leave(&ifq->ifq_task_mtx);
112
113	ml_purge(&ml);
114}
115
116int
117ifq_is_serialized(struct ifqueue *ifq)
118{
119	return (ifq->ifq_serializer == curcpu());
120}
121
122void
123ifq_start_task(void *p)
124{
125	struct ifqueue *ifq = p;
126	struct ifnet *ifp = ifq->ifq_if;
127
128	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
129	    ifq_empty(ifq) || ifq_is_oactive(ifq))
130		return;
131
132	ifp->if_qstart(ifq);
133}
134
135void
136ifq_restart_task(void *p)
137{
138	struct ifqueue *ifq = p;
139	struct ifnet *ifp = ifq->ifq_if;
140
141	ifq_clr_oactive(ifq);
142	ifp->if_qstart(ifq);
143}
144
145void
146ifq_barrier(struct ifqueue *ifq)
147{
148	struct sleep_state sls;
149	unsigned int notdone = 1;
150	struct task t = TASK_INITIALIZER(ifq_barrier_task, &notdone);
151
152	/* this should only be called from converted drivers */
153	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
154
155	if (ifq->ifq_serializer == NULL)
156		return;
157
158	ifq_serialize(ifq, &t);
159
160	while (notdone) {
161		sleep_setup(&sls, &notdone, PWAIT, "ifqbar");
162		sleep_finish(&sls, notdone);
163	}
164}
165
166void
167ifq_barrier_task(void *p)
168{
169	unsigned int *notdone = p;
170
171	*notdone = 0;
172	wakeup_one(notdone);
173}
174
175/*
176 * ifqueue mbuf queue API
177 */
178
179void
180ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
181{
182	ifq->ifq_if = ifp;
183	ifq->ifq_softc = NULL;
184
185	mtx_init(&ifq->ifq_mtx, IPL_NET);
186	ifq->ifq_qdrops = 0;
187
188	/* default to priq */
189	ifq->ifq_ops = &priq_ops;
190	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
191
192	ml_init(&ifq->ifq_free);
193	ifq->ifq_len = 0;
194
195	ifq->ifq_packets = 0;
196	ifq->ifq_bytes = 0;
197	ifq->ifq_qdrops = 0;
198	ifq->ifq_errors = 0;
199	ifq->ifq_mcasts = 0;
200
201	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
202	TAILQ_INIT(&ifq->ifq_task_list);
203	ifq->ifq_serializer = NULL;
204
205	task_set(&ifq->ifq_start, ifq_start_task, ifq);
206	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
207
208	if (ifq->ifq_maxlen == 0)
209		ifq_set_maxlen(ifq, IFQ_MAXLEN);
210
211	ifq->ifq_idx = idx;
212}
213
214void
215ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
216{
217	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
218	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
219	struct mbuf *m;
220	const struct ifq_ops *oldops;
221	void *newq, *oldq;
222
223	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
224
225	mtx_enter(&ifq->ifq_mtx);
226	ifq->ifq_ops->ifqop_purge(ifq, &ml);
227	ifq->ifq_len = 0;
228
229	oldops = ifq->ifq_ops;
230	oldq = ifq->ifq_q;
231
232	ifq->ifq_ops = newops;
233	ifq->ifq_q = newq;
234
235	while ((m = ml_dequeue(&ml)) != NULL) {
236		m = ifq->ifq_ops->ifqop_enq(ifq, m);
237		if (m != NULL) {
238			ifq->ifq_qdrops++;
239			ml_enqueue(&free_ml, m);
240		} else
241			ifq->ifq_len++;
242	}
243	mtx_leave(&ifq->ifq_mtx);
244
245	oldops->ifqop_free(ifq->ifq_idx, oldq);
246
247	ml_purge(&free_ml);
248}
249
250void
251ifq_destroy(struct ifqueue *ifq)
252{
253	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
254
255	/* don't need to lock because this is the last use of the ifq */
256
257	ifq->ifq_ops->ifqop_purge(ifq, &ml);
258	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
259
260	ml_purge(&ml);
261}
262
263int
264ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
265{
266	struct mbuf *dm;
267
268	mtx_enter(&ifq->ifq_mtx);
269	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
270	if (dm != m) {
271		ifq->ifq_packets++;
272		ifq->ifq_bytes += m->m_pkthdr.len;
273		if (ISSET(m->m_flags, M_MCAST))
274			ifq->ifq_mcasts++;
275	}
276
277	if (dm == NULL)
278		ifq->ifq_len++;
279	else
280		ifq->ifq_qdrops++;
281	mtx_leave(&ifq->ifq_mtx);
282
283	if (dm != NULL)
284		m_freem(dm);
285
286	return (dm == m ? ENOBUFS : 0);
287}
288
289struct mbuf *
290ifq_deq_begin(struct ifqueue *ifq)
291{
292	struct mbuf *m = NULL;
293	void *cookie;
294
295	mtx_enter(&ifq->ifq_mtx);
296	if (ifq->ifq_len == 0 ||
297	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
298		mtx_leave(&ifq->ifq_mtx);
299		return (NULL);
300	}
301
302	m->m_pkthdr.ph_cookie = cookie;
303
304	return (m);
305}
306
307void
308ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
309{
310	void *cookie;
311
312	KASSERT(m != NULL);
313	cookie = m->m_pkthdr.ph_cookie;
314
315	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
316	ifq->ifq_len--;
317	mtx_leave(&ifq->ifq_mtx);
318}
319
320void
321ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
322{
323	KASSERT(m != NULL);
324
325	mtx_leave(&ifq->ifq_mtx);
326}
327
328struct mbuf *
329ifq_dequeue(struct ifqueue *ifq)
330{
331	struct mbuf *m;
332
333	m = ifq_deq_begin(ifq);
334	if (m == NULL)
335		return (NULL);
336
337	ifq_deq_commit(ifq, m);
338
339	return (m);
340}
341
342unsigned int
343ifq_purge(struct ifqueue *ifq)
344{
345	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
346	unsigned int rv;
347
348	mtx_enter(&ifq->ifq_mtx);
349	ifq->ifq_ops->ifqop_purge(ifq, &ml);
350	rv = ifq->ifq_len;
351	ifq->ifq_len = 0;
352	ifq->ifq_qdrops += rv;
353	mtx_leave(&ifq->ifq_mtx);
354
355	KASSERT(rv == ml_len(&ml));
356
357	ml_purge(&ml);
358
359	return (rv);
360}
361
362void *
363ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
364{
365	mtx_enter(&ifq->ifq_mtx);
366	if (ifq->ifq_ops == ops)
367		return (ifq->ifq_q);
368
369	mtx_leave(&ifq->ifq_mtx);
370
371	return (NULL);
372}
373
374void
375ifq_q_leave(struct ifqueue *ifq, void *q)
376{
377	KASSERT(q == ifq->ifq_q);
378	mtx_leave(&ifq->ifq_mtx);
379}
380
381void
382ifq_mfreem(struct ifqueue *ifq, struct mbuf *m)
383{
384	IFQ_ASSERT_SERIALIZED(ifq);
385
386	ifq->ifq_qdrops++;
387	ml_enqueue(&ifq->ifq_free, m);
388}
389
390/*
391 * priq implementation
392 */
393
394unsigned int
395priq_idx(unsigned int nqueues, const struct mbuf *m)
396{
397	unsigned int flow = 0;
398
399	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
400		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
401
402	return (flow % nqueues);
403}
404
405void *
406priq_alloc(unsigned int idx, void *null)
407{
408	struct priq *pq;
409	int i;
410
411	pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK);
412	for (i = 0; i < IFQ_NQUEUES; i++)
413		ml_init(&pq->pq_lists[i]);
414	return (pq);
415}
416
417void
418priq_free(unsigned int idx, void *pq)
419{
420	free(pq, M_DEVBUF, sizeof(struct priq));
421}
422
423struct mbuf *
424priq_enq(struct ifqueue *ifq, struct mbuf *m)
425{
426	struct priq *pq;
427	struct mbuf_list *pl;
428	struct mbuf *n = NULL;
429	unsigned int prio;
430
431	pq = ifq->ifq_q;
432	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
433
434	/* Find a lower priority queue to drop from */
435	if (ifq_len(ifq) >= ifq->ifq_maxlen) {
436		for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) {
437			pl = &pq->pq_lists[prio];
438			if (ml_len(pl) > 0) {
439				n = ml_dequeue(pl);
440				goto enqueue;
441			}
442		}
443		/*
444		 * There's no lower priority queue that we can
445		 * drop from so don't enqueue this one.
446		 */
447		return (m);
448	}
449
450 enqueue:
451	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
452	ml_enqueue(pl, m);
453
454	return (n);
455}
456
457struct mbuf *
458priq_deq_begin(struct ifqueue *ifq, void **cookiep)
459{
460	struct priq *pq = ifq->ifq_q;
461	struct mbuf_list *pl;
462	unsigned int prio = nitems(pq->pq_lists);
463	struct mbuf *m;
464
465	do {
466		pl = &pq->pq_lists[--prio];
467		m = MBUF_LIST_FIRST(pl);
468		if (m != NULL) {
469			*cookiep = pl;
470			return (m);
471		}
472	} while (prio > 0);
473
474	return (NULL);
475}
476
477void
478priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
479{
480	struct mbuf_list *pl = cookie;
481
482	KASSERT(MBUF_LIST_FIRST(pl) == m);
483
484	ml_dequeue(pl);
485}
486
487void
488priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
489{
490	struct priq *pq = ifq->ifq_q;
491	struct mbuf_list *pl;
492	unsigned int prio = nitems(pq->pq_lists);
493
494	do {
495		pl = &pq->pq_lists[--prio];
496		ml_enlist(ml, pl);
497	} while (prio > 0);
498}
499